artbase 0.0.1 → 0.2.1

data/lib/artbase/collection/base.rb

@@ -0,0 +1,306 @@
+
+module Artbase
+class Base     ## "abstract" Base collection - check -use a different name - why? why not?
+
+
+def convert_images( overwrite: )
+  image_dir = "./#{slug}/token-i"
+  Image.convert( image_dir, from: 'jpg', to: 'png', overwrite: overwrite )
+  Image.convert( image_dir, from: 'gif', to: 'png', overwrite: overwrite )
+  Image.convert( image_dir, from: 'svg', to: 'png', overwrite: overwrite )
+end
+
+
+
+def make_strip
+  composite_count = @count - @excludes.size
+
+  composite = ImageComposite.new( 9, 1,
+                                  width:  @width,
+                                  height: @height )
+
+  i = 0
+  each_image do |img, id|
+    puts "==> [#{i+1}/9] #{id}"
+    composite << img
+
+    i += 1
+    break if i >= 9
+  end
+
+
+  composite.save( "./#{@slug}/tmp/#{@slug}-strip.png" )
+end
+
+
+
+def make_composite
+  ### use well-known / pre-defined (default) grids
+  ##        (cols x rows) for now - why? why not?
+
+  composite_count = @count - @excludes.size
+  cols, rows = case composite_count
+               when    99 then   [10,  10]
+               when   100 then   [10,  10]
+               when   150 then   [15,  10]
+               when   314 then   [15,  21]
+               when   500 then   [25,  20]
+               when  1000 then   [25,  40]
+               when  2200 then   [50,  44]
+               when  2222 then   [50,  45]
+               when  2469 then   [50,  50]
+               when  3000 then   [100, 30]    ## or use 50*60 - why? why not?
+               when  3500 then   [100, 35]    ## or use 50*x ??
+               when  3979 then   [100, 40]
+               when  4000 then   [100, 40]    ## or use 50x80 - why? why not?
+               when  4444 then   [100, 45]    ## or use 50x??
+               when  5000 then   [100, 50]    ## or use 50x100 - why? why not?
+               when  5555 then   [100, 56]   # 5600 (45 left empty)
+               when  6666 then   [100, 67]   # 6700 (34 left empty)
+               when  6688 then   [100, 67]   # 6700 (12 left empty)
+               when  6969 then   [100, 70]   # 7000 (31 left empty)
+               when  8888 then   [100, 89]
+               when  9969 then   [100,100]
+               when 10000 then   [100,100]
+               else
+                   raise ArgumentError, "sorry - unknown composite count #{composite_count}/#{@count} for now"
+               end
+
+  composite = ImageComposite.new( cols, rows,
+                                  width:  @width,
+                                  height: @height )
+
+  each_image do |img, id|
+    puts "==> #{id}"
+    composite << img
+  end
+
+
+
+  composite.save( "./#{@slug}/tmp/#{@slug}-#{@width}x#{@height}.png" )
+
+  if composite_count < 1000
+    composite.zoom(2).save( "./#{@slug}/tmp/#{@slug}-#{@width}x#{@height}@2x.png" )
+  end
+end
+
+
+
+
+def calc_attribute_counters  ## todo/check: use a different name _counts/_stats etc - why? why not?
+
+  attributes_by_count = { count: 0,
+                          by_count: Hash.new(0)
+                        }
+  counter = {}
+
+
+  each_meta do |meta, id|   ## todo/fix: change id to index
+    traits = meta.traits
+    # print "#{traits.size} - "
+    # pp  traits
+
+    print "#{id}.."   if id % 100 == 0  ## print progress report
+
+    attributes_by_count[ :count ] +=1
+    attributes_by_count[ :by_count ][ traits.size ] += 1
+
+    traits.each do |trait_type, trait_value|
+        trait_type  = _normalize_trait_type( trait_type )
+        trait_value = _normalize_trait_value( trait_value )
+
+
+        rec = counter[ trait_type ] ||= { count: 0,
+                                          by_type: Hash.new(0)
+                                        }
+        rec[ :count ] +=1
+        rec[ :by_type ][ trait_value ] += 1
+    end
+  end
+
+  print "\n"
+  puts
+
+  ## return all-in-one hash
+  {
+    total:  attributes_by_count,
+    traits: counter,
+  }
+end
+
+
+def dump_attributes
+  stats = calc_attribute_counters
+
+  total    = stats[:total]
+  counter  = stats[:traits]
+
+  puts
+  puts "attribute usage / counts:"
+  pp total
+  puts
+
+  puts "#{counter.size} attribute(s):"
+  counter.each do |trait_name, trait_rec|
+     puts "  #{trait_name}  #{trait_rec[:count]}  (#{trait_rec[:by_type].size} uniques)"
+  end
+
+  puts
+  pp counter
+end
+
+
+
+
+##  order - allow "custom" attribute order export
+##  renames - allow renames of attributes
+def export_attributes(
+   order: [],
+   renames: {}
+)
+
+  ## step 1: get counters
+  stats = calc_attribute_counters
+
+  total    = stats[:total]
+  counter  = stats[:traits]
+
+  puts
+  puts "attribute usage / counts:"
+  pp total
+  puts
+
+  puts "#{counter.size} attribute(s):"
+  counter.each do |trait_name, trait_rec|
+     puts "  #{trait_name}  #{trait_rec[:count]}  (#{trait_rec[:by_type].size} uniques)"
+  end
+
+
+  trait_names = []
+  trait_names += order    ## get attributes if any in pre-defined order
+  counter.each do |trait_name, _|
+      if trait_names.include?( trait_name )
+         next    ## skip already included
+      else
+         trait_names  << trait_name
+      end
+  end
+
+
+  recs = []
+
+
+  ## step 2: get tabular data
+  each_meta do |meta, id|   ## todo/fix: change id to index
+
+    traits = meta.traits
+    # print "#{traits.size} - "
+    # pp  traits
+
+    print "#{id}.."   if id % 100 == 0  ## print progress report
+
+    ## setup empty hash table (with all attributes)
+    rec = {}
+
+    ## note: use __Slug__& __Name__
+    ##         to avoid conflict with attribute names
+    ##         e.g. attribute with "Name" will overwrite built-in and so on
+
+    rec['__Slug__'] = if respond_to?( :_meta_slugify )
+                       _meta_slugify( meta, id )
+                  else
+                     ## default to id (six digits) as string with leading zeros
+                     ##    for easy sorting using strings
+                     ##   e.g.  1 => '000001'
+                     ##         2 => '000002'
+                       '%06d' % id
+                  end
+
+    rec['__Name__'] = meta.name
+
+    ## add all attributes/traits names/keys
+    trait_names.reduce( rec ) { |h,value| h[value] = []; h }
+    ## pp rec
+
+    ## note: use an array (to allow multiple values for attributes)
+    traits.each do |trait_type, trait_value|
+       trait_type  = _normalize_trait_type( trait_type )
+       trait_value = _normalize_trait_value( trait_value )
+
+       values = rec[ trait_type ]
+       values << trait_value
+    end
+    recs << rec
+  end
+  print "\n"
+
+  ## pp recs
+
+  ## flatten recs
+  data = []
+  recs.each do |rec|
+     row = rec.values.map do |value|
+                  if value.is_a?( Array )
+                     value.join( ' / ' )
+                  else
+                     value
+                  end
+             end
+     data << row
+  end
+
+
+  ## sort by slug
+  data = data.sort {|l,r| l[0] <=> r[0] }
+  pp data
+
+  ### save dataset
+  ##  note: change first colum Slug to ID - only used for "internal" sort etc.
+  headers = ['ID', 'Name']
+  headers += trait_names.map do |trait_name|   ## check for renames
+                                renames[trait_name] || trait_name
+                             end
+
+
+  path = "./#{@slug}/tmp/#{@slug}.csv"
+  dirname = File.dirname( path )
+  FileUtils.mkdir_p( dirname )  unless Dir.exist?( dirname )
+
+  File.open( path, 'w:utf-8' ) do |f|
+    f.write(  headers.join( ', ' ))
+    f.write( "\n" )
+    ## note: replace ID with our own internal running (zero-based) counter
+    data.each_with_index do |row,i|
+      f.write( ([i]+row[1..-1]).join( ', '))
+      f.write( "\n" )
+    end
+  end
+end
+
+
+
+
+#############
+#  "private" helpers
+
+def _normalize_trait_type( trait_type )
+  if @patch && @patch[:trait_types]
+    @patch[:trait_types][ trait_type ] || trait_type
+  else
+     trait_type
+  end
+end
+
+def _normalize_trait_value( trait_value )
+  if @patch && @patch[:trait_values]
+    @patch[:trait_values][ trait_value ] || trait_value
+  else
+    trait_value
+  end
+end
+
+
+
+
+end  # class Base
+end  # module Artbase

data/lib/artbase/collection/image.rb

@@ -0,0 +1,39 @@
+
+
+class ImageCollection
+
+  attr_reader :slug, :count
+
+  def initialize( slug, count,
+                  image_base: )   # check: rename count to items or such - why? why not?
+    @slug = slug
+    @count = count
+    @image_base = image_base
+  end
+
+  def download_images( range=(0...@count) )
+    start = Time.now
+    delay_in_s = 0.3
+
+    range.each do |offset|
+      image_src = @image_base.sub( '{id}', offset.to_s )
+
+      puts "==> #{offset} - #{@slug}..."
+
+      ## note: will auto-add format file extension (e.g. .png, .jpg)
+      ##        depending on http content type!!!!!
+      copy_image( image_src, "./#{@slug}/image-i/#{offset}" )
+
+      stop = Time.now
+      diff = stop - start
+
+      mins = diff / 60  ## todo - use floor or such?
+      secs = diff % 60
+      puts "up #{mins} mins #{secs} secs (total #{diff} secs)"
+
+      puts "sleeping #{delay_in_s}s..."
+      sleep( delay_in_s )
+    end
+  end
+end # class ImageCollection
+

data/lib/artbase/collection/opensea.rb

@@ -0,0 +1,297 @@
+
+
+class Collection    ## todo/check - change to OpenseaCollection or such - why? why not?
+
+attr_reader :slug, :count
+
+# check: rename count to items or such - why? why not?
+#        default format to '24x24' - why? why not?
+def initialize( slug, count,
+                meta_slugify: nil,
+                image_pixelate: nil,
+                patch: nil,
+                exclude: [],
+                format:,
+                source:  )
+  @slug = slug
+  @count = count
+
+  @meta_slugify   = meta_slugify
+  @image_pixelate = image_pixelate
+
+  @patch  = patch
+
+  @exclude = exclude
+
+  @width, @height = _parse_dimension( format )
+
+
+  ## note: allow multiple source formats / dimensions
+  ### e.g. convert   512x512 into  [ [512,512] ]
+  ##
+  source = [source]  unless source.is_a?( Array )
+  @sources = source.map { |dimension| _parse_dimension( dimension ) }
+end
+
+## e.g. convert dimension (width x height) "24x24" or "24 x 24" to  [24,24]
+def _parse_dimension( str )
+   str.split( /x/i ).map { |str| str.strip.to_i }
+end
+
+
+def _image_pixelate( img )
+  if @image_pixelate
+    @image_pixelate.call( img )
+  else
+    @sources.each do |source_width, source_height|
+      if img.width == source_width && img.height == source_height
+         from = "#{source_width}x#{source_height}"
+         to   = "#{@width}x#{@height}"
+         steps = (Image::DOwNSAMPLING_STEPS[ to ] || {})[ from ]
+         if steps.nil?
+           puts "!! ERROR - no sampling steps defined for #{from} to #{to}; sorry"
+           exit 1
+         end
+
+        return img.pixelate( steps )
+      end
+    end
+
+    puts "!! ERROR - unknown image dimension #{img.width}x#{img.height}; sorry"
+    puts "           supported source dimensions include: #{@sources.inspect}"
+    exit 1
+  end
+end
+
+
+
+
+def download_meta( range=(0...@count) )
+  self.class.download_meta( range, @slug )
+end
+
+def download_images( range=(0...@count) )
+  self.class.download_images( range, @slug )
+end
+
+def download( range=(0...@count) )
+  download_meta( range )
+  download_images( range )
+end
+
+
+
+
+
+def _meta_slugify_match( regex, meta, index )
+  if m=regex.match( meta.name )
+    captures = m.named_captures   ## get named captures in match data as hash (keys as strings)
+    # e.g.
+    #=> {"num"=>"3"}
+    #=> {"num"=>"498", "name"=>"Doge"}
+    pp captures
+
+    num  = captures['num']  ? captures['num'].to_i( 10 ) : nil   ## note: add base 10 (e.g. 015=>15)
+    name = captures['name'] ? captures['name'].strip     : nil
+
+    slug = ''
+    if num
+      slug << "%06d" % num    ## todo/check: always fill/zero-pad with six 000000's - why? why not?
+    end
+
+    if name
+      slug << "-"   if num   ## add separator
+      slug << slugify( name )
+    end
+    slug
+  else
+     nil  ## note: return nil if no match / slug
+  end
+end
+
+def _do_meta_slugify( meta_slugify, meta, index )
+  if meta_slugify.is_a?( Regexp )
+    _meta_slugify_match( meta_slugify, meta, index )
+  elsif meta_slugify.is_a?( Proc )
+    meta_slugify.call( meta, index )
+  else
+    raise ArgumentError, "meta_slugify - unsupported type: #{meta_slugify.class.name}"
+  end
+end
+
+
+def _meta_slugify( meta, index )
+  slug = nil
+
+  if @meta_slugify.is_a?( Array )
+      @meta_slugify.each do |meta_slugify|
+         slug = _do_meta_slugify( meta_slugify, meta, index )
+         return slug   if slug     ## note: short-circuit on first match
+                                   ##   use break instead of return - why? why not?
+      end
+   else  ## assume object e.g. Regexp, Proc, etc.
+     slug = _do_meta_slugify( @meta_slugify, meta, index )
+   end
+
+   ## do nothing
+   if slug.nil?
+      puts "!! ERROR - cannot find id in >#{meta.name}<:"
+      pp meta
+      exit 1
+   end
+
+   slug
+end
+
+
+
+def each_meta( range=(0...@count),
+               exclude: true,      &blk )
+  range.each do |id|    ## todo/fix: change id to index
+    meta = OpenSea::Meta.read( "./#{@slug}/meta/#{id}.json" )
+
+    ####
+    # filter out/skip
+    if exclude && @exclude.include?( meta.name )
+      puts "  skipping / exclude #{id} >#{meta.name}<..."
+      next
+    end
+
+    blk.call( meta, id )
+  end
+end
+
+
+
+
+def pixelate( range=(0...@count) )
+
+  meta_slugs = Hash.new( 0 )   ## deduplicate (auto-add counter if duplicate)
+
+  ### todo/fix: must read slugs starting at 0
+  ###               to work for deduplicate!!!!!!
+
+
+  range.each do |id|
+    meta = OpenSea::Meta.read( "./#{@slug}/meta/#{id}.json" )
+
+    ####
+    # filter out/skip
+    if @exclude.include?( meta.name )
+       puts "  skipping / exclude #{id} >#{meta.name}<..."
+       next
+    end
+
+    puts meta.name
+
+
+    meta_slug = _meta_slugify( meta, id )
+    count = meta_slugs[ meta_slug ] += 1
+
+    meta_slug = "#{meta_slug}_(#{count})"  if count > 1
+
+
+    img = Image.read( "./#{@slug}/i/#{id}.png" )
+
+    pix = _image_pixelate( img )
+
+    path = "./#{@slug}/ii/#{meta_slug}.png"
+    puts "   saving to >#{path}<..."
+    pix.save( path )
+  end
+end
+
+
+
+################################
+# private (static) helpers
+#
+
+def self.download_images( range, collection,
+                            original: false )
+  start = Time.now
+  delay_in_s = 0.3
+
+  range.each do |offset|
+    meta = OpenSea::Meta.read( "./#{collection}/meta/#{offset}.json" )
+
+    puts "==> #{offset}.json  -  #{meta.name}"
+
+    image_src =  if original
+                   meta.image_original_url
+                 else
+                   meta.image_url
+                 end
+
+    puts "    >#{image_src}<"
+    if image_src.nil?
+       puts "!! ERROR - no image url found (use original: #{original}):"
+       pp meta
+       exit 1
+    end
+
+    ## note: use a different directory to avoid size confusion!!!
+    img_slug =  if original
+                   'i_org'
+                else
+                   'i'
+                end
+
+    ## note: will auto-add format file extension (e.g. .png, .jpg)
+    ##        depending on http content type!!!!!
+    copy_image( image_src, "./#{collection}/#{img_slug}/#{offset}" )
+
+    stop = Time.now
+    diff = stop - start
+
+    mins = diff / 60  ## todo - use floor or such?
+    secs = diff % 60
+    puts "up #{mins} mins #{secs} secs (total #{diff} secs)"
+
+    puts "sleeping #{delay_in_s}s..."
+    sleep( delay_in_s )
+  end
+end
+
+
+def self.download_meta( range, collection )
+  start = Time.now
+  delay_in_s = 0.3
+
+  range.each do |offset|
+
+    dest = "./#{collection}/meta/#{offset}.json"
+    meta = nil
+
+    puts "==> #{offset} / #{collection} (#{dest})..."
+
+    data = OpenSea.assets( collection: collection,
+                           offset:     offset )
+    meta = OpenSea::Meta.new( data )
+    puts "  name:      >#{meta.name}<"
+    puts "  image_url: >#{meta.image_url}<"
+
+
+    ## make sure path exists
+    dirname = File.dirname( dest )
+    FileUtils.mkdir_p( dirname )  unless Dir.exist?( dirname )
+
+    File.open( dest, "w:utf-8" ) do |f|
+      f.write( JSON.pretty_generate( data ) )
+    end
+
+
+    stop = Time.now
+    diff = stop - start
+
+    mins = diff / 60  ## todo - use floor or such?
+    secs = diff % 60
+    puts "up #{mins} mins #{secs} secs (total #{diff} secs)"
+
+    puts "  sleeping #{delay_in_s}s..."
+    sleep( delay_in_s )
+  end
+end
+
+
+end # class Collection