ordlite 0.1.2 → 0.2.1

data/lib/ordlite/importer.rb

@@ -0,0 +1,218 @@
+module OrdDb
+
+class Importer
+   Inscribe   =  Model::Inscribe
+   Blob       =  Model::Blob
+   Collection =  Model::Collection 
+
+
+
+def import_collection_csv( path,
+                                name:,
+                                content: true )
+## or use
+##  import_collection( format: 'csv') - why? why not?
+   recs = read_csv( path )
+   puts "  #{recs.size} inscribe id(s)"
+
+   col = Collection.find_by( name: name )
+   if col && col.items.count > 0
+     puts "!! WARN - collection already in db; delete first to reimport"
+     return     
+   elsif col
+     ## do nothing; (re)use collection record; add items
+   else
+     col = Collection.create(
+            name:  name
+            ## max:   recs.size   ## auto-add max - why? why not?
+          )
+   end
+
+ recs.each_with_index do |rec,i|
+    id   = rec['id']
+    name = rec['name'] || rec['title']
+    puts "==> #{i+1}/#{recs.size} >#{name}< @ #{id}..."
+
+    col.items.create( pos: i,
+                      inscribe_id: id,
+                      name: name )
+
+    _import( id, content: content )
+ end
+end
+
+
+def import_collection_inscriptions( path, 
+                                         name:, 
+                                         content: true )
+   recs = read_json( path )
+   puts "  #{recs.size} inscribe id(s)"
+
+   col = Collection.find_by( name: name )
+   if col && col.items.count > 0
+     puts "!! WARN - collection already in db; delete first to reimport"
+     return     
+   elsif col
+      ## do nothing; (re)use collection record; add items
+   else
+     col = Model::Collection.create(
+        name:  name  
+        ## max:   recs.size   ## auto-add max - why? why not?
+      )
+   end
+
+   recs.each_with_index do |rec,i|
+      id   = rec['id']
+      meta = rec['meta']
+      name = meta['name']
+      puts "==> #{i+1}/#{recs.size} >#{name}< @ #{id}..."
+ 
+      col.items.create( pos: i,
+                        inscribe_id: id,
+                        name: name )
+
+     _import( id, content: content ) 
+   end
+end
+
+
+def import_collection( path, content: true )
+   data = read_json( path )
+
+   meta =  data['collection']
+   pp meta
+
+   name = meta['name']
+
+   col = Collection.find_by( name: name )
+   if col
+     puts "!! WARN - collection already in db; delete first to reimport"
+     return     
+   end
+   
+   col = Collection.create(
+      name:  name,
+      desc:  meta['description'],
+      max:   meta['max_supply']
+   )
+
+   items = data['items']
+   puts "  #{items.size} inscribe id(s)"
+   items.each_with_index do |rec,i|
+      id   = rec['inscription_id']
+      name = rec['name']
+      puts "==> #{i+1}/#{items.size} @ #{id}..."
+ 
+      col.items.create( pos: i,
+                        inscribe_id: id,
+                        name: name )
+
+     _import( id, content: content )
+   end
+end
+
+
+def import_csv( path, content: true )
+    recs = read_csv( path )
+    puts "  #{recs.size} inscribe id(s)"
+    #=>  1000 inscribe id(s) 
+     
+    recs.each_with_index do |rec,i|
+      id = rec['id']
+      puts "==> #{i+1}/#{rec.size} @ #{id}..."
+    
+      _import( id, content: content )
+    end  
+end # method import_csv
+
+def import( id_or_ids, content: true )
+   if id_or_ids.is_a?( String )
+      id = id_or_ids
+     _import( id, content: content )
+   else  ## assume array
+      ids = id_or_ids
+      ids.each do |id|
+         _import( id, content: content )
+      end
+   end
+end 
+
+def _import( id, content: true )
+   ## check if inscription / inscribe is already in db?
+   inscribe = Inscribe.find_by( id: id )
+   if inscribe  ## already in db; dump record
+     ## pp inscribe
+   else         ## fetch via ordinals.com api and update db
+      data = Ordinals.inscription( id )
+
+      pp data
+      Inscribe.create_from_api( data )
+   end
+
+   if content
+     ## check if (content) blob is already in db?
+     blob = Blob.find_by( id: id )
+     if blob    ## already in db; do nothing
+     else       ## fetch via ordinals.com api and update db
+        content = Ordinals.content( id )
+
+        puts "  content-type: #{content.type}"
+        puts "  content-length: #{content.length}"
+     
+        Blob.create( id: id, content: content.data )
+     end
+   end              
+end
+
+end  # class Importer
+
+
+
+###
+## convenience helpers
+
+def self.importer   ## "default" importer
+   @importer ||= Importer.new 
+end
+
+def self.import( id_or_ids, content: true )
+   importer.import( id_or_ids, content: content )
+end
+
+
+def self.import_csv( path, content: true )
+   importer.import_csv( path, content: content )
+end
+
+
+def self.import_collection( path, content: true )
+   importer.import_collection( path, content: content )
+end
+
+def self.import_collection_inscriptions( path, 
+          name:, 
+          content: true )
+   importer.import_collection_inscriptions( path, 
+          name: name, 
+          content: content )
+end
+
+def self.import_collection_csv( path,
+          name:,
+          content: true )
+    importer.import_collection_csv( path,
+           name: name,
+           content: content )
+end
+
+
+module Model 
+class Inscribe
+   def self.import( id_or_ids, content: true )
+       OrdDb.importer.import( id_or_ids, content: content )
+   end
+end  # class Inscribe
+end # module Model 
+
+
+end  # module OrdDb

data/lib/ordlite/models/blob.rb

@@ -4,6 +4,10 @@ module OrdDb
   
   class Blob < ActiveRecord::Base
     belongs_to :inscribe, foreign_key: 'id'
+
+    def text
+       content.force_encoding(Encoding::UTF_8)
+    end
   end  # class Blob
   
     end # module Model

data/lib/ordlite/models/collection.rb

@@ -0,0 +1,14 @@
+module OrdDb
+    module Model
+  
+  class Collection < ActiveRecord::Base    
+    has_many :items   
+    ##  -> { order('pos') }
+    ##   note: default_scope (order)
+    ##      will break all count queries and more
+    ##      thus - no "magic" - always sort if pos order required!!!
+    has_many :inscribes, :through => :items                    
+  end  # class Collection
+  
+    end # module Model
+end # module OrdDb

data/lib/ordlite/models/factory.rb

@@ -0,0 +1,25 @@
+
+module OrdDb
+    module Model
+  
+  class Factory < ActiveRecord::Base
+    self.table_name = 'factories' ## non-standard plural (factory/factories)
+
+    belongs_to :inscribe
+    
+    has_many :inscriberefs  ## join table (use habtm - why? why not?)
+    ##  -> { order('pos') }
+    ##   note: default_scope (order)
+    ##      will break all count queries and more
+    ##      thus - no "magic" - always sort if pos order required!!!
+   
+    has_many :layers, :through => :inscriberefs,  
+                      :source => :inscribe  
+
+    has_many :generatives
+    has_many :inscribes, :through => :generatives                    
+  end  # class Factory
+  
+    end # module Model
+end # module OrdDb
+  

data/lib/ordlite/models/forward.rb

@@ -12,12 +12,31 @@ Prop      = ConfDb::Model::Prop
 
 class Inscribe    < ActiveRecord::Base ; end
 class Blob        < ActiveRecord::Base ; end
-
-  end
+class Collection  < ActiveRecord::Base ; end
+class Item        < ActiveRecord::Base    ## change to CollectionItem - why? why not? 
+   belongs_to :collection
+   belongs_to :inscribe
+end  
+
+class Factory     < ActiveRecord::Base ; end
+class Generative  < ActiveRecord::Base ; end
+
+### join tables - add inline here - why? why not?
+##   rename to CollectionLayer? or
+##             CollectionInscribeRef? or
+##             Layerref? or
+##             InscribeRef?
+##             FactoryItem ????
+class Inscriberef < ActiveRecord::Base
+  belongs_to :factory
+  belongs_to :inscribe
+end
+
+  end # module Model
 
   # note: convenience alias for Model
   # lets you use include OrdDb::Models
   Models = Model   
-end # module # OrdDb
+end # module OrdDb
 
 

data/lib/ordlite/models/generative.rb

@@ -0,0 +1,12 @@
+
+module OrdDb
+    module Model
+  
+  class Generative < ActiveRecord::Base
+    belongs_to :inscribe, foreign_key: 'id'
+    belongs_to :factory
+  end  # class Generative
+  
+    end # module Model
+end # module OrdDb
+  

data/lib/ordlite/models/inscribe.rb

@@ -4,12 +4,17 @@ module OrdDb
   
   class Inscribe < ActiveRecord::Base
     has_one :blob, foreign_key: 'id'
+ 
+    has_one :factory     ## optional (auto-added via og/orc-721 deploy)
+    has_one :generative, foreign_key: 'id'   ## optional (auto-added via og/orc-721 deploy)
+
+    ## convernience helper
+    ##  forward to blob.content
+    ##    blob.content - encoding is BINARY (ASCII-7BIT)
+    ##    blob.text    - force_encoding is UTF-8 (return a copy)
+    def content() blob.content; end
+    def text() blob.text; end
 
-    def content
-      ## convernience helper
-      ##  forward to blob.content
-       blob.content
-    end
 
     ################################
     ### scope like helpers
@@ -54,6 +59,228 @@ SQL
 
        joins(:blob).where( where_clause ).order( 'num' )
     end
+
+   def self.sub1k()  where( 'num < 1000' ); end
+   def self.sub2k()  where( 'num < 2000' ); end
+   def self.sub10k()  where( 'num < 10000' ); end
+   def self.sub20k()  where( 'num < 20000' ); end
+   def self.sub100k()  where( 'num < 100000' ); end
+   def self.sub1m()  where( 'num < 1000000' ); end
+   def self.sub2m()  where( 'num < 2000000' ); end
+   def self.sub10m()  where( 'num < 10000000' ); end
+   def self.sub20m()  where( 'num < 20000000' ); end
+   def self.sub21m()  where( 'num < 21000000' ); end
+ 
+
+   def self.largest
+      order( 'bytes DESC' )
+   end
+
+   def self.address_counts
+       group( 'address' )
+        .order( Arel.sql( 'COUNT(*) DESC')).count
+   end 
+
+   def self.block_counts
+       group( 'block' )
+        .order( 'block').count
+   end 
+   
+   def self.block_with_timestamp_counts
+       group( Arel.sql( "block || ' @ ' || date" ))
+        .order( Arel.sql( "block || ' @ ' || date" ) ).count
+   end
+
+   def self.content_type_counts
+       group( 'content_type' )
+       .order( Arel.sql( 'COUNT(*) DESC, content_type')).count
+   end
+
+
+   def self.date_counts
+       ## note: strftime is SQLite specific/only!!!
+      group( Arel.sql("strftime('%Y-%m-%d', date)"))
+        .order( Arel.sql("strftime('%Y-%m-%d', date)")).count
+   end
+
+   def self.month_counts
+      ## note: strftime is SQLite specific/only!!!
+      group( Arel.sql("strftime('%Y-%m', date)"))
+       .order( Arel.sql("strftime('%Y-%m', date)")).count
+   end
+
+   def self.hour_counts
+      ## note: strftime is SQLite specific/only!!!
+       group( Arel.sql("strftime('%Y-%m-%d %Hh', date)"))
+       .order( Arel.sql("strftime('%Y-%m-%d %Hh', date)")).count
+   end
+
+
+   class << self
+      alias_method :biggest, :largest
+      alias_method :counts_by_address,      :address_counts
+      alias_method :counts_by_content_type, :content_type_counts
+      alias_method :counts_by_date,         :date_counts
+      alias_method :counts_by_day,          :date_counts
+      alias_method :counts_by_month,        :month_counts
+      alias_method :counts_by_hour,        :hour_counts
+      alias_method :counts_by_block,        :block_counts
+      alias_method :counts_by_block_with_timestamp,  :block_with_timestamp_counts
+   end
+
+
+   def self.text
+      ## note: for now include:
+      ##   - text/plain (all variants)
+      ##   - text/json (all variants)
+      ##   - text/markdown
+      where( content_type: 
+               ['text/plain',
+                'text/plain;charset=utf-8',
+                'text/markdown',
+                'application/json',
+               ]
+           )
+   end
+   def self.png() where( content_type: 'image/png' ); end
+
+###
+## add support for ordinals.com api txt (headers format) 
+
+
+   def self.create_from_api( data ) create( _parse_api( data )); end
+   class << self
+     alias_method :create_from_cache, :create_from_api   ## add alias - why? why not?
+   end
+
+   
+   def self._parse_api( data )   ## parse api json data
+    ## num via title
+    attributes = {
+     id:  data['id'],
+     num: _title_to_num( data['title'] ),
+     bytes: _content_length_to_bytes( data['content-length'] ), 
+     sat:  data['sat'].to_i(10),
+     content_type:  data['content-type'],
+     block: data['genesis-height'].to_i(10),
+     fee: data['genesis-fee'].to_i(10),
+     tx: data['genesis-transaction'],
+     address: data['address'],
+     output: data['output'],
+     value: data['output-value'].to_i(10),
+     offset: data['offset'].to_i(10),
+     # "2023-06-01 05:00:57 UTC"
+     date:  DateTime.strptime( data['timestamp'], 
+                               '%Y-%m-%d %H:%M:%S %z')
+   }
+ 
+   attributes
+ end
+ 
+ 
+     ##  "title": "Inscription 9992615",
+ TITLE_RX = /^Inscription (?<num>[0-9]+)$/i
+ 
+ def self._title_to_num( str )
+    if m=TITLE_RX.match( str )
+       m[:num].to_i(10)    ## use base 10
+    else
+      puts "!! ERROR - no inscribe num found in title >#{str}<"
+      exit 1   ## not found - raise exception - why? why not?
+    end
+ end
+ 
+ CONTENT_LENGTH_RX = /^(?<num>[0-9]+) bytes$/i
+ 
+ def self._content_length_to_bytes( str )
+     if m=CONTENT_LENGTH_RX.match( str )
+        m[:num].to_i(10)    ## use base 10
+     else
+        puts "!! ERROR - bytes found in content lenght >#{str}<"
+        exit 1   ## not found - raise exception - why? why not?
+     end
+ end
+  
+
+
+ ###
+ # instance methods
+def extname
+  ## map mime type to file extname
+  ## see https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
+  ##  for real-world usage, see https://dune.com/dgtl_assets/bitcoin-ordinals-analysis
+  ##  https://github.com/casey/ord/blob/master/src/media.rs
+
+   if content_type.start_with?( 'text/plain' )
+      '.txt'
+   elsif content_type.start_with?( 'text/markdown' )
+      '.md'
+   elsif content_type.start_with?( 'text/html' )
+      '.html'
+   elsif content_type.start_with?( 'text/javascript' ) ||
+         content_type.start_with?( 'application/javascript' )
+      ## note: application/javascript is considered bad practice/legacy
+      '.js'
+   elsif content_type.start_with?( 'image/png' )
+       ## Portable Network Graphics (PNG)
+      '.png'
+   elsif content_type.start_with?( 'image/jpeg' )
+        ##  Joint Photographic Expert Group image (JPEG) 
+       '.jpg'   ## use jpeg - why? why not?
+   elsif content_type.start_with?( 'image/webp' )
+        ## Web Picture format (WEBP)
+       '.webp'   ## note: no three-letter extension available
+   elsif content_type.start_with?( 'image/svg' )
+        ## Scalable Vector Graphics (SVG) 
+       '.svg'
+   elsif content_type.start_with?( 'image/gif' ) 
+       ##  Graphics Interchange Format (GIF)
+       '.gif'
+   elsif content_type.start_with?( 'image/avif' )  
+       ## AV1 Image File Format (AVIF)
+       '.avif'
+   elsif content_type.start_with?( 'application/epub' )
+       '.epub'
+   elsif content_type.start_with?( 'application/pdf' )
+       '.pdf'
+   elsif content_type.start_with?( 'application/json' )
+       '.json'
+   elsif content_type.start_with?( 'application/pgp-signature' )
+       '.sig'
+   elsif content_type.start_with?( 'audio/mpeg' )
+       '.mp3'
+   elsif content_type.start_with?( 'audio/midi' )
+       '.midi'
+   elsif content_type.start_with?( 'video/mp4' )
+       '.mp4'
+   elsif content_type.start_with?( 'video/webm' )
+       '.wepm'
+   elsif content_type.start_with?( 'audio/mod' )  
+      ## is typo? possible? only one inscription in 20m?
+       '.mod'   ## check/todo/fix if is .wav?? 
+   else
+      puts "!! ERROR - no file extension configured for content type >#{content_type}<; sorry:"
+      pp self
+      exit 1
+   end
+end
+
+def export_path  ## default export path
+   numstr = "%08d" % num   ###  e.g. 00000001  
+   "./tmp/#{numstr}#{extname}" 
+end
+def export( path=export_path )
+   if blob
+     write_blob( path, blob.content )
+   else
+      ## todo/fix: raise exception - no content
+      puts "!! ERROR - inscribe has no content (blob); sorry:"
+      pp self
+      exit 1
+   end
+end
+
+
   end  # class Inscribe
   
     end # module Model

data/lib/ordlite/schema.rb

@@ -87,7 +87,7 @@ create_table :inscribes, :id => :string do |t|
     ##   what is location  ???
     ## "location": "0a3a4dbf6630338bc4df8e36bd081f8f7d2dee9441131cb03a18d43eb4882d5c:0:0",
 
-  ## timestamp at last
+  ## timestamp last
   t.timestamps
 end
 
@@ -99,14 +99,117 @@ create_table :blobs, :id => :string do |t|
     ## t.string   :id,  null: false, index: { unique: true, name: 'blob_uuids' }
  
     t.binary     :content,   null: false
+    t.string     :sha256   ## sha256 hash
+    t.string     :md5      ## md5 hash - add why? why not?  
 
-  ## timestamp at last
+  ## timestamp last
   t.timestamps
 end
 
 
+=begin
+ "name": "Planetary Ordinals",
+  "inscription_icon": "98da33abe2045ec1421fcf1bc376dea5beb17ded15aa70ca5da490f50d95a6d9i0",
+  "supply": "69",
+  "slug": "planetary-ordinals",
+  "description": "",
+  "twitter_link": "https://twitter.com/ordinalswallet",
+  "discord_link": "https://discord.com/invite/ordinalswallet",
+  "website_link": ""
+=end
+
+create_table :collections do |t|
+  t.string   :name, null: false
+  t.string   :slug   
+  t.text     :desc    # description
+  t.integer  :max     # supply
+  t.string   :icon_id    ## rename to inscribe_icon_id or such - why? why not?
+  ## add twitter_link, discord_link, website_link - why? why not?
+
+  ## if on-chain and metadata inscribed - add why? why not??
+  ## t.string   :source_id,  null: false   ## foreign key reference
+
+  ## timestamp last
+  t.timestamps
+end
+
+create_table :items do |t|
+  t.integer   :collection_id, null: false
+  t.string    :inscribe_id,   null: false
+  t.integer   :pos,           null: false
+  t.string    :name
+
+  ## timestamp last
+  t.timestamps
+
+  ## todo/fix: add unique index for :pos+:collection_id !!!
+end
+
+
+
+###
+#  generative (collection) factory
+create_table :factories, :id => :string do |t|
+  t.string   :name
+  t.integer  :max        # max limit
+  t.integer  :maxblock   # max block limit
+  t.string   :dim        # dimension e.g. 24x24 (in px)
+
+  t.string   :inscribe_id,  null: false   ## foreign key reference
+
+  ## timestamp last
+  t.timestamps
+end
+
+#####
+## join table (factory has_many modules)
+##   rename to layer / sprites / blocks / tiles / modules / submodules / subs / mods / ...etc - why? why not?
+##             layerlists or inscribelists or ???
+##  change/rename to factory_items or layer_items or such?
+create_table :inscriberefs, :id => false do |t|
+  t.string  :factory_id,    null: false
+  t.string  :inscribe_id,   null: false
+  t.integer :pos,           null: false    ## position (index) in list (starting at 0)
+  ## todo/fix: make factory_id + inscribe_id + pos unique index - why? why not?
+
+  ## timestamp last
+  t.timestamps
+end
+
+
+create_table :generatives, :id => :string do |t|
+  t.string  :factory_id,    null: false
+  t.string  :g,             null: false  ##  use space separated numbers - why? why not?    
+  t.binary  :content    ### optional for now - why? why not?
+ 
+  ## timestamp last
+  t.timestamps
+end
+
+
+
+
 end # block Schema.define
 
 end # method up
 end # class CreateDb
+
+###
+#  migrations helpers
+class AddGeneratives
+
+def up  
+  ActiveRecord::Schema.define do
+    create_table :generatives, :id => :string do |t|
+      t.string  :factory_id, null: false
+      t.string  :g,          null: false  ##  use space separated numbers - why? why not?    
+      t.binary  :content    ### optional for now - why? why not?
+     
+      ## timestamp last
+      t.timestamps
+    end
+  end # block Schema.define
+end # method up
+end # class AddGeneratives
+  
 end # module OrdDb

data/lib/ordlite/version.rb

@@ -3,8 +3,8 @@ module Ordlite
 
   # sync version w/ sport.db n friends - why? why not?
   MAJOR = 0 ## todo: namespace inside version or something - why? why not??
-  MINOR = 1
-  PATCH = 2
+  MINOR = 2
+  PATCH = 1
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version