scribelite 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8f774ffec5b1c7a02d7ff82d92e7dea2ca83aa918f498fb2d24969870e26d819
-  data.tar.gz: cf9faa2257cb17c595a01685f2dc1b97d4d6fb315f7e65a8e2b6b9663d59872d
+  metadata.gz: 8423f47a4a80b7687f04a31e0457500faa3b52e48a40e239377f0877271aface
+  data.tar.gz: 86413fdd0e42317307cb97f0989fca3f8b82736539d4333687ff6c9f158f5915
 SHA512:
-  metadata.gz: 272471a546baa300468d4dbc0558297930c12f20fd6447366dbd5b02bb6d34ff2aec04090731abcf7259447a389a1a0a20350c2c26659b95e2b9752808479f3d
-  data.tar.gz: 517b655428b5a043851c3bc0df41689648a9ca51839593d2f3a0284845a207e8570da852d36a165ef65fb6a6892ef1a0a5004490f4816823dd93d13cd977c6bb
+  metadata.gz: bfb49887358be1a89b68c855cdce8a0c2dbafa3439f76a0ef0c8dea47ceb00316d73d82577ba01f8112a839d7dc468e312ebcc9c4e0882f7f50fd3c5b560ae9e
+  data.tar.gz: d77a2a72689f2e49292edbcb4c02f74b96b1135a2ccf32a33c20c6e3bd79c12288143f8237ef4e06305d9bfa95c9fd5f7934ca36d1f521284d5269c2a8460cc4

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-### 0.1.0
+### 0.2.1
 ### 0.0.1 / 2023-11-21
 
 * Everything is new. First release

data/Manifest.txt

@@ -3,8 +3,9 @@ Manifest.txt
 README.md
 Rakefile
 lib/scribelite.rb
-lib/scribelite/models/calldata.rb
+lib/scribelite/importer.rb
 lib/scribelite/models/forward.rb
-lib/scribelite/models/inscribe.rb
+lib/scribelite/models/scribe.rb
+lib/scribelite/models/tx.rb
 lib/scribelite/schema.rb
 lib/scribelite/version.rb

data/README.md

@@ -19,11 +19,121 @@ See [Introducing Ethscriptions - A new way of creating and sharing digital artif
 
 ## Usage
 
+The work-in-progess database schema looks like:
+
+``` ruby
+ActiveRecord::Schema.define do
+
+create_table :scribes, :id => :string do |t|    
+    t.integer    :num, null: false, index: { unique: true, name: 'scribe_nums' }
+    t.integer    :bytes
+    t.string     :content_type
+    ## add allow duplicate opt-in protocol flag e.g. esip6
+    t.boolean    :duplicate      ## allows duplicates flag 
+    t.boolean    :flagged,  null: false,  default: false  ## censored flag / removed on request
+    t.string     :sha,    null: false    ## sha hash as hexstring (but no leading 0)
+end
+
+create_table :txs, :id => :string do |t|
+    t.binary     :data     # ,  null: false
+    t.datetime   :date,  null: false
+    t.integer    :block, null: false
+    t.integer    :idx,   null: false  ## transaction index (number)
+
+    t.string     :from,  null: false
+    t.string     :to,    null: false   
+
+    t.integer    :fee
+    t.integer    :value
+end  
+end 
+```
+
+
+and to setup use it like:
+
+``` ruby
+require 'scribelite'
+
+ScribeDb.connect( adapter:  'sqlite3',
+                  database: './scribe.db' )
+
+ScribeDb.create_all   ## build schema
+```
+
+and lets import the first hundred (page size is 25)  ethscriptions on mainnet (via the ethscriptions.com api):
+
+
+``` ruby
+require 'scribelite'
+
+ScribeDb.open( './scribe.db' )
+
+(1..4).each do |page|
+   ScribeDb.import_ethscriptions( page: page )
+end
+```
+
+
+
+and to query use it like:
+
+``` ruby
+require 'scribelite'
+
+ScribeDb.open( './scribe.db' )
+
+puts
+puts "  #{Scribe.count} scribe(s)"
+puts "  #{Tx.count} tx(s)"
+
+
+## how many flagged in top 100?
+limit = 100
+
+flagged_count =  Scribe.order( :num ).limit(limit).where( flagged: true ).count
+pp flagged_count    #=> 75 !!!!!!!!!
+unflagged_count =  Scribe.order( :num).limit(limit).where( flagged: false ).count
+pp unflagged_count  #=> 25
+
+
+Scribe.order( :num ).limit(limit).each do |scribe|
+    if scribe.flagged?
+      print " xx "
+    else
+      print "==> "
+    end
+    print "#{scribe.num} / #{scribe.content_type}   -   #{scribe.tx.date} @ #{scribe.tx.block}"
+    print "\n"
+end
+
+
+## Let's query for all inscriptions grouped by date (day) 
+## and dump the results:
+pp Scribe.counts_by_day   
+pp Scribe.counts_by_year
+pp Scribe.counts_by_month
+pp Scribe.counts_by_hour
+
+
+## Let's query for all content types and group by count (descending) 
+## and dump the results:
+pp Scribe.counts_by_content_type
+
+pp Scribe.counts_by_block
+pp Scribe.counts_by_block_with_timestamp
+
+pp Scribe.counts_by_address   # from (creator/minter) address 
+
+# ...
+```
+
 To be continued...
 
 
 
 
+
 ## Bonus - More Blockchain (Crypto) Tools, Libraries & Scripts In Ruby
 
 See [**/blockchain**](https://github.com/rubycocos/blockchain) 

data/Rakefile

@@ -18,7 +18,8 @@ Hoe.spec 'scribelite' do
   self.history_file = 'CHANGELOG.md'
 
   self.extra_deps = [
-    ['ethscribe'],   
+    ['ethscribe'], 
+    ['calldata'],    ## todo - check if included via ethscribe in the future?  
     ['activerecord'],
     ['activerecord-utils'],
     ['logutils'],

data/lib/scribelite/importer.rb

@@ -0,0 +1,91 @@
+
+
+module ScribeDb
+
+## note: by default - sort asc(ending) - oldest first (0,1,2,3, .etc.)
+def self.import_ethscriptions( page: 1, sort_order: 'asc' )
+    net   = Ethscribe::Api.mainnet
+    recs = net.ethscriptions( page: page, sort_order: sort_order )
+
+    recs.each_with_index do |rec,i|
+        puts "==> page #{page}/#{i+1} - #{rec['transaction_hash']}..."
+
+        txid  = rec['transaction_hash']
+        block = rec['block_number']
+        idx   = rec['transaction_index']
+        
+        from  = rec['creator']
+        to    = rec['initial_owner']
+
+        ## todo - double check if daylight saving time (dst) breaks timestamp == utc identity/conversion?
+        ##  2001-02-03T04:05:06+07:00
+        ##  2016-05-29T22:28:15.000Z
+        ##  check if %z can handle  .000Z ??
+        ##  DateTime.strptime( '2016-05-29T22:28:15.000Z', '%Y-%m-%dT%H:%M:%S%z' )
+        ## pp rec['creation_timestamp']
+        date  = DateTime.strptime( rec['creation_timestamp'], '%Y-%m-%dT%H:%M:%S.000Z' )   
+        
+   
+        num   = rec['ethscription_number']
+        content_type = rec['mimetype']
+        data = rec['content_uri']
+        sha  = rec['sha']
+
+        duplicate = rec['esip6']
+        flagged   = rec['image_removed_by_request_of_rights_holder']
+
+       ### check - if flagged
+       ##  content_type always set to text/plain
+       ##     and data to data:,  ???
+       ##   (re)set to nil/null - why? why not?
+         if flagged
+            data         = nil
+            content_type = nil
+         end
+        
+        tx_attribs = {
+           id:   txid,
+           block: block,
+           idx:   idx,
+           date:  date,
+           from:  from,
+           to:    to,
+           data:  data,   ## note: for now saved as utf8 string (not hex!!!!)
+        }
+
+        scribe_attribs = {
+           id: txid,
+           num:  num,
+           content_type: content_type,
+           duplicate: duplicate,
+           flagged: flagged,
+           sha:   sha, 
+           bytes: data ? data.length : nil    ## auto-add/calc bytes (content length)
+        }
+
+
+
+
+        scribe = Scribe.find_by( id: txid )
+        if scribe
+            ## skip for now - found id db
+        else
+            puts "scribe_attribs:"
+            pp scribe_attribs
+            Scribe.create!( **scribe_attribs )
+        end
+ 
+        tx = Tx.find_by( id: txid )
+        if tx
+            ## skip for now - found id db
+        else
+            puts "tx_attribs:"
+            pp tx_attribs
+            Tx.create!( **tx_attribs )
+        end
+    end
+
+end  # method import_ethscriptions
+
+end  # module ScribeDb
+ 

data/lib/scribelite/models/forward.rb

@@ -10,8 +10,8 @@ module Model
 Prop      = ConfDb::Model::Prop
 
 
-class Inscribe    < ActiveRecord::Base ; end
-class Calldata    < ActiveRecord::Base ; end
+class Scribe      < ActiveRecord::Base ; end
+class Tx          < ActiveRecord::Base ; end
 
 
 end # module Model

data/lib/scribelite/models/{inscribe.rb → scribe.rb}

@@ -2,8 +2,8 @@
 module ScribeDb
     module Model
   
-  class Inscribe < ActiveRecord::Base
-    has_one :calldata, foreign_key: 'id'
+  class Scribe < ActiveRecord::Base
+    has_one :tx, foreign_key: 'id'
  
     ## convernience helper
     ##  forward to blob.content
@@ -17,7 +17,8 @@ module ScribeDb
     ### scope like helpers
     def self.png() where( content_type: 'image/png' ); end
     def self.gif() where( content_type: 'image/gif' ); end
-    def self.jpg() where( content_type: 'image/jpeg' ); end
+    def self.jpg() where( content_type: ['image/jpeg',
+                                         'image/jpg'] ); end
     def self.webp() where( content_type: 'image/webp' ); end
     def self.svg()  where( content_type: 'image/svg+xml' ); end
     def self.avif() where( content_type: 'image/avif' ); end
@@ -31,6 +32,7 @@ module ScribeDb
         where( content_type: [
             'image/png',
             'image/jpeg',
+            'image/jpg',
             'image/gif',
             'image/webp',
             'image/svg+xml',
@@ -98,91 +100,64 @@ module ScribeDb
        .order( Arel.sql( 'COUNT(*) DESC, content_type')).count
    end
 
-   class << self
-      alias_method :biggest, :largest
-      alias_method :counts_by_content_type, :content_type_counts
+
+   def self.block_counts
+       joins(:tx).group( 'block' )
+         .order( 'block').count
+   end 
+
+   def self.block_with_timestamp_counts
+       joins(:tx).group( Arel.sql( "block || ' @ ' || date" ))
+         .order( 'block' ).count
+   end
+
+
+   def self.date_counts
+      ## note: strftime is SQLite specific/only!!!
+      joins(:tx).group( Arel.sql("strftime('%Y-%m-%d', date)"))
+       .order( Arel.sql("strftime('%Y-%m-%d', date)")).count
+   end
+
+   def self.month_counts
+      ## note: strftime is SQLite specific/only!!!
+      joins(:tx).group( Arel.sql("strftime('%Y-%m', date)"))
+       .order( Arel.sql("strftime('%Y-%m', date)")).count
    end
 
- ###
- # instance methods
-def extname
-  ## map mime type to file extname
-  ## see https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
-  ##  for real-world usage, see https://dune.com/dgtl_assets/bitcoin-ordinals-analysis
-  ##  https://github.com/casey/ord/blob/master/src/media.rs
-
-   if content_type.start_with?( 'text/plain' )
-      '.txt'
-   elsif content_type.start_with?( 'text/markdown' )
-      '.md'
-   elsif content_type.start_with?( 'text/html' )
-      '.html'
-   elsif content_type.start_with?( 'text/javascript' ) ||
-         content_type.start_with?( 'application/javascript' )
-      ## note: application/javascript is considered bad practice/legacy
-      '.js'
-   elsif content_type.start_with?( 'image/png' )
-       ## Portable Network Graphics (PNG)
-      '.png'
-   elsif content_type.start_with?( 'image/jpeg' )
-        ##  Joint Photographic Expert Group image (JPEG) 
-       '.jpg'   ## use jpeg - why? why not?
-   elsif content_type.start_with?( 'image/webp' )
-        ## Web Picture format (WEBP)
-       '.webp'   ## note: no three-letter extension available
-   elsif content_type.start_with?( 'image/svg' )
-        ## Scalable Vector Graphics (SVG) 
-       '.svg'
-   elsif content_type.start_with?( 'image/gif' ) 
-       ##  Graphics Interchange Format (GIF)
-       '.gif'
-   elsif content_type.start_with?( 'image/avif' )  
-       ## AV1 Image File Format (AVIF)
-       '.avif'
-   elsif content_type.start_with?( 'application/epub' )
-       '.epub'
-   elsif content_type.start_with?( 'application/pdf' )
-       '.pdf'
-   elsif content_type.start_with?( 'application/json' )
-       '.json'
-   elsif content_type.start_with?( 'application/pgp-signature' )
-       '.sig'
-   elsif content_type.start_with?( 'audio/mpeg' )
-       '.mp3'
-   elsif content_type.start_with?( 'audio/midi' )
-       '.midi'
-   elsif content_type.start_with?( 'video/mp4' )
-       '.mp4'
-   elsif content_type.start_with?( 'video/webm' )
-       '.wepm'
-   elsif content_type.start_with?( 'audio/mod' )  
-      ## is typo? possible? only one inscription in 20m?
-       '.mod'   ## check/todo/fix if is .wav?? 
-   else
-      puts "!! ERROR - no file extension configured for content type >#{content_type}<; sorry:"
-      pp self
-      exit 1
+   def self.year_counts
+      ## note: strftime is SQLite specific/only!!!
+      joins(:tx).group( Arel.sql("strftime('%Y', date)"))
+       .order( Arel.sql("strftime('%Y', date)")).count
    end
-end
-
-=begin
-def export_path  ## default export path
-   numstr = "%08d" % num   ###  e.g. 00000001  
-   "./tmp/#{numstr}#{extname}" 
-end
-def export( path=export_path )
-   if blob
-     write_blob( path, blob.content )
-   else
-      ## todo/fix: raise exception - no content
-      puts "!! ERROR - inscribe has no content (blob); sorry:"
-      pp self
-      exit 1
+ 
+   def self.hour_counts
+     ## note: strftime is SQLite specific/only!!!
+      joins(:tx).group( Arel.sql("strftime('%Y-%m-%d %Hh', date)"))
+      .order( Arel.sql("strftime('%Y-%m-%d %Hh', date)")).count
    end
-end
-=end
 
-  end  # class Inscribe
+
+   def self.from_counts
+    ## note: from is sql keyword!!!
+    ##  wrap in [] for sqlite - check if works for others!!!  
+      joins(:tx).group( '[from]' )
+       .order( Arel.sql( 'COUNT(*) DESC')).count
+   end 
+
+
+   class << self
+      alias_method :biggest, :largest
+      alias_method :counts_by_content_type, :content_type_counts
+      alias_method :counts_by_date,         :date_counts
+      alias_method :counts_by_day,          :date_counts
+      alias_method :counts_by_month,        :month_counts
+      alias_method :counts_by_year,         :year_counts
+      alias_method :counts_by_hour,         :hour_counts
+      alias_method :counts_by_block,        :block_counts
+      alias_method :counts_by_block_with_timestamp,  :block_with_timestamp_counts
+      alias_method :counts_by_address,      :from_counts
+    end
+  end  # class Scribe
   
     end # module Model
 end # module ScribeDb

data/lib/scribelite/models/{calldata.rb → tx.rb}

@@ -2,17 +2,16 @@
 module ScribeDb
     module Model
   
-  class Calldata < ActiveRecord::Base
-    self.table_name = 'calldatas'   ## check if infers data? why? why not?
+  class Tx < ActiveRecord::Base
+    self.table_name = 'txs'   ## note auto-infers txes change to txs
 
     belongs_to :inscribe, foreign_key: 'id'
-
 =begin    
     def text
        content.force_encoding(Encoding::UTF_8)
     end
 =end
-  end  # class Calldata
+  end  # class Tx
   
     end # module Model
 end # module ScribeDb

data/lib/scribelite/schema.rb

@@ -8,15 +8,15 @@ def up
 
 ActiveRecord::Schema.define do
 
-create_table :inscribes, :id => :string do |t|    
+create_table :scribes, :id => :string do |t|    
     ## "id": "0a3a4dbf6630338bc4df8e36bd081f8f7d2dee9441131cb03a18d43eb4882d5ci0",
     ## note: change to uuid (universally unique id) - why? why not?
     ##        id gets used by row_id (internal orm db machinery) and is int
-   ## t.string   :uuid,  null: false, index: { unique: true, name: 'inscribe_uuids' }
+   ## t.string   :uuid,  null: false, index: { unique: true, name: 'scribe_uuids' }
 
    ## "title": "Inscription 10371414",
    ##   note: use num/no. from title only - why? why not?
-    t.integer  :num, null: false, index: { unique: true, name: 'inscribe_nums' }
+    t.integer  :num, null: false, index: { unique: true, name: 'scribe_nums' }
    
     ##  "content length": "85 bytes",
     ## note:  extract bytes as integer!!!
@@ -25,11 +25,14 @@ create_table :inscribes, :id => :string do |t|
     ## "content type": "text/plain;charset=utf-8",
     ## note: make sure always lower/down case!!!
     t.string   :content_type
-    
- 
+
     ## add allow duplicate opt-in protocol flag e.g. esip6
-    ## t.boolean    :duplicate   -
- 
+    t.boolean    :duplicate      ## allows duplicates flag - make duplicate the default - why? why not?
+    t.boolean    :flagged,  null: false,  default: false  ## censored flag / removed on request
+
+    ## move sha to tx - why? why not?
+    t.string     :sha    # ,  null: false    ## sha hash as hexstring
+
   ## timestamp last
   t.timestamps
 end
@@ -37,14 +40,13 @@ end
 
 ## change to tx/txs or txn/txns - why? why not?
 
-create_table :calldatas, :id => :string do |t|
+create_table :txs, :id => :string do |t|
     ## "id": "0a3a4dbf6630338bc4df8e36bd081f8f7d2dee9441131cb03a18d43eb4882d5ci0",
     ## note: change to uuid (universally unique id) - why? why not?
     ##        id gets used by row_id (internal orm db machinery) and is int
     ## t.string   :id,  null: false, index: { unique: true, name: 'blob_uuids' }
  
-    t.binary     :data   # ,  null: false
-    t.string     :sha    # ,  null: false    ## sha hash as hexstring
+    t.binary     :data     # ,  null: false
  
     ## "timestamp": "2023-06-01 05:00:57 UTC"
     ##   or use date_utc ???
@@ -52,19 +54,16 @@ create_table :calldatas, :id => :string do |t|
     t.datetime    :date, null: false
 
     t.integer    :block, null: false
-    t.integer    :fee
-    t.integer    :value
+    t.integer    :idx,   null: false  ## transaction index (number)
 
-    ###
-    ## "address": "bc1p3h4eecuxjj2g72sq38gyva732866u5w29lhxgeqfe6c0sg8xmagsuau63k",
-    ##    is this minter/inscriber addr???
-    ##  change to minter?? or such - why? why not?
-    ##             creator 
     t.string     :from, null: false
+    t.string     :to,   null: false   
 
-    ## add to address too - why? why not?
-    t.string     :to   
-  ## timestamp last
+    t.integer    :fee
+    t.integer    :value
+
+  
+    ## timestamp last
   t.timestamps
 end
 

data/lib/scribelite/version.rb

@@ -1,8 +1,8 @@
 
 module Scribelite
   MAJOR = 0 ## todo: namespace inside version or something - why? why not??
-  MINOR = 1
-  PATCH = 0
+  MINOR = 2
+  PATCH = 1
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/scribelite.rb

@@ -1,10 +1,11 @@
 
 # core and stlibs
 require 'ethscribe'  ## will pull-in cocos & friends
-
+require 'calldata'
 
 
 require 'logger'    # Note: use for ActiveRecord::Base.logger -- remove/replace later w/ LogUtils::Logger ???
+require 'date'      ## check if date & datetime required ??
 
 
 # 3rd party gems / libs
@@ -27,14 +28,12 @@ require_relative 'scribelite/version'   # always goes first
 
 require_relative 'scribelite/models/forward'
 
-require_relative 'scribelite/models/inscribe'
-require_relative 'scribelite/models/calldata'
+require_relative 'scribelite/models/scribe'
+require_relative 'scribelite/models/tx'
 
 
 require_relative 'scribelite/schema'      
 
-# require_relative 'cache'
-# require_relative 'importer'  ## note: require (soft dep) ordinals gems!!!
 
 
 
@@ -66,7 +65,7 @@ module ScribeDb
       ConfDb.create    # add props table
     end
 
-    unless ScribeDb::Model::Inscribe.table_exists?
+    unless ScribeDb::Model::Scribe.table_exists?
       ScribeDb.create
     end
   end # method auto_migrate!
@@ -133,11 +132,23 @@ end  # module ScribeDb
 
 
 
+# add ethscriptions / ethscripe importer - why? why not?
+require_relative 'scribelite/importer'
+
+
+
 
 
 ## add convenience helpers
-Inscribe  = ScribeDb::Model::Inscribe
-Calldata  = ScribeDb::Model::Calldata
+Scribe    = ScribeDb::Model::Scribe
+Tx        = ScribeDb::Model::Tx
+
+
+
+require 'active_support/number_helper'
+include ActiveSupport::NumberHelper   ## e.g. number_to_human_size
+
+
 
 
 # say hello

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scribelite
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-21 00:00:00.000000000 Z
+date: 2023-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ethscribe
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: calldata
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -171,9 +185,10 @@ files:
 - README.md
 - Rakefile
 - lib/scribelite.rb
-- lib/scribelite/models/calldata.rb
+- lib/scribelite/importer.rb
 - lib/scribelite/models/forward.rb
-- lib/scribelite/models/inscribe.rb
+- lib/scribelite/models/scribe.rb
+- lib/scribelite/models/tx.rb
 - lib/scribelite/schema.rb
 - lib/scribelite/version.rb
 homepage: https://github.com/s6ruby/rubidity