scribelite 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0608ffa940a6f65b4432a6e8e494ec2213e20a6fb661e15eafb9f0408f1175be'
-  data.tar.gz: 5b019fbe73890c1b20a5c4106a617e2c573431e805b47825fb056f7bd50449aa
+  metadata.gz: 8423f47a4a80b7687f04a31e0457500faa3b52e48a40e239377f0877271aface
+  data.tar.gz: 86413fdd0e42317307cb97f0989fca3f8b82736539d4333687ff6c9f158f5915
 SHA512:
-  metadata.gz: 3a363643f683a32c1ccbd04448d790b6582f5ccc7d8d3176ab65506ff92d826a5b54e37b41add46bb8c8b577a8f76aa9d8ee7800a7d946aa7cf5ce9802fb283b
-  data.tar.gz: 491db0258ff4a4dc27974f9e3e8e8147a9437690dfdd2c30bf8d81b8710aee0f7ee4a119cd3657b6d48f1cc3399570e4787ef9fbdc87d17ec4f4115acb79d1e7
+  metadata.gz: bfb49887358be1a89b68c855cdce8a0c2dbafa3439f76a0ef0c8dea47ceb00316d73d82577ba01f8112a839d7dc468e312ebcc9c4e0882f7f50fd3c5b560ae9e
+  data.tar.gz: d77a2a72689f2e49292edbcb4c02f74b96b1135a2ccf32a33c20c6e3bd79c12288143f8237ef4e06305d9bfa95c9fd5f7934ca36d1f521284d5269c2a8460cc4

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-### 0.2.0
+### 0.2.1
 ### 0.0.1 / 2023-11-21
 
 * Everything is new. First release

data/Manifest.txt

@@ -3,6 +3,7 @@ Manifest.txt
 README.md
 Rakefile
 lib/scribelite.rb
+lib/scribelite/importer.rb
 lib/scribelite/models/forward.rb
 lib/scribelite/models/scribe.rb
 lib/scribelite/models/tx.rb

data/README.md

@@ -25,24 +25,23 @@ The work-in-progess database schema looks like:
 ActiveRecord::Schema.define do
 
 create_table :scribes, :id => :string do |t|    
-    t.integer  :num, null: false, index: { unique: true, name: 'scribe_nums' }
-    t.integer  :bytes
-    t.string   :content_type
+    t.integer    :num, null: false, index: { unique: true, name: 'scribe_nums' }
+    t.integer    :bytes
+    t.string     :content_type
     ## add allow duplicate opt-in protocol flag e.g. esip6
     t.boolean    :duplicate      ## allows duplicates flag 
     t.boolean    :flagged,  null: false,  default: false  ## censored flag / removed on request
-    ## move sha to tx - why? why not?
     t.string     :sha,    null: false    ## sha hash as hexstring (but no leading 0)
 end
 
 create_table :txs, :id => :string do |t|
     t.binary     :data     # ,  null: false
-    t.datetime   :date, null: false
+    t.datetime   :date,  null: false
     t.integer    :block, null: false
     t.integer    :idx,   null: false  ## transaction index (number)
 
-    t.string     :from, null: false
-    t.string     :to,   null: false   
+    t.string     :from,  null: false
+    t.string     :to,    null: false   
 
     t.integer    :fee
     t.integer    :value
@@ -62,6 +61,20 @@ ScribeDb.connect( adapter:  'sqlite3',
 ScribeDb.create_all   ## build schema
 ```
 
+and lets import the first hundred (page size is 25)  ethscriptions on mainnet (via the ethscriptions.com api):
+
+
+``` ruby
+require 'scribelite'
+
+ScribeDb.open( './scribe.db' )
+
+(1..4).each do |page|
+   ScribeDb.import_ethscriptions( page: page )
+end
+```
+
+
 
 and to query use it like:
 
@@ -93,9 +106,28 @@ Scribe.order( :num ).limit(limit).each do |scribe|
     print "#{scribe.num} / #{scribe.content_type}   -   #{scribe.tx.date} @ #{scribe.tx.block}"
     print "\n"
 end
-```
 
 
+## Let's query for all inscriptions grouped by date (day) 
+## and dump the results:
+pp Scribe.counts_by_day   
+pp Scribe.counts_by_year
+pp Scribe.counts_by_month
+pp Scribe.counts_by_hour
+
+
+## Let's query for all content types and group by count (descending) 
+## and dump the results:
+pp Scribe.counts_by_content_type
+
+pp Scribe.counts_by_block
+pp Scribe.counts_by_block_with_timestamp
+
+pp Scribe.counts_by_address   # from (creator/minter) address 
+
+# ...
+```
+
 To be continued...
 
 

data/lib/scribelite/importer.rb

@@ -0,0 +1,91 @@
+
+
+module ScribeDb
+
+## note: by default - sort asc(ending) - oldest first (0,1,2,3, .etc.)
+def self.import_ethscriptions( page: 1, sort_order: 'asc' )
+    net   = Ethscribe::Api.mainnet
+    recs = net.ethscriptions( page: page, sort_order: sort_order )
+
+    recs.each_with_index do |rec,i|
+        puts "==> page #{page}/#{i+1} - #{rec['transaction_hash']}..."
+
+        txid  = rec['transaction_hash']
+        block = rec['block_number']
+        idx   = rec['transaction_index']
+        
+        from  = rec['creator']
+        to    = rec['initial_owner']
+
+        ## todo - double check if daylight saving time (dst) breaks timestamp == utc identity/conversion?
+        ##  2001-02-03T04:05:06+07:00
+        ##  2016-05-29T22:28:15.000Z
+        ##  check if %z can handle  .000Z ??
+        ##  DateTime.strptime( '2016-05-29T22:28:15.000Z', '%Y-%m-%dT%H:%M:%S%z' )
+        ## pp rec['creation_timestamp']
+        date  = DateTime.strptime( rec['creation_timestamp'], '%Y-%m-%dT%H:%M:%S.000Z' )   
+        
+   
+        num   = rec['ethscription_number']
+        content_type = rec['mimetype']
+        data = rec['content_uri']
+        sha  = rec['sha']
+
+        duplicate = rec['esip6']
+        flagged   = rec['image_removed_by_request_of_rights_holder']
+
+       ### check - if flagged
+       ##  content_type always set to text/plain
+       ##     and data to data:,  ???
+       ##   (re)set to nil/null - why? why not?
+         if flagged
+            data         = nil
+            content_type = nil
+         end
+        
+        tx_attribs = {
+           id:   txid,
+           block: block,
+           idx:   idx,
+           date:  date,
+           from:  from,
+           to:    to,
+           data:  data,   ## note: for now saved as utf8 string (not hex!!!!)
+        }
+
+        scribe_attribs = {
+           id: txid,
+           num:  num,
+           content_type: content_type,
+           duplicate: duplicate,
+           flagged: flagged,
+           sha:   sha, 
+           bytes: data ? data.length : nil    ## auto-add/calc bytes (content length)
+        }
+
+
+
+
+        scribe = Scribe.find_by( id: txid )
+        if scribe
+            ## skip for now - found id db
+        else
+            puts "scribe_attribs:"
+            pp scribe_attribs
+            Scribe.create!( **scribe_attribs )
+        end
+ 
+        tx = Tx.find_by( id: txid )
+        if tx
+            ## skip for now - found id db
+        else
+            puts "tx_attribs:"
+            pp tx_attribs
+            Tx.create!( **tx_attribs )
+        end
+    end
+
+end  # method import_ethscriptions
+
+end  # module ScribeDb
+ 

data/lib/scribelite/models/scribe.rb

@@ -17,7 +17,8 @@ module ScribeDb
     ### scope like helpers
     def self.png() where( content_type: 'image/png' ); end
     def self.gif() where( content_type: 'image/gif' ); end
-    def self.jpg() where( content_type: 'image/jpeg' ); end
+    def self.jpg() where( content_type: ['image/jpeg',
+                                         'image/jpg'] ); end
     def self.webp() where( content_type: 'image/webp' ); end
     def self.svg()  where( content_type: 'image/svg+xml' ); end
     def self.avif() where( content_type: 'image/avif' ); end
@@ -31,6 +32,7 @@ module ScribeDb
         where( content_type: [
             'image/png',
             'image/jpeg',
+            'image/jpg',
             'image/gif',
             'image/webp',
             'image/svg+xml',
@@ -98,90 +100,63 @@ module ScribeDb
        .order( Arel.sql( 'COUNT(*) DESC, content_type')).count
    end
 
-   class << self
-      alias_method :biggest, :largest
-      alias_method :counts_by_content_type, :content_type_counts
+
+   def self.block_counts
+       joins(:tx).group( 'block' )
+         .order( 'block').count
+   end 
+
+   def self.block_with_timestamp_counts
+       joins(:tx).group( Arel.sql( "block || ' @ ' || date" ))
+         .order( 'block' ).count
+   end
+
+
+   def self.date_counts
+      ## note: strftime is SQLite specific/only!!!
+      joins(:tx).group( Arel.sql("strftime('%Y-%m-%d', date)"))
+       .order( Arel.sql("strftime('%Y-%m-%d', date)")).count
+   end
+
+   def self.month_counts
+      ## note: strftime is SQLite specific/only!!!
+      joins(:tx).group( Arel.sql("strftime('%Y-%m', date)"))
+       .order( Arel.sql("strftime('%Y-%m', date)")).count
    end
 
- ###
- # instance methods
-def extname
-  ## map mime type to file extname
-  ## see https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
-  ##  for real-world usage, see https://dune.com/dgtl_assets/bitcoin-ordinals-analysis
-  ##  https://github.com/casey/ord/blob/master/src/media.rs
-
-   if content_type.start_with?( 'text/plain' )
-      '.txt'
-   elsif content_type.start_with?( 'text/markdown' )
-      '.md'
-   elsif content_type.start_with?( 'text/html' )
-      '.html'
-   elsif content_type.start_with?( 'text/javascript' ) ||
-         content_type.start_with?( 'application/javascript' )
-      ## note: application/javascript is considered bad practice/legacy
-      '.js'
-   elsif content_type.start_with?( 'image/png' )
-       ## Portable Network Graphics (PNG)
-      '.png'
-   elsif content_type.start_with?( 'image/jpeg' )
-        ##  Joint Photographic Expert Group image (JPEG) 
-       '.jpg'   ## use jpeg - why? why not?
-   elsif content_type.start_with?( 'image/webp' )
-        ## Web Picture format (WEBP)
-       '.webp'   ## note: no three-letter extension available
-   elsif content_type.start_with?( 'image/svg' )
-        ## Scalable Vector Graphics (SVG) 
-       '.svg'
-   elsif content_type.start_with?( 'image/gif' ) 
-       ##  Graphics Interchange Format (GIF)
-       '.gif'
-   elsif content_type.start_with?( 'image/avif' )  
-       ## AV1 Image File Format (AVIF)
-       '.avif'
-   elsif content_type.start_with?( 'application/epub' )
-       '.epub'
-   elsif content_type.start_with?( 'application/pdf' )
-       '.pdf'
-   elsif content_type.start_with?( 'application/json' )
-       '.json'
-   elsif content_type.start_with?( 'application/pgp-signature' )
-       '.sig'
-   elsif content_type.start_with?( 'audio/mpeg' )
-       '.mp3'
-   elsif content_type.start_with?( 'audio/midi' )
-       '.midi'
-   elsif content_type.start_with?( 'video/mp4' )
-       '.mp4'
-   elsif content_type.start_with?( 'video/webm' )
-       '.wepm'
-   elsif content_type.start_with?( 'audio/mod' )  
-      ## is typo? possible? only one inscription in 20m?
-       '.mod'   ## check/todo/fix if is .wav?? 
-   else
-      puts "!! ERROR - no file extension configured for content type >#{content_type}<; sorry:"
-      pp self
-      exit 1
+   def self.year_counts
+      ## note: strftime is SQLite specific/only!!!
+      joins(:tx).group( Arel.sql("strftime('%Y', date)"))
+       .order( Arel.sql("strftime('%Y', date)")).count
    end
-end
-
-=begin
-def export_path  ## default export path
-   numstr = "%08d" % num   ###  e.g. 00000001  
-   "./tmp/#{numstr}#{extname}" 
-end
-def export( path=export_path )
-   if blob
-     write_blob( path, blob.content )
-   else
-      ## todo/fix: raise exception - no content
-      puts "!! ERROR - inscribe has no content (blob); sorry:"
-      pp self
-      exit 1
+ 
+   def self.hour_counts
+     ## note: strftime is SQLite specific/only!!!
+      joins(:tx).group( Arel.sql("strftime('%Y-%m-%d %Hh', date)"))
+      .order( Arel.sql("strftime('%Y-%m-%d %Hh', date)")).count
    end
-end
-=end
 
+
+   def self.from_counts
+    ## note: from is sql keyword!!!
+    ##  wrap in [] for sqlite - check if works for others!!!  
+      joins(:tx).group( '[from]' )
+       .order( Arel.sql( 'COUNT(*) DESC')).count
+   end 
+
+
+   class << self
+      alias_method :biggest, :largest
+      alias_method :counts_by_content_type, :content_type_counts
+      alias_method :counts_by_date,         :date_counts
+      alias_method :counts_by_day,          :date_counts
+      alias_method :counts_by_month,        :month_counts
+      alias_method :counts_by_year,         :year_counts
+      alias_method :counts_by_hour,         :hour_counts
+      alias_method :counts_by_block,        :block_counts
+      alias_method :counts_by_block_with_timestamp,  :block_with_timestamp_counts
+      alias_method :counts_by_address,      :from_counts
+    end
   end  # class Scribe
   
     end # module Model

data/lib/scribelite/version.rb

@@ -2,7 +2,7 @@
 module Scribelite
   MAJOR = 0 ## todo: namespace inside version or something - why? why not??
   MINOR = 2
-  PATCH = 0
+  PATCH = 1
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/scribelite.rb

@@ -34,8 +34,6 @@ require_relative 'scribelite/models/tx'
 
 require_relative 'scribelite/schema'      
 
-# require_relative 'cache'
-# require_relative 'importer'  ## note: require (soft dep) ordinals gems!!!
 
 
 
@@ -134,6 +132,11 @@ end  # module ScribeDb
 
 
 
+# add ethscriptions / ethscripe importer - why? why not?
+require_relative 'scribelite/importer'
+
+
+
 
 
 ## add convenience helpers
@@ -141,6 +144,13 @@ Scribe    = ScribeDb::Model::Scribe
 Tx        = ScribeDb::Model::Tx
 
 
+
+require 'active_support/number_helper'
+include ActiveSupport::NumberHelper   ## e.g. number_to_human_size
+
+
+
+
 # say hello
 puts Scribelite.banner     ## if defined?($RUBYCOCOS_DEBUG) && $RUBCOCOS_DEBUG
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scribelite
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-25 00:00:00.000000000 Z
+date: 2023-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ethscribe
@@ -185,6 +185,7 @@ files:
 - README.md
 - Rakefile
 - lib/scribelite.rb
+- lib/scribelite/importer.rb
 - lib/scribelite/models/forward.rb
 - lib/scribelite/models/scribe.rb
 - lib/scribelite/models/tx.rb