scribelite 0.2.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0608ffa940a6f65b4432a6e8e494ec2213e20a6fb661e15eafb9f0408f1175be'
-  data.tar.gz: 5b019fbe73890c1b20a5c4106a617e2c573431e805b47825fb056f7bd50449aa
+  metadata.gz: d2a98f4c256e13196f739fb4f538a954f50f51b85ed4bd3fedb3041e66dd3580
+  data.tar.gz: 5e9009cef8b739e5e02d630f0e751528a93665d81ba08cecb6ac0e50f8739875
 SHA512:
-  metadata.gz: 3a363643f683a32c1ccbd04448d790b6582f5ccc7d8d3176ab65506ff92d826a5b54e37b41add46bb8c8b577a8f76aa9d8ee7800a7d946aa7cf5ce9802fb283b
-  data.tar.gz: 491db0258ff4a4dc27974f9e3e8e8147a9437690dfdd2c30bf8d81b8710aee0f7ee4a119cd3657b6d48f1cc3399570e4787ef9fbdc87d17ec4f4115acb79d1e7
+  metadata.gz: 2b920e7a86fe0c47a4b75870c2ae1d27805210b82511bab685cca55af763a93f4f7c9b3124d449eb350b2f22953a920119749b389f6cb357ce5b5750f5282633
+  data.tar.gz: 69177a203454d697914a49bf9cfc1b7bc94e00cb149b4a012d01e808433fe7ab0c6d6b3748de8222860d9860a265d2e7d82dc7dcb7c30f246001646aa0a39ec0

data/CHANGELOG.md

@@ -1,4 +1,5 @@
-### 0.2.0
+### 1.0.0
+### 0.2.1
 ### 0.0.1 / 2023-11-21
 
 * Everything is new. First release

data/Manifest.txt

@@ -3,6 +3,7 @@ Manifest.txt
 README.md
 Rakefile
 lib/scribelite.rb
+lib/scribelite/importer.rb
 lib/scribelite/models/forward.rb
 lib/scribelite/models/scribe.rb
 lib/scribelite/models/tx.rb

data/README.md

@@ -4,8 +4,8 @@ scribelite - inscription / inscribe (ethscription calldata) database for ethereu
 
 
 
-* home  :: [github.com/s6ruby/rubidity](https://github.com/s6ruby/rubidity)
-* bugs  :: [github.com/s6ruby/rubidity/issues](https://github.com/s6ruby/rubidity/issues)
+* home  :: [github.com/0xCompute/ethscribe](https://github.com/0xCompute/ethscribe)
+* bugs  :: [github.com/0xCompute/ethscribe/issues](https://github.com/0xCompute/ethscribe/issues)
 * gem   :: [rubygems.org/gems/scribelite](https://rubygems.org/gems/scribelite)
 * rdoc  :: [rubydoc.info/gems/scribelite](http://rubydoc.info/gems/scribelite)
 
@@ -25,24 +25,23 @@ The work-in-progess database schema looks like:
 ActiveRecord::Schema.define do
 
 create_table :scribes, :id => :string do |t|    
-    t.integer  :num, null: false, index: { unique: true, name: 'scribe_nums' }
-    t.integer  :bytes
-    t.string   :content_type
+    t.integer    :num, null: false, index: { unique: true, name: 'scribe_nums' }
+    t.integer    :bytes
+    t.string     :content_type
     ## add allow duplicate opt-in protocol flag e.g. esip6
     t.boolean    :duplicate      ## allows duplicates flag 
     t.boolean    :flagged,  null: false,  default: false  ## censored flag / removed on request
-    ## move sha to tx - why? why not?
     t.string     :sha,    null: false    ## sha hash as hexstring (but no leading 0)
 end
 
 create_table :txs, :id => :string do |t|
     t.binary     :data     # ,  null: false
-    t.datetime   :date, null: false
+    t.datetime   :date,  null: false
     t.integer    :block, null: false
     t.integer    :idx,   null: false  ## transaction index (number)
 
-    t.string     :from, null: false
-    t.string     :to,   null: false   
+    t.string     :from,  null: false
+    t.string     :to,    null: false   
 
     t.integer    :fee
     t.integer    :value
@@ -62,6 +61,20 @@ ScribeDb.connect( adapter:  'sqlite3',
 ScribeDb.create_all   ## build schema
 ```
 
+and lets import the first hundred (page size is 25)  ethscriptions on mainnet (via the ethscriptions.com api):
+
+
+``` ruby
+require 'scribelite'
+
+ScribeDb.open( './scribe.db' )
+
+(1..4).each do |page|
+   ScribeDb.import_ethscriptions( page: page )
+end
+```
+
+
 
 and to query use it like:
 
@@ -93,9 +106,28 @@ Scribe.order( :num ).limit(limit).each do |scribe|
     print "#{scribe.num} / #{scribe.content_type}   -   #{scribe.tx.date} @ #{scribe.tx.block}"
     print "\n"
 end
-```
 
 
+## Let's query for all inscriptions grouped by date (day) 
+## and dump the results:
+pp Scribe.counts_by_day   
+pp Scribe.counts_by_year
+pp Scribe.counts_by_month
+pp Scribe.counts_by_hour
+
+
+## Let's query for all content types and group by count (descending) 
+## and dump the results:
+pp Scribe.counts_by_content_type
+
+pp Scribe.counts_by_block
+pp Scribe.counts_by_block_with_timestamp
+
+pp Scribe.counts_by_address   # from (creator/minter) address 
+
+# ...
+```
+
 To be continued...
 
 
@@ -110,7 +142,9 @@ at the ruby code commons (rubycocos) org.
 
 ## Questions? Comments?
 
-Join us in the [Rubidity (community) discord (chat server)](https://discord.gg/3JRnDUap6y). Yes you can.
+Join us in the [0xCompute discord (chat server)](https://discord.gg/3JRnDUap6y) 
+(or in the more general Ethscription discord). 
+Yes you can.
 Your questions and commentary welcome.
 
 Or post them over at the [Help & Support](https://github.com/geraldb/help) page. Thanks.

data/Rakefile

@@ -8,7 +8,7 @@ Hoe.spec 'scribelite' do
   self.summary = "scribelite gem - inscription / inscribe (ethscription calldata) database for ethereum & co; let's you query via sql and more"
   self.description = summary
 
-  self.urls    = { home: 'https://github.com/s6ruby/rubidity' }
+  self.urls    = { home: 'https://github.com/0xCompute/ethscribe' }
 
   self.author  = 'Gerald Bauer'
   self.email   = 'gerald.bauer@gmail.com'

data/lib/scribelite/importer.rb

@@ -0,0 +1,155 @@
+
+
+module ScribeDb
+
+
+
+## note: by default - sort asc(ending) - oldest first (0,1,2,3, .etc.)
+def self.import_ethscriptions( page: 1, 
+                               per_page: 50, 
+                               sort_order: 'asc' )
+    net   = Ethscribe.config.client
+    recs = net.ethscriptions( page: page,
+                              per_page: per_page, 
+                              sort_order: sort_order )
+
+    puts "  #{recs.size} record(s)"                          
+
+    recs.each_with_index do |rec,i|
+        puts "==> page #{page}/#{i+1} - #{rec['transaction_hash']}..."
+        _import_ethscription( rec )
+    end
+
+    recs.size   ## return number of records fetched for now
+end  # method import_ethscriptions
+
+
+def self.sync_facet_txs
+  net   = Ethscribe.config.client
+
+  ## add block argument - why? why not? 
+  ## get count by block e.g.
+  ##   our_count = Scribe.joins(:tx).where( 'block < ?', new_block_number).count
+ 
+
+  count = Scribe.count
+ 
+  block =  if count > 0
+               last_scribe =  Scribe.where( num: Scribe.maximum(:num) ).first
+               last_scribe.tx.block + 1
+           else
+             0
+           end
+
+  loop do 
+    res = net.newer_facet_txs( block, max: 2500, count: count )
+
+    break   if res['blocks'].size == 0
+
+    first_block = res['blocks'][0]['block_number']
+    last_block  = res['blocks'][-1]['block_number']
+    puts "==>  #{res['blocks'].size} block(s)  - #{first_block} to #{last_block}..."
+    puts "        total_future_ethscriptions: #{res['total_future_ethscriptions']}"
+
+    batch_count = 0
+    res['blocks'].each do |block_rec|
+        block_rec['ethscriptions'].each do |rec|
+             _import_ethscription( rec )
+             count += 1
+             batch_count += 1
+        end
+    end
+    puts "      #{batch_count} record(s) added"
+
+    block = last_block + 1
+
+    break  if res['total_future_ethscriptions'] == 0
+  end
+end  # method sync_facet_txs
+class << self
+  alias_method :sync_facet_txns, :sync_facet_txs
+end
+
+
+
+def self._import_ethscription( rec )
+    txid  = rec['transaction_hash']
+    block = rec['block_number']
+    idx   = rec['transaction_index']
+    
+    from  = rec['creator']
+    to    = rec['initial_owner']
+
+    ## todo - double check if daylight saving time (dst) breaks timestamp == utc identity/conversion?
+    ##  2001-02-03T04:05:06+07:00
+    ##  2016-05-29T22:28:15.000Z
+    ##  check if %z can handle  .000Z ??
+    ##  DateTime.strptime( '2016-05-29T22:28:15.000Z', '%Y-%m-%dT%H:%M:%S%z' )
+    ## pp rec['creation_timestamp']
+    date  = DateTime.strptime( rec['creation_timestamp'], '%Y-%m-%dT%H:%M:%S.000Z' )   
+    
+
+    num   = rec['ethscription_number']
+    content_type = rec['mimetype']
+    data = rec['content_uri']
+    sha  = rec['sha']
+
+    duplicate = rec['esip6']
+    flagged   = rec['image_removed_by_request_of_rights_holder']
+
+   ### check - if flagged
+   ##  content_type always set to text/plain
+   ##     and data to data:,  ???
+   ##   (re)set to nil/null - why? why not?
+     if flagged
+        data         = nil
+        content_type = nil
+     end
+    
+    tx_attribs = {
+       id:   txid,
+       block: block,
+       idx:   idx,
+       date:  date,
+       from:  from,
+       to:    to,
+       data:  data,   ## note: for now saved as utf8 string (not hex!!!!)
+    }
+
+    scribe_attribs = {
+       id: txid,
+       num:  num,
+       content_type: content_type,
+       duplicate: duplicate,
+       flagged: flagged,
+       sha:   sha, 
+       bytes: data ? data.length : nil    ## auto-add/calc bytes (content length)
+    }
+
+    if num.nil?
+         puts "!! skipping unconfirmed / unassigned inscribe - no. num"
+         return
+    end
+
+
+    scribe = Scribe.find_by( id: txid )
+    if scribe
+        ## skip for now - found id db
+    else
+        # puts "scribe_attribs:"
+        # pp scribe_attribs
+        Scribe.create!( **scribe_attribs )
+    end
+
+    tx = Tx.find_by( id: txid )
+    if tx
+        ## skip for now - found id db
+    else
+        # puts "tx_attribs:"
+        # pp tx_attribs
+        Tx.create!( **tx_attribs )
+    end
+end
+
+end  # module ScribeDb
+ 

data/lib/scribelite/models/scribe.rb

@@ -17,11 +17,18 @@ module ScribeDb
     ### scope like helpers
     def self.png() where( content_type: 'image/png' ); end
     def self.gif() where( content_type: 'image/gif' ); end
-    def self.jpg() where( content_type: 'image/jpeg' ); end
+    def self.jpg() where( content_type: ['image/jpeg',
+                                         'image/jpg'] ); end
     def self.webp() where( content_type: 'image/webp' ); end
     def self.svg()  where( content_type: 'image/svg+xml' ); end
     def self.avif() where( content_type: 'image/avif' ); end
 
+    def self.pdf()  where( content_type: 'application/pdf' ); end
+
+    def self.facet()  where( content_type: 'application/vnd.facet.tx+json' ); end
+
+
+        
     class << self
        alias_method :jpeg, :jpg
     end     
@@ -31,6 +38,7 @@ module ScribeDb
         where( content_type: [
             'image/png',
             'image/jpeg',
+            'image/jpg',
             'image/gif',
             'image/webp',
             'image/svg+xml',
@@ -98,90 +106,63 @@ module ScribeDb
        .order( Arel.sql( 'COUNT(*) DESC, content_type')).count
    end
 
-   class << self
-      alias_method :biggest, :largest
-      alias_method :counts_by_content_type, :content_type_counts
+
+   def self.block_counts
+       joins(:tx).group( 'block' )
+         .order( 'block').count
+   end 
+
+   def self.block_with_timestamp_counts
+       joins(:tx).group( Arel.sql( "block || ' @ ' || date" ))
+         .order( 'block' ).count
    end
 
- ###
- # instance methods
-def extname
-  ## map mime type to file extname
-  ## see https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
-  ##  for real-world usage, see https://dune.com/dgtl_assets/bitcoin-ordinals-analysis
-  ##  https://github.com/casey/ord/blob/master/src/media.rs
-
-   if content_type.start_with?( 'text/plain' )
-      '.txt'
-   elsif content_type.start_with?( 'text/markdown' )
-      '.md'
-   elsif content_type.start_with?( 'text/html' )
-      '.html'
-   elsif content_type.start_with?( 'text/javascript' ) ||
-         content_type.start_with?( 'application/javascript' )
-      ## note: application/javascript is considered bad practice/legacy
-      '.js'
-   elsif content_type.start_with?( 'image/png' )
-       ## Portable Network Graphics (PNG)
-      '.png'
-   elsif content_type.start_with?( 'image/jpeg' )
-        ##  Joint Photographic Expert Group image (JPEG) 
-       '.jpg'   ## use jpeg - why? why not?
-   elsif content_type.start_with?( 'image/webp' )
-        ## Web Picture format (WEBP)
-       '.webp'   ## note: no three-letter extension available
-   elsif content_type.start_with?( 'image/svg' )
-        ## Scalable Vector Graphics (SVG) 
-       '.svg'
-   elsif content_type.start_with?( 'image/gif' ) 
-       ##  Graphics Interchange Format (GIF)
-       '.gif'
-   elsif content_type.start_with?( 'image/avif' )  
-       ## AV1 Image File Format (AVIF)
-       '.avif'
-   elsif content_type.start_with?( 'application/epub' )
-       '.epub'
-   elsif content_type.start_with?( 'application/pdf' )
-       '.pdf'
-   elsif content_type.start_with?( 'application/json' )
-       '.json'
-   elsif content_type.start_with?( 'application/pgp-signature' )
-       '.sig'
-   elsif content_type.start_with?( 'audio/mpeg' )
-       '.mp3'
-   elsif content_type.start_with?( 'audio/midi' )
-       '.midi'
-   elsif content_type.start_with?( 'video/mp4' )
-       '.mp4'
-   elsif content_type.start_with?( 'video/webm' )
-       '.wepm'
-   elsif content_type.start_with?( 'audio/mod' )  
-      ## is typo? possible? only one inscription in 20m?
-       '.mod'   ## check/todo/fix if is .wav?? 
-   else
-      puts "!! ERROR - no file extension configured for content type >#{content_type}<; sorry:"
-      pp self
-      exit 1
+
+   def self.date_counts
+      ## note: strftime is SQLite specific/only!!!
+      joins(:tx).group( Arel.sql("strftime('%Y-%m-%d', date)"))
+       .order( Arel.sql("strftime('%Y-%m-%d', date)")).count
    end
-end
-
-=begin
-def export_path  ## default export path
-   numstr = "%08d" % num   ###  e.g. 00000001  
-   "./tmp/#{numstr}#{extname}" 
-end
-def export( path=export_path )
-   if blob
-     write_blob( path, blob.content )
-   else
-      ## todo/fix: raise exception - no content
-      puts "!! ERROR - inscribe has no content (blob); sorry:"
-      pp self
-      exit 1
+
+   def self.month_counts
+      ## note: strftime is SQLite specific/only!!!
+      joins(:tx).group( Arel.sql("strftime('%Y-%m', date)"))
+       .order( Arel.sql("strftime('%Y-%m', date)")).count
    end
-end
-=end
 
+   def self.year_counts
+      ## note: strftime is SQLite specific/only!!!
+      joins(:tx).group( Arel.sql("strftime('%Y', date)"))
+       .order( Arel.sql("strftime('%Y', date)")).count
+   end
+ 
+   def self.hour_counts
+     ## note: strftime is SQLite specific/only!!!
+      joins(:tx).group( Arel.sql("strftime('%Y-%m-%d %Hh', date)"))
+      .order( Arel.sql("strftime('%Y-%m-%d %Hh', date)")).count
+   end
+
+
+   def self.from_counts
+    ## note: from is sql keyword!!!
+    ##  wrap in [] for sqlite - check if works for others!!!  
+      joins(:tx).group( '[from]' )
+       .order( Arel.sql( 'COUNT(*) DESC')).count
+   end 
+
+
+   class << self
+      alias_method :biggest, :largest
+      alias_method :counts_by_content_type, :content_type_counts
+      alias_method :counts_by_date,         :date_counts
+      alias_method :counts_by_day,          :date_counts
+      alias_method :counts_by_month,        :month_counts
+      alias_method :counts_by_year,         :year_counts
+      alias_method :counts_by_hour,         :hour_counts
+      alias_method :counts_by_block,        :block_counts
+      alias_method :counts_by_block_with_timestamp,  :block_with_timestamp_counts
+      alias_method :counts_by_address,      :from_counts
+    end
   end  # class Scribe
   
     end # module Model

data/lib/scribelite/version.rb

@@ -1,7 +1,7 @@
 
 module Scribelite
-  MAJOR = 0 ## todo: namespace inside version or something - why? why not??
-  MINOR = 2
+  MAJOR = 1 ## todo: namespace inside version or something - why? why not??
+  MINOR = 0
   PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 

data/lib/scribelite.rb

@@ -34,8 +34,6 @@ require_relative 'scribelite/models/tx'
 
 require_relative 'scribelite/schema'      
 
-# require_relative 'cache'
-# require_relative 'importer'  ## note: require (soft dep) ordinals gems!!!
 
 
 
@@ -134,6 +132,11 @@ end  # module ScribeDb
 
 
 
+# add ethscriptions / ethscripe importer - why? why not?
+require_relative 'scribelite/importer'
+
+
+
 
 
 ## add convenience helpers
@@ -141,6 +144,13 @@ Scribe    = ScribeDb::Model::Scribe
 Tx        = ScribeDb::Model::Tx
 
 
+
+require 'active_support/number_helper'
+include ActiveSupport::NumberHelper   ## e.g. number_to_human_size
+
+
+
+
 # say hello
 puts Scribelite.banner     ## if defined?($RUBYCOCOS_DEBUG) && $RUBCOCOS_DEBUG
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scribelite
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-25 00:00:00.000000000 Z
+date: 2024-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ethscribe
@@ -162,14 +162,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.1'
 description: scribelite gem - inscription / inscribe (ethscription calldata) database
   for ethereum & co; let's you query via sql and more
 email: gerald.bauer@gmail.com
@@ -185,12 +185,13 @@ files:
 - README.md
 - Rakefile
 - lib/scribelite.rb
+- lib/scribelite/importer.rb
 - lib/scribelite/models/forward.rb
 - lib/scribelite/models/scribe.rb
 - lib/scribelite/models/tx.rb
 - lib/scribelite/schema.rb
 - lib/scribelite/version.rb
-homepage: https://github.com/s6ruby/rubidity
+homepage: https://github.com/0xCompute/ethscribe
 licenses:
 - Public Domain
 metadata: {}