meeseeker 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 857cc4efa38d7428ba13457a6ffc3db8cb01aab92a26a3cf052b15698d91ca56
-  data.tar.gz: 3f6398e6bbad560624fe0b5e331f0cba2b3b207edfa15074627b7f964edd262d
+SHA1:
+  metadata.gz: 3204b511fca2e18ee8612eb32cd520edaf8bc72e
+  data.tar.gz: 803ab700225d7781a0de824ed1f3221f1dce8033
 SHA512:
-  metadata.gz: 6ae2f1bdea39419820d35ce2fedb63ceac5c0c59d61c63ef9f7a01470baea9cc0d784a630996997750ece65fdee78ec5b8deb7b339b2e8343e82db3c38682527
-  data.tar.gz: 635057da03b642a258436e6d78c0bb2c830ddb475a595c16cfdc80b33b4b92cf58e5782bd4a4cd0b6dd3e85c06f7d137170f2ae6395543086e9070cf595a2cfd
+  metadata.gz: '079b763077af4e430b84183e96b2bc683668a89677aa6d90bf4238f8a1533200c1c0c64ffe3e00a2f838ecb1e99e6c12a93f3be9ea4e188b24e501de8cd596cd'
+  data.tar.gz: 13cc852acec9b743017e20dc58c6566f9acb0f513a0feb30187536540abe8cd6aacb09707f981adf0f69022b5bdb699211fb76096c6a8ee84d310ea1869ef109

data/README.md

@@ -121,7 +121,7 @@ When running `meeseeker sync`, the following channels are available:
 
 As mentioned in the first `whatever` example, for ops, [all operation types](https://developers.steem.io/apidefinitions/broadcast-ops) can be subscribed to as channels, including virtual operations, if enabled.
 
-In the second `whatever` example, for `custom_json.id`, if you want to subscribe to the `follow` channel, use `steem:op:custom_json:follow`.  Or if you want to subscribe to the `sm_team_reveal` channel, use `steem:op:custom_json:follow`.  The `custom_json.id` channels are not enabled by default.  To enable it, set the `MEESEEKER_PUBLISH_OP_CUSTOM_ID` to `true` (see example below).
+In the second `whatever` example, for `custom_json.id`, if you want to subscribe to the `follow` channel, use `steem:op:custom_json:follow`.  Or if you want to subscribe to the `sm_team_reveal` channel, use `steem:op:custom_json:sm_team_reveal`.  The `custom_json.id` channels are not enabled by default.  To enable it, set the `MEESEEKER_PUBLISH_OP_CUSTOM_ID` to `true` (see example below).
 
 For example, from `redis-cli`, if we wanted to stream block numbers:
 

data/Rakefile

@@ -99,8 +99,11 @@ task :reset, [:chain] => [:check_schema] do |t, args|
   print 'Dropping keys for set: %s ...' % chain.to_s
   
   case chain
-  when :steem, :all then keys += Meeseeker.redis.keys('steem:*')
-  when :steem_engine, :all then keys += Meeseeker.redis.keys('steem_engine:*')
+  when :steem then keys += Meeseeker.redis.keys('steem:*')
+  when :steem_engine then keys += Meeseeker.redis.keys('steem_engine:*')
+  when :all
+    keys += Meeseeker.redis.keys('steem:*')
+    keys += Meeseeker.redis.keys('steem_engine:*')
   end
   
   if keys.any?
@@ -323,12 +326,13 @@ namespace :verify do
           
           # If we have all the keys, we should also have all transaction ids.
           expected_ids = keys.map { |k| k.split(':')[2] }.uniq
+          expected_ids -= [Meeseeker::VIRTUAL_TRX_ID]
           actual_ids = nil
           
           agent.block(block_num).tap do |block|
             raise 'Got empty block result.' if block.nil?
             
-            actual_ids = block['transactions'].map{|trx| trx['transactionId'].split('-').first}.uniq
+            actual_ids = block['transactions'].map{|trx| trx['transactionId'].to_s.split('-').first}.uniq
           end
           
           # We do an intersection to make sure there's no difference between
@@ -360,6 +364,8 @@ namespace :verify do
         end
       end
     end
+    
+    agent.shutdown
   end
   
   desc 'Verifies Steem Engine sidechain against the mainnet.'
@@ -377,7 +383,7 @@ namespace :verify do
       block_num = transaction['refSteemBlockNumber']
       
       block_trxs[block_num] ||= []
-      block_trxs[block_num] << transaction['transactionId'].split('-').first
+      block_trxs[block_num] << transaction['transactionId'].to_s.split('-').first
     end
     
     puts "Related mainnet blocks: #{block_trxs.keys.size}"
@@ -457,7 +463,6 @@ namespace :verify do
         schedule_channel = 'steem:witness:schedule'
         redis_url = ENV.fetch('MEESEEKER_REDIS_URL', 'redis://127.0.0.1:6379/0')
         subscription = Redis.new(url: redis_url)
-        ctx = Redis.new(url: redis_url)
         timeout = (max_blocks).to_i * 3
         
         subscribe_mode, subscribe_args = if timeout > 0

data/lib/meeseeker/block_follower_job.rb

@@ -18,7 +18,7 @@ module Meeseeker
             trx_index += 1
           else
             if !!last_key_prefix
-              n, b, t = last_key_prefix.split(':')
+              _, b, t = last_key_prefix.split(':')
               transaction_payload = {
                 block_num: b.to_i,
                 transaction_id: t,
@@ -129,12 +129,12 @@ module Meeseeker
           # to embed it into op values.  This should also reduce streaming
           # overhead since we no longer stream block_headers inder the hood.
           
-          stream.blocks(options) do |block, block_num|
-            block.transactions.each_with_index do |transaction, index|
+          stream.blocks(options) do |b, n|
+            b.transactions.each_with_index do |transaction, index|
               transaction.operations.each do |op|
-                op = op.merge(timestamp: block.timestamp)
+                op = op.merge(timestamp: b.timestamp)
                 
-                yield op, block.transaction_ids[index], block_num
+                yield op, b.transaction_ids[index], n
               end
             end
             
@@ -152,7 +152,7 @@ module Meeseeker
             
             loop do
               condenser_api ||= Steem::CondenserApi.new(url: Meeseeker.node_url)
-              condenser_api.get_ops_in_block(block_num, true) do |vops|
+              condenser_api.get_ops_in_block(n, true) do |vops|
                 redo if vops.nil?
                 
                 if vops.empty? && mode != :head
@@ -169,13 +169,13 @@ module Meeseeker
                     redo
                   end
                   
-                  puts "Gave up retrying virtual ops lookup on block #{block_num}"
+                  puts "Gave up retrying virtual ops lookup on block #{n}"
                   
                   break
                 end
                 
                 if retries > 0
-                  puts "Found virtual ops for block #{block_num} aftere #{retries} retrie(s)"
+                  puts "Found virtual ops for block #{n} aftere #{retries} retrie(s)"
                 end
                 
                 vops.each do |vop|

data/lib/meeseeker/steem_engine/follower_job.rb

@@ -1,4 +1,6 @@
 module Meeseeker::SteemEngine
+  MAX_RETRY_INTERVAL = 18.0
+  
   class FollowerJob
     def perform(options = {})
       redis = Meeseeker.redis
@@ -7,9 +9,12 @@ module Meeseeker::SteemEngine
       current_block_num = nil
       block_transactions = []
       
-      stream_transactions(options) do |transaction, block|
+      stream_transactions(options) do |data, block|
+        transaction = data[:transaction]
+        virtual = !!data[:virtual]
+        
         begin
-          trx_id = transaction['transactionId'].split('-').first
+          trx_id = transaction['transactionId'].to_s.split('-').first
           block_num = block['blockNumber']
           current_key_prefix = "steem_engine:#{block_num}:#{trx_id}"
           contract = transaction['contract']
@@ -19,7 +24,7 @@ module Meeseeker::SteemEngine
             trx_index += 1
           else
             if !!last_key_prefix
-              n, b, t = last_key_prefix.split(':')
+              _, b, t = last_key_prefix.split(':')
               transaction_payload = {
                 block_num: b.to_i,
                 transaction_id: t,
@@ -27,7 +32,14 @@ module Meeseeker::SteemEngine
               }
               
               block_transactions << trx_id
-              redis.publish('steem_engine:transaction', transaction_payload.to_json)
+              
+              trx_pub_key = if !!virtual
+                'steem_engine:virtual_transaction'
+              else
+                'steem_engine:transaction'
+              end
+              
+              redis.publish(trx_pub_key, transaction_payload.to_json)
             end
             
             last_key_prefix = "steem_engine:#{block_num}:#{trx_id}"
@@ -57,10 +69,31 @@ module Meeseeker::SteemEngine
       end
     end
   private
+    def agent
+      @agent ||= Agent.new
+    end
+    
+    def agent_reset
+      return if @agent.nil?
+      
+      @agent.shutdown
+      @agent = nil
+    end
+    
+    def retry_interval
+      @retry_interval ||= 0.1
+      @retry_interval *= 2
+      
+      [@retry_interval, MAX_RETRY_INTERVAL].min
+    end
+    
+    def reset_retry_interval
+      @retry_interval = nil
+    end
+    
     def stream_transactions(options = {}, &block)
       redis = Meeseeker.redis
       last_block_num = nil
-      agent = Agent.new
       until_block_num = options[:until_block_num].to_i
       
       if !!options[:at_block_num]
@@ -96,7 +129,15 @@ module Meeseeker::SteemEngine
       block_num = last_block_num
       
       loop do
-        block = agent.block(block_num)
+        begin
+          block = agent.block(block_num)
+          reset_retry_interval
+        rescue Net::HTTP::Persistent::Error => e
+          puts "Retrying: #{e}"
+          agent_reset
+          sleep retry_interval
+          redo
+        end
         
         if block.nil?
           sleep 3 # sleep for one mainnet block interval
@@ -106,7 +147,19 @@ module Meeseeker::SteemEngine
         transactions = block['transactions']
         
         transactions.each do |transaction|
-          yield transaction.merge(timestamp: block['timestamp']), block
+          yield({transaction: transaction.merge(timestamp: block['timestamp'])}, block)
+        end
+        
+        virtual_transactions = block['virtualTransactions']
+        
+        virtual_transactions.each do |virtual_transaction|
+          _, vtrx_in_block = virtual_transaction['transactionId'].split('-')
+          virtual_transaction = virtual_transaction.merge(
+            timestamp: block['timestamp'],
+            'transactionId' => "#{Meeseeker::VIRTUAL_TRX_ID}-#{vtrx_in_block}"
+          )
+          
+          yield({transaction: virtual_transaction, virtual: true}, block)
         end
         
         break if until_block_num != 0 && block_num > until_block_num

data/lib/meeseeker/version.rb

@@ -1,4 +1,4 @@
 module Meeseeker
-  VERSION = '0.0.7'
+  VERSION = '0.0.8'
   AGENT_ID = "meeseeker/#{VERSION}"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: meeseeker
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Anthony Martin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-04 00:00:00.000000000 Z
+date: 2019-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -255,7 +255,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: Redis based block follower is an efficient way for multiple apps to stream