meeseeker 0.0.8 → 2.0.1.pre.1

data/bin/meeseeker

@@ -32,7 +32,7 @@ when 'sync', 'witness:schedule'
     Rake::Task['sync'].reenable
   end; end
 when 'find' then Rake::Task['find'].invoke(*ARGV[1..-1])
-when 'reset' then Rake::Task['reset'].invoke
+when 'reset' then Rake::Task['reset'].invoke(*ARGV[1..-1])
 else
   puts "\nBegin/resume sync:"
   puts "\t#{filename} sync [chain] [block_num]\n\n"
@@ -44,7 +44,7 @@ else
   puts "\t#{filename} find block 3044538"
   puts "\t#{filename} find trx 983c5e5c6aef52f1647d952a18771f76b885e6de\n\n"
   puts 'Clear all keys:'
-  puts "\t#{filename} reset\n\n"
+  puts "\t#{filename} reset [chain]\n\n"
   puts "Note, using find and reset is not intended for routine operations.  It is intended for analysis only."
   puts "See: https://github.com/inertia186/meeseeker#usage"
 end

data/lib/meeseeker.rb

@@ -1,28 +1,200 @@
 require 'redis'
 require 'steem'
+require 'hive'
 
 require 'meeseeker/version'
 require 'meeseeker/block_follower_job'
 require 'meeseeker/witness_schedule_job'
 require 'meeseeker/steem_engine/agent'
 require 'meeseeker/steem_engine/follower_job'
+require 'meeseeker/hive_engine'
 
 module Meeseeker
-  LAST_BLOCK_NUM_KEY = 'steem:meeseeker:last_block_num'
-  LAST_STEEM_ENGINE_BLOCK_NUM_KEY = 'steem_engine:meeseeker:last_block_num'
+  STEEM_CHAIN_ID = '0000000000000000000000000000000000000000000000000000000000000000'
+  HIVE_LEGACY_CHAIN_ID = '0000000000000000000000000000000000000000000000000000000000000000'
+  HIVE_CHAIN_ID = 'beeab0de00000000000000000000000000000000000000000000000000000000'
+  STEEM_CHAIN_KEY_PREFIX = 'steem'
+  HIVE_CHAIN_KEY_PREFIX = 'hive'
+  STEEM_ENGINE_CHAIN_KEY_PREFIX = 'steem_engine'
+  HIVE_ENGINE_CHAIN_KEY_PREFIX = 'hive_engine'
+  LAST_BLOCK_NUM_KEY_SUFFIX = ':meeseeker:last_block_num'
+  LAST_STEEM_ENGINE_BLOCK_NUM_KEY_SUFFIX = ':meeseeker:last_block_num'
   BLOCKS_PER_DAY = 28800
   VIRTUAL_TRX_ID = '0000000000000000000000000000000000000000'
+  BLOCK_INTERVAL = 3
+  SHUFFLE_URL = 'shuffle'
+  DEFAULT_STEEM_URL = 'https://api.steemit.com'
+  DEFAULT_STEEM_FAILOVER_URLS = [
+    DEFAULT_STEEM_URL,
+    # 'https://steemd.minnowsupportproject.org',
+    # 'https://anyx.io',
+    # 'http://anyx.io',
+    # 'https://steemd.privex.io',
+    # 'https://api.steem.house'
+  ]
+  DEFAULT_HIVE_URL = 'https://api.openhive.network'
+  DEFAULT_HIVE_FAILOVER_URLS = [
+    DEFAULT_HIVE_URL,
+    'https://api.hivekings.com',
+    'https://anyx.io',
+    'http://anyx.io',
+    'https://techcoderx.com',
+    'https://rpc.esteem.app',
+    'https://hived.privex.io',
+    'https://api.pharesim.me',
+    'https://api.hive.blog',
+    'https://rpc.ausbit.dev'
+  ]
+  
+  def default_chain_key_prefix
+    ENV.fetch('MEESEEKER_CHAIN_KEY_PREFIX', chain_key_prefix)
+  end
+  
+  def self.chain_key_prefix
+    @chain_key_prefix ||= {}
+    url = default_url(HIVE_CHAIN_KEY_PREFIX)
+    
+    return @chain_key_prefix[url] if !!@chain_key_prefix[url]
+    
+    # Just use the Hive API for either chain, until we know which one we're
+    # using.
+    api = Hive::DatabaseApi.new(url: url)
+    
+    api.get_config do |config|
+      @chain_key_prefix[node_url] = if !!config.HIVE_CHAIN_ID && config.HIVE_CHAIN_ID == HIVE_CHAIN_ID
+        HIVE_CHAIN_KEY_PREFIX
+      elsif !!config.HIVE_CHAIN_ID && config.HIVE_CHAIN_ID == HIVE_LEGACY_CHAIN_ID
+        HIVE_CHAIN_KEY_PREFIX
+      elsif !!config.STEEM_CHAIN_ID && config.STEEM_CHAIN_ID == STEEM_CHAIN_ID
+        STEEM_CHAIN_KEY_PREFIX
+      else
+        config.keys.find{|k| k.end_with? '_CHAIN_ID'}.split('_').first.downcase.tap do |guess|
+          warn "Guessing chain_key_prefix = '#{guess}' for unknown chain on: #{node_url}"
+        end
+      end
+    end
+  end
+  
+  def self.default_url(chain = default_chain_key_prefix)
+    ENV.fetch('MEESEEKER_NODE_URL') do
+      case chain.to_s
+      when STEEM_CHAIN_KEY_PREFIX then DEFAULT_STEEM_URL
+      when HIVE_CHAIN_KEY_PREFIX then DEFAULT_HIVE_URL
+      else
+        raise "Unknown chain: #{chain}"
+      end
+    end
+  end
+  
+  @problem_node_urls = []
+  
   @redis = Redis.new(url: ENV.fetch('MEESEEKER_REDIS_URL', 'redis://127.0.0.1:6379/0'))
-  @node_url = ENV.fetch('MEESEEKER_NODE_URL', 'https://api.steemit.com')
+  @node_url = default_url(ENV.fetch('MEESEEKER_CHAIN_KEY_PREFIX', HIVE_CHAIN_KEY_PREFIX))
   @steem_engine_node_url = ENV.fetch('MEESEEKER_STEEM_ENGINE_NODE_URL', 'https://api.steem-engine.com/rpc')
+  @hive_engine_node_url = ENV.fetch('MEESEEKER_HIVE_ENGINE_NODE_URL', 'https://api.hive-engine.com/rpc')
   @stream_mode = ENV.fetch('MEESEEKER_STREAM_MODE', 'head').downcase.to_sym
   @include_virtual = ENV.fetch('MEESEEKER_INCLUDE_VIRTUAL', 'true').downcase == 'true'
   @include_block_header = ENV.fetch('MEESEEKER_INCLUDE_BLOCK_HEADER', 'true').downcase == 'true'
   @publish_op_custom_id = ENV.fetch('MEESEEKER_PUBLISH_OP_CUSTOM_ID', 'false').downcase == 'true'
-  @expire_keys = ENV.fetch('MEESEEKER_EXPIRE_KEYS', BLOCKS_PER_DAY * 3).to_i
+  @expire_keys = ENV.fetch('MEESEEKER_EXPIRE_KEYS', BLOCKS_PER_DAY * BLOCK_INTERVAL).to_i
+  @max_keys = ENV.fetch('MEESEEKER_MAX_KEYS', '-1').to_i
   
   extend self
 
-  attr_accessor :redis, :node_url, :steem_engine_node_url, :expire_keys,
-    :stream_mode, :include_virtual, :include_block_header, :publish_op_custom_id
+  attr_accessor :redis, :node_url, :steem_engine_node_url,
+    :hive_engine_node_url, :expire_keys, :max_keys, :stream_mode,
+    :include_virtual, :include_block_header, :publish_op_custom_id
+  
+  def self.shuffle_node_url(chain = ENV.fetch('MEESEEKER_CHAIN_KEY_PREFIX', HIVE_CHAIN_KEY_PREFIX))
+    chain = chain.to_s
+    node_url = ENV.fetch('MEESEEKER_NODE_URL', default_url(ENV.fetch('MEESEEKER_CHAIN_KEY_PREFIX', chain)))
+    return node_url unless node_url == SHUFFLE_URL
+    
+    @problem_node_urls = [] if rand(1..1000) == 13
+    shuffle_node_url!(chain)
+  end
+  
+  def self.api_class(chain = default_chain_key_prefix)
+    case chain.to_s
+    when STEEM_CHAIN_KEY_PREFIX then Steem::Api
+    when HIVE_CHAIN_KEY_PREFIX then Hive::Api
+    else
+      raise "Unknown chain: #{chain}"
+    end
+  end
+  
+  def self.condenser_api_class(chain = default_chain_key_prefix)
+    case chain.to_s
+    when STEEM_CHAIN_KEY_PREFIX then Steem::CondenserApi
+    when HIVE_CHAIN_KEY_PREFIX then Hive::CondenserApi
+    else
+      raise "Unknown chain: #{chain}"
+    end
+  end
+  
+  def self.block_api_class(chain = default_chain_key_prefix)
+    case chain.to_s
+    when STEEM_CHAIN_KEY_PREFIX then Steem::BlockApi
+    when HIVE_CHAIN_KEY_PREFIX then Hive::BlockApi
+    else
+      raise "Unknown chain: #{chain}"
+    end
+  end
+  
+  def self.database_api_class(chain = default_chain_key_prefix)
+    case chain.to_s
+    when STEEM_CHAIN_KEY_PREFIX then Steem::DatabaseApi
+    when HIVE_CHAIN_KEY_PREFIX then Hive::DatabaseApi
+    else
+      raise "Unknown chain: #{chain}"
+    end
+  end
+  
+  def self.stream_class(chain = default_chain_key_prefix)
+    case chain.to_s
+    when STEEM_CHAIN_KEY_PREFIX then Steem::Stream
+    when HIVE_CHAIN_KEY_PREFIX then Hive::Stream
+    else
+      raise "Unknown chain: #{chain}"
+    end
+  end
+  
+  def self.shuffle_node_url!(chain = ENV.fetch('MEESEEKER_CHAIN_KEY_PREFIX', HIVE_CHAIN_KEY_PREFIX))
+    chain = chain.to_s
+    failover_urls = case chain
+      when STEEM_CHAIN_KEY_PREFIX then DEFAULT_STEEM_FAILOVER_URLS - @problem_node_urls
+      when HIVE_CHAIN_KEY_PREFIX then DEFAULT_HIVE_FAILOVER_URLS - @problem_node_urls
+      else; []
+    end
+    url = failover_urls.sample
+    api = api_class(chain).new(url: url)
+    
+    api.get_accounts(['fullnodeupdate']) do |accounts|
+      fullnodeupdate = accounts.first
+      metadata = (JSON[fullnodeupdate.json_metadata] rescue nil) || {}
+      
+      nodes = metadata.fetch('report', []).map do |report|
+        next if chain == HIVE_CHAIN_KEY_PREFIX && !report[HIVE_CHAIN_KEY_PREFIX]
+        next if chain != HIVE_CHAIN_KEY_PREFIX && !!report[HIVE_CHAIN_KEY_PREFIX]
+        
+        report['node']
+      end.compact.uniq
+      
+      nodes -= @problem_node_urls
+      
+      if nodes.any?
+        nodes.sample
+      else
+        @node_url = failover_urls.sample
+      end
+    end
+  rescue => e
+    puts "#{url}: #{e}"
+    
+    @problem_node_urls << url
+    failover_urls -= @problem_node_urls
+    failover_urls.sample
+  end
+  
+  shuffle_node_url! if @node_url == SHUFFLE_URL
 end

data/lib/meeseeker/block_follower_job.rb

@@ -1,18 +1,24 @@
+require 'radiator'
+
 module Meeseeker
   class BlockFollowerJob
     MAX_VOP_RETRY = 3
     
     def perform(options = {})
-      block_api = Steem::BlockApi.new(url: Meeseeker.node_url)
+      chain = (options[:chain] || 'hive').to_sym
+      url = Meeseeker.default_url(chain)
+      block_api = Meeseeker.block_api_class(chain).new(url: url)
       redis = Meeseeker.redis
       last_key_prefix = nil
       trx_index = 0
       current_block_num = nil
       block_transactions = []
+      chain_key_prefix = chain.to_s if !!options[:chain]
+      chain_key_prefix ||= Meeseeker.default_chain_key_prefix
       
       stream_operations(options) do |op, trx_id, block_num|
         begin
-          current_key_prefix = "steem:#{block_num}:#{trx_id}"
+          current_key_prefix = "#{chain_key_prefix}:#{block_num}:#{trx_id}"
           
           if current_key_prefix == last_key_prefix
             trx_index += 1
@@ -26,17 +32,28 @@ module Meeseeker
               }
               
               block_transactions << trx_id unless trx_id == VIRTUAL_TRX_ID
-              redis.publish('steem:transaction', transaction_payload.to_json)
+              redis.publish("#{chain_key_prefix}:transaction", transaction_payload.to_json)
             end
-            last_key_prefix = "steem:#{block_num}:#{trx_id}"
+            last_key_prefix = "#{chain_key_prefix}:#{block_num}:#{trx_id}"
             trx_index = 0
           end
           
-          op_type = op.type.split('_')[0..-2].join('_')
+          op_type = if op.type.end_with? '_operation'
+            op.type.split('_')[0..-2].join('_')
+          else
+            op.type
+          end
+          
           key = "#{current_key_prefix}:#{trx_index}:#{op_type}"
           puts key
         end
         
+        unless Meeseeker.max_keys == -1
+          while redis.keys("#{chain_key_prefix}:*").size > Meeseeker.max_keys
+            sleep Meeseeker::BLOCK_INTERVAL
+          end
+        end
+        
         redis.set(key, op.to_json)
         redis.expire(key, Meeseeker.expire_keys) unless Meeseeker.expire_keys == -1
         
@@ -49,26 +66,30 @@ module Meeseeker
           if Meeseeker.include_block_header
             catch :block_header do
               block_api.get_block_header(block_num: block_num) do |result|
-                throw :block_header if result.nil || result.header.nil?
+                if result.nil? || result.header.nil?
+                  puts "Node returned empty result for block_header on block_num: #{block_num} (rate limiting?).  Retrying ..."
+                  sleep Meeseeker::BLOCK_INTERVAL
+                  throw :block_header
+                end
                 
                 block_payload.merge!(result.header.to_h)
               end
             end
           end
           
-          redis.set(LAST_BLOCK_NUM_KEY, block_num)
-          redis.publish('steem:block', block_payload.to_json)
+          redis.set(chain_key_prefix + LAST_BLOCK_NUM_KEY_SUFFIX, block_num)
+          redis.publish("#{chain_key_prefix}:block", block_payload.to_json)
           current_block_num = block_num
         end
         
-        redis.publish("steem:op:#{op_type}", {key: key}.to_json)
+        redis.publish("#{chain_key_prefix}:op:#{op_type}", {key: key}.to_json)
         
         if Meeseeker.publish_op_custom_id
           if %w(custom custom_binary custom_json).include? op_type
             id = (op["value"]["id"] rescue nil).to_s
             
             if id.size > 0
-              redis.publish("steem:op:#{op_type}:#{id}", {key: key}.to_json)
+              redis.publish("#{chain_key_prefix}:op:#{op_type}:#{id}", {key: key}.to_json)
             end
           end
         end
@@ -76,7 +97,10 @@ module Meeseeker
     end
   private
     def stream_operations(options = {}, &block)
+      chain = (options[:chain] || 'hive').to_sym
       redis = Meeseeker.redis
+      chain_key_prefix = chain.to_s if !!options[:chain]
+      chain_key_prefix ||= Meeseeker.chain_key_prefix
       last_block_num = nil
       mode = options.delete(:mode) || Meeseeker.stream_mode
       options[:include_virtual] ||= Meeseeker.include_virtual
@@ -84,8 +108,9 @@ module Meeseeker
       if !!options[:at_block_num]
         last_block_num = options[:at_block_num].to_i
       else
-        database_api = Steem::DatabaseApi.new(url: Meeseeker.node_url)
-        last_block_num = redis.get(LAST_BLOCK_NUM_KEY).to_i + 1
+        url = Meeseeker.default_url(chain)
+        database_api = Meeseeker.database_api_class(chain).new(url: url)
+        last_block_num = redis.get(chain_key_prefix + LAST_BLOCK_NUM_KEY_SUFFIX).to_i + 1
         
         block_num = catch :dynamic_global_properties do
           database_api.get_dynamic_global_properties do |dgpo|
@@ -116,11 +141,12 @@ module Meeseeker
       end
       
       begin
-        stream_options = {url: Meeseeker.node_url, mode: mode}
-        options = options.merge(at_block_num: last_block_num)
+        url = Meeseeker.default_url(chain)
+        stream_options = {chain: chain, url: url}
+        stream_args = [last_block_num, mode]
         condenser_api = nil
         
-        Steem::Stream.new(stream_options).tap do |stream|
+        Radiator::Stream.new(stream_options).tap do |stream|
           puts "Stream begin: #{stream_options.to_json}; #{options.to_json}"
           
           # Prior to v0.0.4, we only streamed operations with stream.operations.
@@ -129,67 +155,89 @@ module Meeseeker
           # to embed it into op values.  This should also reduce streaming
           # overhead since we no longer stream block_headers inder the hood.
           
-          stream.blocks(options) do |b, n|
-            b.transactions.each_with_index do |transaction, index|
-              transaction.operations.each do |op|
-                op = op.merge(timestamp: b.timestamp)
-                
-                yield op, b.transaction_ids[index], n
-              end
-            end
-            
-            next unless !!Meeseeker.include_virtual
-            
-            retries = 0
-            
-            # This is where it gets tricky.  Virtual ops sometims don't show up
-            # right away, especially if we're streaming on head blocks.  In that
-            # situation, we might only need to wait about 1 block.  This loop
-            # will likely one execute one iteration, but we have fallback logic
-            # in case there are complications.
-            # 
-            # See: https://developers.steem.io/tutorials-recipes/virtual-operations-when-streaming-blockchain-transactions
-            
-            loop do
-              condenser_api ||= Steem::CondenserApi.new(url: Meeseeker.node_url)
-              condenser_api.get_ops_in_block(n, true) do |vops|
-                redo if vops.nil?
+          loop do
+            begin
+              stream.blocks(*stream_args) do |b, n, condenser_api|
+                redo if b.nil?
                 
-                if vops.empty? && mode != :head
-                  # Usually, we just need to slow down to allow virtual ops to
-                  # show up after a short delay.  Adding this delay doesn't
-                  # impact overall performance because steem-ruby will batch
-                  # when block streams fall behind.
-                  
-                  if retries < MAX_VOP_RETRY
-                    retries = retries + 1
-                    condenser_api = nil
-                    sleep 3 * retries
+                b.transactions.each_with_index do |transaction, index|
+                  transaction.operations.each do |op|
+                    op_value = op[1].merge(timestamp: b.timestamp)
                     
-                    redo
+                    yield Hashie::Mash.new(type: op[0], value: op_value), b.transaction_ids[index], n
                   end
-                  
-                  puts "Gave up retrying virtual ops lookup on block #{n}"
-                  
-                  break
                 end
                 
-                if retries > 0
-                  puts "Found virtual ops for block #{n} aftere #{retries} retrie(s)"
-                end
+                next unless !!Meeseeker.include_virtual
+                
+                retries = 0
+                
+                # This is where it gets tricky.  Virtual ops sometims don't show up
+                # right away, especially if we're streaming on head blocks.  In that
+                # situation, we might only need to wait about 1 block.  This loop
+                # will likely one execute one iteration, but we have fallback logic
+                # in case there are complications.
+                # 
+                # See: https://developers.steem.io/tutorials-recipes/virtual-operations-when-streaming-blockchain-transactions
                 
-                vops.each do |vop|
-                  normalized_op = Hashie::Mash.new(
-                    type: vop.op[0],
-                    value: vop.op[1], 
-                    timestamp: vop.timestamp
-                  )
+                loop do
+                  # TODO (HF23) Switch to account_history_api.enum_virtual_ops if supported.
+                  url = Meeseeker.default_url(chain)
+                  condenser_api ||= Meeseeker.condenser_api_class(chain).new(url: url)
+                  condenser_api.get_ops_in_block(n, true) do |vops|
+                    if vops.nil?
+                      puts "Node returned empty result for get_ops_in_block on block_num: #{n} (rate limiting?).  Retrying ..."
+                      vops = []
+                    end
+                    
+                    if vops.empty? && mode != :head
+                      # Usually, we just need to slow down to allow virtual ops to
+                      # show up after a short delay.  Adding this delay doesn't
+                      # impact overall performance because steem-ruby will batch
+                      # when block streams fall behind.
+                      
+                      if retries < MAX_VOP_RETRY
+                        retries = retries + 1
+                        condenser_api = nil
+                        sleep Meeseeker::BLOCK_INTERVAL * retries
+                        
+                        redo
+                      end
+                      
+                      puts "Gave up retrying virtual ops lookup on block #{n}"
+                      
+                      break
+                    end
+                    
+                    if retries > 0
+                      puts "Found virtual ops for block #{n} aftere #{retries} retrie(s)"
+                    end
+                    
+                    vops.each do |vop|
+                      normalized_op = Hashie::Mash.new(
+                        type: vop.op[0],
+                        value: vop.op[1], 
+                        timestamp: vop.timestamp
+                      )
+                      
+                      yield normalized_op, vop.trx_id, vop.block
+                    end
+                  end
                   
-                  yield normalized_op, vop.trx_id, vop.block
+                  break
                 end
               end
               
               break
+            rescue => e
+              raise e unless e.to_s.include? 'Request Entity Too Large'
+              
+              # We need to tell steem-ruby to avoid json-rpc-batch on this
+              # node.
+              
+              Meeseeker.block_api_class(chain).const_set 'MAX_RANGE_SIZE', 1
+              sleep Meeseeker::BLOCK_INTERVAL
+              redo
             end
           end
         end