meeseeker 1.0.0 → 2.0.0.pre.1

data/lib/meeseeker.rb

@@ -1,28 +1,200 @@
 require 'redis'
 require 'steem'
+require 'hive'
 
 require 'meeseeker/version'
 require 'meeseeker/block_follower_job'
 require 'meeseeker/witness_schedule_job'
 require 'meeseeker/steem_engine/agent'
 require 'meeseeker/steem_engine/follower_job'
+require 'meeseeker/hive_engine'
 
 module Meeseeker
-  LAST_BLOCK_NUM_KEY = 'steem:meeseeker:last_block_num'
-  LAST_STEEM_ENGINE_BLOCK_NUM_KEY = 'steem_engine:meeseeker:last_block_num'
+  STEEM_CHAIN_ID = '0000000000000000000000000000000000000000000000000000000000000000'
+  HIVE_LEGACY_CHAIN_ID = '0000000000000000000000000000000000000000000000000000000000000000'
+  HIVE_CHAIN_ID = 'beeab0de00000000000000000000000000000000000000000000000000000000'
+  STEEM_CHAIN_KEY_PREFIX = 'steem'
+  HIVE_CHAIN_KEY_PREFIX = 'hive'
+  STEEM_ENGINE_CHAIN_KEY_PREFIX = 'steem_engine'
+  HIVE_ENGINE_CHAIN_KEY_PREFIX = 'hive_engine'
+  LAST_BLOCK_NUM_KEY_SUFFIX = ':meeseeker:last_block_num'
+  LAST_STEEM_ENGINE_BLOCK_NUM_KEY_SUFFIX = ':meeseeker:last_block_num'
   BLOCKS_PER_DAY = 28800
   VIRTUAL_TRX_ID = '0000000000000000000000000000000000000000'
+  BLOCK_INTERVAL = 3
+  SHUFFLE_URL = 'shuffle'
+  DEFAULT_STEEM_URL = 'https://api.steemit.com'
+  DEFAULT_STEEM_FAILOVER_URLS = [
+    DEFAULT_STEEM_URL,
+    # 'https://steemd.minnowsupportproject.org',
+    # 'https://anyx.io',
+    # 'http://anyx.io',
+    # 'https://steemd.privex.io',
+    # 'https://api.steem.house'
+  ]
+  DEFAULT_HIVE_URL = 'https://api.openhive.network'
+  DEFAULT_HIVE_FAILOVER_URLS = [
+    DEFAULT_HIVE_URL,
+    'https://api.hivekings.com',
+    'https://anyx.io',
+    'http://anyx.io',
+    'https://techcoderx.com',
+    'https://rpc.esteem.app',
+    'https://hived.privex.io',
+    'https://api.pharesim.me',
+    'https://api.hive.blog',
+    'https://rpc.ausbit.dev'
+  ]
+  
+  def default_chain_key_prefix
+    ENV.fetch('MEESEEKER_CHAIN_KEY_PREFIX', chain_key_prefix)
+  end
+  
+  def self.chain_key_prefix
+    @chain_key_prefix ||= {}
+    url = default_url(HIVE_CHAIN_KEY_PREFIX)
+    
+    return @chain_key_prefix[url] if !!@chain_key_prefix[url]
+    
+    # Just use the Hive API for either chain, until we know which one we're
+    # using.
+    api = Hive::DatabaseApi.new(url: url)
+    
+    api.get_config do |config|
+      @chain_key_prefix[node_url] = if !!config.HIVE_CHAIN_ID && config.HIVE_CHAIN_ID == HIVE_CHAIN_ID
+        HIVE_CHAIN_KEY_PREFIX
+      elsif !!config.HIVE_CHAIN_ID && config.HIVE_CHAIN_ID == HIVE_LEGACY_CHAIN_ID
+        HIVE_CHAIN_KEY_PREFIX
+      elsif !!config.STEEM_CHAIN_ID && config.STEEM_CHAIN_ID == STEEM_CHAIN_ID
+        STEEM_CHAIN_KEY_PREFIX
+      else
+        config.keys.find{|k| k.end_with? '_CHAIN_ID'}.split('_').first.downcase.tap do |guess|
+          warn "Guessing chain_key_prefix = '#{guess}' for unknown chain on: #{node_url}"
+        end
+      end
+    end
+  end
+  
+  def self.default_url(chain = default_chain_key_prefix)
+    ENV.fetch('MEESEEKER_NODE_URL') do
+      case chain.to_s
+      when STEEM_CHAIN_KEY_PREFIX then DEFAULT_STEEM_URL
+      when HIVE_CHAIN_KEY_PREFIX then DEFAULT_HIVE_URL
+      else
+        raise "Unknown chain: #{chain}"
+      end
+    end
+  end
+  
+  @problem_node_urls = []
+  
   @redis = Redis.new(url: ENV.fetch('MEESEEKER_REDIS_URL', 'redis://127.0.0.1:6379/0'))
-  @node_url = ENV.fetch('MEESEEKER_NODE_URL', 'https://api.steemit.com')
+  @node_url = default_url(ENV.fetch('MEESEEKER_CHAIN_KEY_PREFIX', HIVE_CHAIN_KEY_PREFIX))
   @steem_engine_node_url = ENV.fetch('MEESEEKER_STEEM_ENGINE_NODE_URL', 'https://api.steem-engine.com/rpc')
+  @hive_engine_node_url = ENV.fetch('MEESEEKER_HIVE_ENGINE_NODE_URL', 'https://api.hive-engine.com/rpc')
   @stream_mode = ENV.fetch('MEESEEKER_STREAM_MODE', 'head').downcase.to_sym
   @include_virtual = ENV.fetch('MEESEEKER_INCLUDE_VIRTUAL', 'true').downcase == 'true'
   @include_block_header = ENV.fetch('MEESEEKER_INCLUDE_BLOCK_HEADER', 'true').downcase == 'true'
   @publish_op_custom_id = ENV.fetch('MEESEEKER_PUBLISH_OP_CUSTOM_ID', 'false').downcase == 'true'
-  @expire_keys = ENV.fetch('MEESEEKER_EXPIRE_KEYS', BLOCKS_PER_DAY * 3).to_i
+  @expire_keys = ENV.fetch('MEESEEKER_EXPIRE_KEYS', BLOCKS_PER_DAY * BLOCK_INTERVAL).to_i
+  @max_keys = ENV.fetch('MEESEEKER_MAX_KEYS', '-1').to_i
   
   extend self
 
-  attr_accessor :redis, :node_url, :steem_engine_node_url, :expire_keys,
-    :stream_mode, :include_virtual, :include_block_header, :publish_op_custom_id
+  attr_accessor :redis, :node_url, :steem_engine_node_url,
+    :hive_engine_node_url, :expire_keys, :max_keys, :stream_mode,
+    :include_virtual, :include_block_header, :publish_op_custom_id
+  
+  def self.shuffle_node_url(chain = ENV.fetch('MEESEEKER_CHAIN_KEY_PREFIX', HIVE_CHAIN_KEY_PREFIX))
+    chain = chain.to_s
+    node_url = ENV.fetch('MEESEEKER_NODE_URL', default_url(ENV.fetch('MEESEEKER_CHAIN_KEY_PREFIX', chain)))
+    return node_url unless node_url == SHUFFLE_URL
+    
+    @problem_node_urls = [] if rand(1..1000) == 13
+    shuffle_node_url!(chain)
+  end
+  
+  def self.api_class(chain = default_chain_key_prefix)
+    case chain.to_s
+    when STEEM_CHAIN_KEY_PREFIX then Steem::Api
+    when HIVE_CHAIN_KEY_PREFIX then Hive::Api
+    else
+      raise "Unknown chain: #{chain}"
+    end
+  end
+  
+  def self.condenser_api_class(chain = default_chain_key_prefix)
+    case chain.to_s
+    when STEEM_CHAIN_KEY_PREFIX then Steem::CondenserApi
+    when HIVE_CHAIN_KEY_PREFIX then Hive::CondenserApi
+    else
+      raise "Unknown chain: #{chain}"
+    end
+  end
+  
+  def self.block_api_class(chain = default_chain_key_prefix)
+    case chain.to_s
+    when STEEM_CHAIN_KEY_PREFIX then Steem::BlockApi
+    when HIVE_CHAIN_KEY_PREFIX then Hive::BlockApi
+    else
+      raise "Unknown chain: #{chain}"
+    end
+  end
+  
+  def self.database_api_class(chain = default_chain_key_prefix)
+    case chain.to_s
+    when STEEM_CHAIN_KEY_PREFIX then Steem::DatabaseApi
+    when HIVE_CHAIN_KEY_PREFIX then Hive::DatabaseApi
+    else
+      raise "Unknown chain: #{chain}"
+    end
+  end
+  
+  def self.stream_class(chain = default_chain_key_prefix)
+    case chain.to_s
+    when STEEM_CHAIN_KEY_PREFIX then Steem::Stream
+    when HIVE_CHAIN_KEY_PREFIX then Hive::Stream
+    else
+      raise "Unknown chain: #{chain}"
+    end
+  end
+  
+  def self.shuffle_node_url!(chain = ENV.fetch('MEESEEKER_CHAIN_KEY_PREFIX', HIVE_CHAIN_KEY_PREFIX))
+    chain = chain.to_s
+    failover_urls = case chain
+      when STEEM_CHAIN_KEY_PREFIX then DEFAULT_STEEM_FAILOVER_URLS - @problem_node_urls
+      when HIVE_CHAIN_KEY_PREFIX then DEFAULT_HIVE_FAILOVER_URLS - @problem_node_urls
+      else; []
+    end
+    url = failover_urls.sample
+    api = api_class(chain).new(url: url)
+    
+    api.get_accounts(['fullnodeupdate']) do |accounts|
+      fullnodeupdate = accounts.first
+      metadata = (JSON[fullnodeupdate.json_metadata] rescue nil) || {}
+      
+      nodes = metadata.fetch('report', []).map do |report|
+        next if chain == HIVE_CHAIN_KEY_PREFIX && !report[HIVE_CHAIN_KEY_PREFIX]
+        next if chain != HIVE_CHAIN_KEY_PREFIX && !!report[HIVE_CHAIN_KEY_PREFIX]
+        
+        report['node']
+      end.compact.uniq
+      
+      nodes -= @problem_node_urls
+      
+      if nodes.any?
+        nodes.sample
+      else
+        @node_url = failover_urls.sample
+      end
+    end
+  rescue => e
+    puts "#{url}: #{e}"
+    
+    @problem_node_urls << url
+    failover_urls -= @problem_node_urls
+    failover_urls.sample
+  end
+  
+  shuffle_node_url! if @node_url == SHUFFLE_URL
 end

data/lib/meeseeker/block_follower_job.rb

@@ -3,16 +3,20 @@ module Meeseeker
     MAX_VOP_RETRY = 3
     
     def perform(options = {})
-      block_api = Steem::BlockApi.new(url: Meeseeker.node_url)
+      chain = (options[:chain] || 'hive').to_sym
+      url = Meeseeker.default_url(chain)
+      block_api = Meeseeker.block_api_class(chain).new(url: url)
       redis = Meeseeker.redis
       last_key_prefix = nil
       trx_index = 0
       current_block_num = nil
       block_transactions = []
+      chain_key_prefix = chain.to_s if !!options[:chain]
+      chain_key_prefix ||= Meeseeker.default_chain_key_prefix
       
       stream_operations(options) do |op, trx_id, block_num|
         begin
-          current_key_prefix = "steem:#{block_num}:#{trx_id}"
+          current_key_prefix = "#{chain_key_prefix}:#{block_num}:#{trx_id}"
           
           if current_key_prefix == last_key_prefix
             trx_index += 1
@@ -26,9 +30,9 @@ module Meeseeker
               }
               
               block_transactions << trx_id unless trx_id == VIRTUAL_TRX_ID
-              redis.publish('steem:transaction', transaction_payload.to_json)
+              redis.publish("#{chain_key_prefix}:transaction", transaction_payload.to_json)
             end
-            last_key_prefix = "steem:#{block_num}:#{trx_id}"
+            last_key_prefix = "#{chain_key_prefix}:#{block_num}:#{trx_id}"
             trx_index = 0
           end
           
@@ -42,6 +46,12 @@ module Meeseeker
           puts key
         end
         
+        unless Meeseeker.max_keys == -1
+          while redis.keys("#{chain_key_prefix}:*").size > Meeseeker.max_keys
+            sleep Meeseeker::BLOCK_INTERVAL
+          end
+        end
+        
         redis.set(key, op.to_json)
         redis.expire(key, Meeseeker.expire_keys) unless Meeseeker.expire_keys == -1
         
@@ -56,7 +66,7 @@ module Meeseeker
               block_api.get_block_header(block_num: block_num) do |result|
                 if result.nil? || result.header.nil?
                   puts "Node returned empty result for block_header on block_num: #{block_num} (rate limiting?).  Retrying ..."
-                  sleep 3
+                  sleep Meeseeker::BLOCK_INTERVAL
                   throw :block_header
                 end
                 
@@ -65,19 +75,19 @@ module Meeseeker
             end
           end
           
-          redis.set(LAST_BLOCK_NUM_KEY, block_num)
-          redis.publish('steem:block', block_payload.to_json)
+          redis.set(chain_key_prefix + LAST_BLOCK_NUM_KEY_SUFFIX, block_num)
+          redis.publish("#{chain_key_prefix}:block", block_payload.to_json)
           current_block_num = block_num
         end
         
-        redis.publish("steem:op:#{op_type}", {key: key}.to_json)
+        redis.publish("#{chain_key_prefix}:op:#{op_type}", {key: key}.to_json)
         
         if Meeseeker.publish_op_custom_id
           if %w(custom custom_binary custom_json).include? op_type
             id = (op["value"]["id"] rescue nil).to_s
             
             if id.size > 0
-              redis.publish("steem:op:#{op_type}:#{id}", {key: key}.to_json)
+              redis.publish("#{chain_key_prefix}:op:#{op_type}:#{id}", {key: key}.to_json)
             end
           end
         end
@@ -85,7 +95,10 @@ module Meeseeker
     end
   private
     def stream_operations(options = {}, &block)
+      chain = (options[:chain] || 'hive').to_sym
       redis = Meeseeker.redis
+      chain_key_prefix = chain.to_s if !!options[:chain]
+      chain_key_prefix ||= Meeseeker.chain_key_prefix
       last_block_num = nil
       mode = options.delete(:mode) || Meeseeker.stream_mode
       options[:include_virtual] ||= Meeseeker.include_virtual
@@ -93,8 +106,9 @@ module Meeseeker
       if !!options[:at_block_num]
         last_block_num = options[:at_block_num].to_i
       else
-        database_api = Steem::DatabaseApi.new(url: Meeseeker.node_url)
-        last_block_num = redis.get(LAST_BLOCK_NUM_KEY).to_i + 1
+        url = Meeseeker.default_url(chain)
+        database_api = Meeseeker.database_api_class(chain).new(url: url)
+        last_block_num = redis.get(chain_key_prefix + LAST_BLOCK_NUM_KEY_SUFFIX).to_i + 1
         
         block_num = catch :dynamic_global_properties do
           database_api.get_dynamic_global_properties do |dgpo|
@@ -125,11 +139,12 @@ module Meeseeker
       end
       
       begin
-        stream_options = {url: Meeseeker.node_url, mode: mode}
+        url = Meeseeker.default_url(chain)
+        stream_options = {url: url, mode: mode}
         options = options.merge(at_block_num: last_block_num)
         condenser_api = nil
         
-        Steem::Stream.new(stream_options).tap do |stream|
+        Meeseeker.stream_class.new(stream_options).tap do |stream|
           puts "Stream begin: #{stream_options.to_json}; #{options.to_json}"
           
           # Prior to v0.0.4, we only streamed operations with stream.operations.
@@ -165,7 +180,8 @@ module Meeseeker
                 
                 loop do
                   # TODO (HF23) Switch to account_history_api.enum_virtual_ops if supported.
-                  condenser_api ||= Steem::CondenserApi.new(url: Meeseeker.node_url)
+                  url = Meeseeker.default_url(chain)
+                  condenser_api ||= Meeseeker.condenser_api_class(chain).new(url: url)
                   condenser_api.get_ops_in_block(n, true) do |vops|
                     if vops.nil?
                       puts "Node returned empty result for get_ops_in_block on block_num: #{n} (rate limiting?).  Retrying ..."
@@ -181,7 +197,7 @@ module Meeseeker
                       if retries < MAX_VOP_RETRY
                         retries = retries + 1
                         condenser_api = nil
-                        sleep 3 * retries
+                        sleep Meeseeker::BLOCK_INTERVAL * retries
                         
                         redo
                       end
@@ -217,8 +233,8 @@ module Meeseeker
               # We need to tell steem-ruby to avoid json-rpc-batch on this
               # node.
               
-              Steem::BlockApi.const_set 'MAX_RANGE_SIZE', 1
-              sleep 3
+              Meeseeker.block_api_class(chain).const_set 'MAX_RANGE_SIZE', 1
+              sleep Meeseeker::BLOCK_INTERVAL
               redo
             end
           end

data/lib/meeseeker/hive_engine.rb

@@ -0,0 +1,20 @@
+module Meeseeker::HiveEngine
+  
+  class Agent < Meeseeker::SteemEngine::Agent
+    def initialize(options = {})
+      super
+      
+      self.user_agent = Meeseeker::AGENT_ID
+      self.max_history = 0
+      self.default_encoding = 'UTF-8'
+      
+      @node_url = options[:url] || Meeseeker::hive_engine_node_url
+    end
+  end
+  
+  class FollowerJob < Meeseeker::SteemEngine::FollowerJob
+    def initialize(options = {})
+      @chain_key_prefix = options[:chain_key_prefix] || Meeseeker::HIVE_ENGINE_CHAIN_KEY_PREFIX
+    end
+  end
+end

data/lib/meeseeker/steem_engine/agent.rb

@@ -26,30 +26,44 @@ module Meeseeker::SteemEngine
     end
 
     def latest_block_info
-      request_body = {
-        jsonrpc: "2.0",
-        method: :getLatestBlockInfo,
-        id: rpc_id
-      }.to_json
+      5.times do
+        request_body = {
+          jsonrpc: "2.0",
+          method: :getLatestBlockInfo,
+          id: rpc_id
+        }.to_json
+        
+        response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
+        latest_block_info = JSON[response.body]["result"]
+        
+        return latest_block_info if !!latest_block_info
+        
+        sleep 3
+      end
       
-      response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
-      
-      JSON[response.body]["result"]
+      return nil
     end
 
     def block(block_num)
-      request_body = {
-        jsonrpc: "2.0",
-        method: :getBlockInfo,
-        params: {
-          blockNumber: block_num
-        },
-        id: rpc_id
-      }.to_json
-      
-      response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
+      5.times do
+        request_body = {
+          jsonrpc: "2.0",
+          method: :getBlockInfo,
+          params: {
+            blockNumber: block_num.to_i
+          },
+          id: rpc_id
+        }.to_json
+        
+        response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
+        block = JSON[response.body]["result"]
+        
+        return block if !!block
+        
+        sleep 3
+      end
       
-      JSON[response.body]["result"]
+      return nil
     end
   private
     def rpc_id

data/lib/meeseeker/steem_engine/follower_job.rb

@@ -2,6 +2,14 @@ module Meeseeker::SteemEngine
   MAX_RETRY_INTERVAL = 18.0
   
   class FollowerJob
+    def initialize(options = {})
+      @chain_key_prefix = options[:chain_key_prefix] || Meeseeker::STEEM_ENGINE_CHAIN_KEY_PREFIX
+    end
+    
+    def chain_name
+      @chain_key_prefix.split('_').map(&:capitalize).join(' ')
+    end
+    
     def perform(options = {})
       redis = Meeseeker.redis
       last_key_prefix = nil
@@ -16,7 +24,7 @@ module Meeseeker::SteemEngine
         begin
           trx_id = transaction['transactionId'].to_s.split('-').first
           block_num = block['blockNumber']
-          current_key_prefix = "steem_engine:#{block_num}:#{trx_id}"
+          current_key_prefix = "#{@chain_key_prefix}:#{block_num}:#{trx_id}"
           contract = transaction['contract']
           action = transaction['action']
 
@@ -34,15 +42,15 @@ module Meeseeker::SteemEngine
               block_transactions << trx_id
               
               trx_pub_key = if !!virtual
-                'steem_engine:virtual_transaction'
+                "#{@chain_key_prefix}:virtual_transaction"
               else
-                'steem_engine:transaction'
+                "#{@chain_key_prefix}:transaction"
               end
               
               redis.publish(trx_pub_key, transaction_payload.to_json)
             end
             
-            last_key_prefix = "steem_engine:#{block_num}:#{trx_id}"
+            last_key_prefix = "#{@chain_key_prefix}:#{block_num}:#{trx_id}"
             trx_index = 0
           end
           
@@ -50,6 +58,12 @@ module Meeseeker::SteemEngine
           puts key
         end
 
+        unless Meeseeker.max_keys == -1
+          while redis.keys("#{@chain_key_prefix}:*").size > Meeseeker.max_keys
+            sleep Meeseeker::BLOCK_INTERVAL
+          end
+        end
+        
         redis.set(key, transaction.to_json)
         redis.expire(key, Meeseeker.expire_keys) unless Meeseeker.expire_keys == -1
         
@@ -59,18 +73,21 @@ module Meeseeker::SteemEngine
             block_num: block_num
           }
           
-          redis.set(Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY, block_num)
-          redis.publish('steem_engine:block', block_payload.to_json)
+          redis.set(@chain_key_prefix + Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY_SUFFIX, block_num)
+          redis.publish("#{@chain_key_prefix}:block", block_payload.to_json)
           current_block_num = block_num
         end
         
-        redis.publish("steem_engine:#{contract}", {key: key}.to_json)
-        redis.publish("steem_engine:#{contract}:#{action}", {key: key}.to_json)
+        redis.publish("#{@chain_key_prefix}:#{contract}", {key: key}.to_json)
+        redis.publish("#{@chain_key_prefix}:#{contract}:#{action}", {key: key}.to_json)
       end
     end
   private
     def agent
-      @agent ||= Agent.new
+      @agent ||= case @chain_key_prefix
+      when 'steem_engine' then Agent.new
+      when 'hive_engine' then Meeseeker::HiveEngine::Agent.new
+      end
     end
     
     def agent_reset
@@ -100,7 +117,7 @@ module Meeseeker::SteemEngine
         last_block_num = options[:at_block_num].to_i
       else
         new_sync = false
-        last_block_num = redis.get(Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY)
+        last_block_num = redis.get(@chain_key_prefix + Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY_SUFFIX)
         block_info = agent.latest_block_info
         block_num = block_info['blockNumber']
         last_block = agent.block(block_num)
@@ -116,13 +133,13 @@ module Meeseeker::SteemEngine
         if Meeseeker.expire_keys == -1
           last_block_num = [last_block_num, block_num].max
           
-          puts "Sync Steem Engine from: #{last_block_num}"
+          puts "Sync #{chain_name} from: #{last_block_num}"
         elsif new_sync || (Time.now.utc - last_block_timestamp > Meeseeker.expire_keys)
           last_block_num = block_num + 1
           
-          puts 'Starting new Steem Engine sync.'
+          puts "Starting new #{chain_name} sync."
         else
-          puts "Resuming from Steem Engine block #{last_block_num} ..."
+          puts "Resuming from #{chain_name} block #{last_block_num} ..."
         end
       end
       
@@ -140,7 +157,7 @@ module Meeseeker::SteemEngine
         end
         
         if block.nil?
-          sleep 3 # sleep for one mainnet block interval
+          sleep Meeseeker::BLOCK_INTERVAL
           redo
         end