meeseeker 0.0.6 → 2.0.0.pre.1

data/lib/meeseeker/hive_engine.rb

@@ -0,0 +1,20 @@
+module Meeseeker::HiveEngine
+  
+  class Agent < Meeseeker::SteemEngine::Agent
+    def initialize(options = {})
+      super
+      
+      self.user_agent = Meeseeker::AGENT_ID
+      self.max_history = 0
+      self.default_encoding = 'UTF-8'
+      
+      @node_url = options[:url] || Meeseeker::hive_engine_node_url
+    end
+  end
+  
+  class FollowerJob < Meeseeker::SteemEngine::FollowerJob
+    def initialize(options = {})
+      @chain_key_prefix = options[:chain_key_prefix] || Meeseeker::HIVE_ENGINE_CHAIN_KEY_PREFIX
+    end
+  end
+end

data/lib/meeseeker/steem_engine/agent.rb

@@ -26,30 +26,44 @@ module Meeseeker::SteemEngine
     end
 
     def latest_block_info
-      request_body = {
-        jsonrpc: "2.0",
-        method: :getLatestBlockInfo,
-        id: rpc_id
-      }.to_json
+      5.times do
+        request_body = {
+          jsonrpc: "2.0",
+          method: :getLatestBlockInfo,
+          id: rpc_id
+        }.to_json
+        
+        response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
+        latest_block_info = JSON[response.body]["result"]
+        
+        return latest_block_info if !!latest_block_info
+        
+        sleep 3
+      end
       
-      response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
-      
-      JSON[response.body]["result"]
+      return nil
     end
 
     def block(block_num)
-      request_body = {
-        jsonrpc: "2.0",
-        method: :getBlockInfo,
-        params: {
-          blockNumber: block_num
-        },
-        id: rpc_id
-      }.to_json
-      
-      response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
+      5.times do
+        request_body = {
+          jsonrpc: "2.0",
+          method: :getBlockInfo,
+          params: {
+            blockNumber: block_num.to_i
+          },
+          id: rpc_id
+        }.to_json
+        
+        response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
+        block = JSON[response.body]["result"]
+        
+        return block if !!block
+        
+        sleep 3
+      end
       
-      JSON[response.body]["result"]
+      return nil
     end
   private
     def rpc_id

data/lib/meeseeker/steem_engine/follower_job.rb

@@ -1,5 +1,15 @@
 module Meeseeker::SteemEngine
+  MAX_RETRY_INTERVAL = 18.0
+  
   class FollowerJob
+    def initialize(options = {})
+      @chain_key_prefix = options[:chain_key_prefix] || Meeseeker::STEEM_ENGINE_CHAIN_KEY_PREFIX
+    end
+    
+    def chain_name
+      @chain_key_prefix.split('_').map(&:capitalize).join(' ')
+    end
+    
     def perform(options = {})
       redis = Meeseeker.redis
       last_key_prefix = nil
@@ -7,11 +17,14 @@ module Meeseeker::SteemEngine
       current_block_num = nil
       block_transactions = []
       
-      stream_transactions(options) do |transaction, block|
+      stream_transactions(options) do |data, block|
+        transaction = data[:transaction]
+        virtual = !!data[:virtual]
+        
         begin
-          trx_id = transaction['transactionId'].split('-').first
+          trx_id = transaction['transactionId'].to_s.split('-').first
           block_num = block['blockNumber']
-          current_key_prefix = "steem_engine:#{block_num}:#{trx_id}"
+          current_key_prefix = "#{@chain_key_prefix}:#{block_num}:#{trx_id}"
           contract = transaction['contract']
           action = transaction['action']
 
@@ -19,7 +32,7 @@ module Meeseeker::SteemEngine
             trx_index += 1
           else
             if !!last_key_prefix
-              n, b, t = last_key_prefix.split(':')
+              _, b, t = last_key_prefix.split(':')
               transaction_payload = {
                 block_num: b.to_i,
                 transaction_id: t,
@@ -27,10 +40,17 @@ module Meeseeker::SteemEngine
               }
               
               block_transactions << trx_id
-              redis.publish('steem_engine:transaction', transaction_payload.to_json)
+              
+              trx_pub_key = if !!virtual
+                "#{@chain_key_prefix}:virtual_transaction"
+              else
+                "#{@chain_key_prefix}:transaction"
+              end
+              
+              redis.publish(trx_pub_key, transaction_payload.to_json)
             end
             
-            last_key_prefix = "steem_engine:#{block_num}:#{trx_id}"
+            last_key_prefix = "#{@chain_key_prefix}:#{block_num}:#{trx_id}"
             trx_index = 0
           end
           
@@ -38,6 +58,12 @@ module Meeseeker::SteemEngine
           puts key
         end
 
+        unless Meeseeker.max_keys == -1
+          while redis.keys("#{@chain_key_prefix}:*").size > Meeseeker.max_keys
+            sleep Meeseeker::BLOCK_INTERVAL
+          end
+        end
+        
         redis.set(key, transaction.to_json)
         redis.expire(key, Meeseeker.expire_keys) unless Meeseeker.expire_keys == -1
         
@@ -47,59 +73,114 @@ module Meeseeker::SteemEngine
             block_num: block_num
           }
           
-          redis.set(Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY, block_num)
-          redis.publish('steem_engine:block', block_payload.to_json)
+          redis.set(@chain_key_prefix + Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY_SUFFIX, block_num)
+          redis.publish("#{@chain_key_prefix}:block", block_payload.to_json)
           current_block_num = block_num
         end
         
-        redis.publish("steem_engine:#{contract}:#{action}", {key: key}.to_json)
+        redis.publish("#{@chain_key_prefix}:#{contract}", {key: key}.to_json)
+        redis.publish("#{@chain_key_prefix}:#{contract}:#{action}", {key: key}.to_json)
       end
     end
   private
+    def agent
+      @agent ||= case @chain_key_prefix
+      when 'steem_engine' then Agent.new
+      when 'hive_engine' then Meeseeker::HiveEngine::Agent.new
+      end
+    end
+    
+    def agent_reset
+      return if @agent.nil?
+      
+      @agent.shutdown
+      @agent = nil
+    end
+    
+    def retry_interval
+      @retry_interval ||= 0.1
+      @retry_interval *= 2
+      
+      [@retry_interval, MAX_RETRY_INTERVAL].min
+    end
+    
+    def reset_retry_interval
+      @retry_interval = nil
+    end
+    
     def stream_transactions(options = {}, &block)
       redis = Meeseeker.redis
       last_block_num = nil
-      agent = Agent.new
+      until_block_num = options[:until_block_num].to_i
       
       if !!options[:at_block_num]
         last_block_num = options[:at_block_num].to_i
       else
-        last_block_num = redis.get(Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY).to_i + 1
+        new_sync = false
+        last_block_num = redis.get(@chain_key_prefix + Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY_SUFFIX)
         block_info = agent.latest_block_info
         block_num = block_info['blockNumber']
-        
-        last_block = agent.block(last_block_num)
+        last_block = agent.block(block_num)
         last_block_timestamp = Time.parse(last_block['timestamp'] + 'Z')
+                
+        if last_block_num.nil?
+          new_sync = true
+          last_block_num = block_num
+        else
+          last_block_num = last_block_num.to_i + 1
+        end
         
         if Meeseeker.expire_keys == -1
           last_block_num = [last_block_num, block_num].max
           
-          puts "Sync Steem Engine from: #{last_block_num}"
-        elsif Time.now.utc - last_block_timestamp > Meeseeker.expire_keys
+          puts "Sync #{chain_name} from: #{last_block_num}"
+        elsif new_sync || (Time.now.utc - last_block_timestamp > Meeseeker.expire_keys)
           last_block_num = block_num + 1
           
-          puts 'Starting new Steem Engine sync.'
+          puts "Starting new #{chain_name} sync."
         else
-          puts "Resuming from Steem Engine block #{last_block_num} ..."
+          puts "Resuming from #{chain_name} block #{last_block_num} ..."
         end
       end
       
       block_num = last_block_num
       
       loop do
-        block = agent.block(block_num)
+        begin
+          block = agent.block(block_num)
+          reset_retry_interval
+        rescue Net::HTTP::Persistent::Error => e
+          puts "Retrying: #{e}"
+          agent_reset
+          sleep retry_interval
+          redo
+        end
         
         if block.nil?
-          sleep 3 # sleep for one mainnet block interval
+          sleep Meeseeker::BLOCK_INTERVAL
           redo
         end
         
         transactions = block['transactions']
         
         transactions.each do |transaction|
-          yield transaction.merge(timestamp: block['timestamp']), block
+          yield({transaction: transaction.merge(timestamp: block['timestamp'])}, block)
+        end
+        
+        virtual_transactions = block['virtualTransactions']
+        
+        virtual_transactions.each do |virtual_transaction|
+          _, vtrx_in_block = virtual_transaction['transactionId'].split('-')
+          virtual_transaction = virtual_transaction.merge(
+            timestamp: block['timestamp'],
+            'transactionId' => "#{Meeseeker::VIRTUAL_TRX_ID}-#{vtrx_in_block}"
+          )
+          
+          yield({transaction: virtual_transaction, virtual: true}, block)
         end
         
+        break if until_block_num != 0 && block_num > until_block_num
+        
         block_num = block_num + 1
       end
     end

data/lib/meeseeker/version.rb

@@ -1,4 +1,4 @@
 module Meeseeker
-  VERSION = '0.0.6'
+  VERSION = '2.0.0-1'
   AGENT_ID = "meeseeker/#{VERSION}"
 end

data/lib/meeseeker/witness_schedule_job.rb

@@ -1,7 +1,11 @@
 module Meeseeker
   class WitnessScheduleJob
     def perform(options = {})
-      database_api = Steem::DatabaseApi.new(url: Meeseeker.node_url)
+      chain = (options[:chain] || 'hive').to_sym
+      chain_key_prefix = chain.to_s if !!options[:chain]
+      chain_key_prefix ||= Meeseeker.default_chain_key_prefix
+      url = Meeseeker.default_url(chain_key_prefix)
+      database_api = Meeseeker.database_api_class(chain_key_prefix).new(url: url)
       redis = Meeseeker.redis
       mode = options.delete(:mode) || Meeseeker.stream_mode
       schedule = nil
@@ -53,7 +57,7 @@ module Meeseeker
         
         if next_shuffle_block_num != last_shuffle_block_num
           puts "next_shuffle_block_num: #{next_shuffle_block_num}; current_shuffled_witnesses: #{schedule.current_shuffled_witnesses.join(', ')}"
-          redis.publish('steem:witness:schedule', schedule.to_json)
+          redis.publish("#{chain_key_prefix}:witness:schedule", schedule.to_json)
           last_shuffle_block_num = next_shuffle_block_num
         end
         

data/meeseeker.gemspec

@@ -27,8 +27,11 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'simplecov', '~> 0.15', '>= 0.15.1'
   s.add_development_dependency 'pry', '~> 0.11', '>= 0.11.3'
   s.add_development_dependency 'irb', '~> 1.0', '>= 1.0.0'
+  s.add_development_dependency 'mock_redis', '~> 0.22', '>= 0.22.0'
 
   s.add_dependency 'redis', '~> 4.1', '>= 4.1.0'
-  s.add_dependency 'steem-mechanize', '~> 0.0', '>= 0.0.5'
+  s.add_dependency 'steem-ruby', '~> 0.9', '>= 0.9.4'
+  s.add_dependency 'hive-ruby', '~> 1.0.0.pre-1', '>= 1.0.0.pre-1'
+  s.add_dependency 'mechanize', '~> 2.7', '>= 2.7.6'
   s.add_dependency 'rb-readline', '~> 0.5', '>= 0.5.5'
 end

data/test/meeseeker/meeseeker_test.rb

@@ -4,28 +4,265 @@ require 'rake'
 module Meeseeker
   class MeeseekerTest < Meeseeker::Test
     def setup
-      gem_dir = File.expand_path("..", File.dirname(__FILE__))
-      $LOAD_PATH.unshift gem_dir
-
-      pwd = Dir.pwd
-      Dir.chdir(gem_dir)
-      Rake.application.init
-      Rake.application.load_rakefile
-      Dir.chdir(pwd)
+      @max_blocks = 30 # must be at least 15 to get past irreversible
+    end
+    
+    def test_verify_hive_jobs
+      chain = 'hive'
+      check_keys(chain)
+      keys = []
+      
+      begin
+        Rake::Task['verify:block_org'].reenable
+        assert Rake::Task['verify:block_org'].invoke('hive', @max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+        
+      begin
+        Rake::Task['verify:witness:schedule'].reenable
+        assert Rake::Task['verify:witness:schedule'].invoke('hive', @max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+        
+      block_api = Hive::BlockApi.new(url: 'http://anyx.io')
+      keys = Meeseeker.redis.keys('hive:*')
+      data = keys.map do |key|
+        next if key == 'hive:meeseeker:last_block_num'
+        
+        n, b, t, i, o = key.split(':')
+        
+        assert_equal 'hive', n, "expected hive key, got: #{key}"
+        
+        [b, t]
+      end.compact.sample(10).to_h
       
-      if !!Meeseeker.redis.get(Meeseeker::LAST_BLOCK_NUM_KEY)
-        fail "Found existing keys.  Please use 'rake reset' to enable this test."
+      assert data.any?, 'expect hive data'
+      
+      data.each do |b, t|
+        block_api.get_block(block_num: b) do |result|
+          block = result.block
+          
+          refute_nil block, "did not expect nil block (#{b})"
+          
+          if !!block.transaction_ids
+            assert block.transaction_ids.include?(t), "Could not find hive trx_id (#{t}) in block (#{b})."
+          else
+            puts "Skipped check for hive trx_id (#{t}) in block (#{b}) because API does not support lookup."
+          end
+        end
+      end
+      
+      if keys.any?
+        dropped = Meeseeker.redis.del(*keys)
+        puts "Dropped #{dropped} keys."
+      else
+        fail 'No keys.'
       end
     end
     
-    def test_verify_all_jobs
-      max_blocks = 30 # must be at least 15 to get past irreversible
+    def test_verify_steem_jobs
+      chain = 'steem'
+      check_keys(chain)
+      keys = []
+      
+      begin
+        Rake::Task['verify:block_org'].reenable
+        assert Rake::Task['verify:block_org'].invoke('steem', @max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+      
+      begin
+        Rake::Task['verify:witness:schedule'].reenable
+        assert Rake::Task['verify:witness:schedule'].invoke('steem', @max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+      
+      block_api = Steem::BlockApi.new
+      keys = Meeseeker.redis.keys('steem:*')
+      data = keys.map do |key|
+        next if key == 'steem:meeseeker:last_block_num'
+        
+        n, b, t, i, o = key.split(':')
+        
+        assert_equal 'steem', n, "expected steem key, got: #{key}"
+        
+        [b, t]
+      end.compact.sample(10).to_h
+      
+      assert data.any?, 'expect steem data'
+      
+      data.each do |b, t|
+        block_api.get_block(block_num: b) do |result|
+          block = result.block
+          
+          refute_nil block, "did not expect nil block (#{b})"
+          
+          if !!block.transaction_ids
+            assert block.transaction_ids.include?(t), "Could not find steem trx_id (#{t}) in block (#{b})."
+          else
+            puts "Skipped check for steem trx_id (#{t}) in block (#{b}) because API does not support lookup."
+          end
+        end
+      end
+      
+      if keys.any?
+        dropped = Meeseeker.redis.del(*keys)
+        puts "Dropped #{dropped} keys."
+      else
+        fail 'No keys.'
+      end
+    end
     
-      assert Rake::Task['verify:block_org'].invoke(max_blocks)
-      assert Rake::Task['verify:steem_engine_block_org'].invoke(max_blocks)
-      assert Rake::Task['verify:witness:schedule'].invoke(max_blocks)
+    def test_verify_steem_engine_jobs
+      chain = 'steem_engine' 
+      check_keys(chain)
+      keys = []
+      
+      begin
+        Rake::Task['verify:steem_engine_block_org'].reenable
+        Rake::Task['verify:engine_block_org'].reenable
+        assert Rake::Task['verify:steem_engine_block_org'].invoke(@max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+      
+      begin
+        Rake::Task['verify:steem_engine_ref_blocks'].reenable
+        Rake::Task['verify:engine_ref_blocks'].reenable
+        assert Rake::Task['verify:steem_engine_ref_blocks'].invoke(@max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
       
-      Rake::Task['reset'].invoke
+      agent = Meeseeker::SteemEngine::Agent.new
+      keys = Meeseeker.redis.keys('steem_engine:*')
+      data = keys.map do |key|
+        n, b, t, i, o = key.split(':')
+        
+        assert_equal chain, n, "expected steem_engine key, got: #{key}"
+        
+        next if t == Meeseeker::VIRTUAL_TRX_ID
+        
+        [b, t]
+      end.compact.sample(10).to_h
+      
+      assert data.any?, 'expect steem_engine data'
+      
+      data.each do |b, t|
+        block = agent.block(b)
+        refute_nil block, "did not expect nil block (#{b})"
+        
+        count = block['transactions'].select do |trx|
+          trx['transactionId'].include? t
+        end.size
+        
+        assert count > 0, "Could not find steem_engine trx_id (#{t}) in block (#{b})."
+      end
+      
+      agent.shutdown
+      
+      if keys.any?
+        dropped = Meeseeker.redis.del(*keys)
+        puts "Dropped #{dropped} keys."
+      else
+        fail 'No keys.'
+      end
+    end
+    
+    def test_verify_hive_engine_jobs
+      chain = 'hive_engine'
+      check_keys(chain)
+      keys = []
+      
+      begin
+        Rake::Task['verify:hive_engine_block_org'].reenable
+        Rake::Task['verify:engine_block_org'].reenable
+        assert Rake::Task['verify:hive_engine_block_org'].invoke(@max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+      
+      begin
+        Rake::Task['verify:hive_engine_ref_blocks'].reenable
+        Rake::Task['verify:engine_ref_blocks'].reenable
+        assert Rake::Task['verify:hive_engine_ref_blocks'].invoke(@max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+      
+      agent = Meeseeker::HiveEngine::Agent.new
+      keys = Meeseeker.redis.keys('hive_engine:*')
+      data = keys.map do |key|
+        n, b, t, i, o = key.split(':')
+        
+        assert_equal chain, n, "expected hive_engine key, got: #{key}"
+        
+        next if t == Meeseeker::VIRTUAL_TRX_ID
+        
+        [b, t]
+      end.compact.sample(10).to_h
+      
+      assert data.any?, 'expect hive_engine data'
+      
+      data.each do |b, t|
+        block = agent.block(b)
+        refute_nil block, "did not expect nil block (#{b})"
+        
+        count = block['transactions'].select do |trx|
+          trx['transactionId'].include? t
+        end.size
+        
+        assert count > 0, "Could not find hive_engine trx_id (#{t}) in block (#{b})."
+      end
+      
+      agent.shutdown
+      
+      if keys.any?
+        dropped = Meeseeker.redis.del(*keys)
+        puts "Dropped #{dropped} keys."
+      else
+        fail 'No keys.'
+      end
+    end
+  private
+    def check_keys(chain)
+      chain = chain_key_prefix = chain.to_s
+      Meeseeker.node_url = case chain.to_sym
+      when :hive_engine then Meeseeker.shuffle_node_url('hive')
+      when :steem_engine then Meeseeker.shuffle_node_url('steem')
+      else
+        Meeseeker.shuffle_node_url(chain.to_s)
+      end
+      
+      begin
+        if !!Meeseeker.redis.get(chain_key_prefix + Meeseeker::LAST_BLOCK_NUM_KEY_SUFFIX)
+          fail "Found existing keys.  Please use 'rake reset' to enable this test."
+        end
+      rescue Redis::CannotConnectError => e
+        warn "Cannot connect to redis, using MockRedis instead."
+        
+        Meeseeker.redis = MockRedis.new  
+      end
     end
   end
 end