meeseeker 0.0.8 → 2.0.1.pre.1

data/lib/meeseeker/hive_engine.rb

@@ -0,0 +1,20 @@
+module Meeseeker::HiveEngine
+  
+  class Agent < Meeseeker::SteemEngine::Agent
+    def initialize(options = {})
+      super
+      
+      self.user_agent = Meeseeker::AGENT_ID
+      self.max_history = 0
+      self.default_encoding = 'UTF-8'
+      
+      @node_url = options[:url] || Meeseeker::hive_engine_node_url
+    end
+  end
+  
+  class FollowerJob < Meeseeker::SteemEngine::FollowerJob
+    def initialize(options = {})
+      @chain_key_prefix = options[:chain_key_prefix] || Meeseeker::HIVE_ENGINE_CHAIN_KEY_PREFIX
+    end
+  end
+end

data/lib/meeseeker/steem_engine/agent.rb

@@ -26,30 +26,44 @@ module Meeseeker::SteemEngine
     end
 
     def latest_block_info
-      request_body = {
-        jsonrpc: "2.0",
-        method: :getLatestBlockInfo,
-        id: rpc_id
-      }.to_json
+      5.times do
+        request_body = {
+          jsonrpc: "2.0",
+          method: :getLatestBlockInfo,
+          id: rpc_id
+        }.to_json
+        
+        response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
+        latest_block_info = JSON[response.body]["result"]
+        
+        return latest_block_info if !!latest_block_info
+        
+        sleep 3
+      end
       
-      response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
-      
-      JSON[response.body]["result"]
+      return nil
     end
 
     def block(block_num)
-      request_body = {
-        jsonrpc: "2.0",
-        method: :getBlockInfo,
-        params: {
-          blockNumber: block_num
-        },
-        id: rpc_id
-      }.to_json
-      
-      response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
+      5.times do
+        request_body = {
+          jsonrpc: "2.0",
+          method: :getBlockInfo,
+          params: {
+            blockNumber: block_num.to_i
+          },
+          id: rpc_id
+        }.to_json
+        
+        response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
+        block = JSON[response.body]["result"]
+        
+        return block if !!block
+        
+        sleep 3
+      end
       
-      JSON[response.body]["result"]
+      return nil
     end
   private
     def rpc_id

data/lib/meeseeker/steem_engine/follower_job.rb

@@ -2,6 +2,14 @@ module Meeseeker::SteemEngine
   MAX_RETRY_INTERVAL = 18.0
   
   class FollowerJob
+    def initialize(options = {})
+      @chain_key_prefix = options[:chain_key_prefix] || Meeseeker::STEEM_ENGINE_CHAIN_KEY_PREFIX
+    end
+    
+    def chain_name
+      @chain_key_prefix.split('_').map(&:capitalize).join(' ')
+    end
+    
     def perform(options = {})
       redis = Meeseeker.redis
       last_key_prefix = nil
@@ -16,7 +24,7 @@ module Meeseeker::SteemEngine
         begin
           trx_id = transaction['transactionId'].to_s.split('-').first
           block_num = block['blockNumber']
-          current_key_prefix = "steem_engine:#{block_num}:#{trx_id}"
+          current_key_prefix = "#{@chain_key_prefix}:#{block_num}:#{trx_id}"
           contract = transaction['contract']
           action = transaction['action']
 
@@ -34,15 +42,15 @@ module Meeseeker::SteemEngine
               block_transactions << trx_id
               
               trx_pub_key = if !!virtual
-                'steem_engine:virtual_transaction'
+                "#{@chain_key_prefix}:virtual_transaction"
               else
-                'steem_engine:transaction'
+                "#{@chain_key_prefix}:transaction"
               end
               
               redis.publish(trx_pub_key, transaction_payload.to_json)
             end
             
-            last_key_prefix = "steem_engine:#{block_num}:#{trx_id}"
+            last_key_prefix = "#{@chain_key_prefix}:#{block_num}:#{trx_id}"
             trx_index = 0
           end
           
@@ -50,6 +58,12 @@ module Meeseeker::SteemEngine
           puts key
         end
 
+        unless Meeseeker.max_keys == -1
+          while redis.keys("#{@chain_key_prefix}:*").size > Meeseeker.max_keys
+            sleep Meeseeker::BLOCK_INTERVAL
+          end
+        end
+        
         redis.set(key, transaction.to_json)
         redis.expire(key, Meeseeker.expire_keys) unless Meeseeker.expire_keys == -1
         
@@ -59,18 +73,21 @@ module Meeseeker::SteemEngine
             block_num: block_num
           }
           
-          redis.set(Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY, block_num)
-          redis.publish('steem_engine:block', block_payload.to_json)
+          redis.set(@chain_key_prefix + Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY_SUFFIX, block_num)
+          redis.publish("#{@chain_key_prefix}:block", block_payload.to_json)
           current_block_num = block_num
         end
         
-        redis.publish("steem_engine:#{contract}", {key: key}.to_json)
-        redis.publish("steem_engine:#{contract}:#{action}", {key: key}.to_json)
+        redis.publish("#{@chain_key_prefix}:#{contract}", {key: key}.to_json)
+        redis.publish("#{@chain_key_prefix}:#{contract}:#{action}", {key: key}.to_json)
       end
     end
   private
     def agent
-      @agent ||= Agent.new
+      @agent ||= case @chain_key_prefix
+      when 'steem_engine' then Agent.new
+      when 'hive_engine' then Meeseeker::HiveEngine::Agent.new
+      end
     end
     
     def agent_reset
@@ -100,7 +117,7 @@ module Meeseeker::SteemEngine
         last_block_num = options[:at_block_num].to_i
       else
         new_sync = false
-        last_block_num = redis.get(Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY)
+        last_block_num = redis.get(@chain_key_prefix + Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY_SUFFIX)
         block_info = agent.latest_block_info
         block_num = block_info['blockNumber']
         last_block = agent.block(block_num)
@@ -116,13 +133,13 @@ module Meeseeker::SteemEngine
         if Meeseeker.expire_keys == -1
           last_block_num = [last_block_num, block_num].max
           
-          puts "Sync Steem Engine from: #{last_block_num}"
+          puts "Sync #{chain_name} from: #{last_block_num}"
         elsif new_sync || (Time.now.utc - last_block_timestamp > Meeseeker.expire_keys)
           last_block_num = block_num + 1
           
-          puts 'Starting new Steem Engine sync.'
+          puts "Starting new #{chain_name} sync."
         else
-          puts "Resuming from Steem Engine block #{last_block_num} ..."
+          puts "Resuming from #{chain_name} block #{last_block_num} ..."
         end
       end
       
@@ -140,7 +157,7 @@ module Meeseeker::SteemEngine
         end
         
         if block.nil?
-          sleep 3 # sleep for one mainnet block interval
+          sleep Meeseeker::BLOCK_INTERVAL
           redo
         end
         

data/lib/meeseeker/version.rb

@@ -1,4 +1,4 @@
 module Meeseeker
-  VERSION = '0.0.8'
+  VERSION = '2.0.1.pre.1'
   AGENT_ID = "meeseeker/#{VERSION}"
 end

data/lib/meeseeker/witness_schedule_job.rb

@@ -1,7 +1,11 @@
 module Meeseeker
   class WitnessScheduleJob
     def perform(options = {})
-      database_api = Steem::DatabaseApi.new(url: Meeseeker.node_url)
+      chain = (options[:chain] || 'hive').to_sym
+      chain_key_prefix = chain.to_s if !!options[:chain]
+      chain_key_prefix ||= Meeseeker.default_chain_key_prefix
+      url = Meeseeker.default_url(chain_key_prefix)
+      database_api = Meeseeker.database_api_class(chain_key_prefix).new(url: url)
       redis = Meeseeker.redis
       mode = options.delete(:mode) || Meeseeker.stream_mode
       schedule = nil
@@ -53,7 +57,7 @@ module Meeseeker
         
         if next_shuffle_block_num != last_shuffle_block_num
           puts "next_shuffle_block_num: #{next_shuffle_block_num}; current_shuffled_witnesses: #{schedule.current_shuffled_witnesses.join(', ')}"
-          redis.publish('steem:witness:schedule', schedule.to_json)
+          redis.publish("#{chain_key_prefix}:witness:schedule", schedule.to_json)
           last_shuffle_block_num = next_shuffle_block_num
         end
         

data/meeseeker.gemspec

@@ -27,8 +27,10 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'simplecov', '~> 0.15', '>= 0.15.1'
   s.add_development_dependency 'pry', '~> 0.11', '>= 0.11.3'
   s.add_development_dependency 'irb', '~> 1.0', '>= 1.0.0'
+  s.add_development_dependency 'mock_redis', '~> 0.22', '>= 0.22.0'
 
   s.add_dependency 'redis', '~> 4.1', '>= 4.1.0'
-  s.add_dependency 'steem-mechanize', '~> 0.0', '>= 0.0.5'
+  s.add_dependency 'radiator', '~> 0.4', '>= 0.4.8'
+  s.add_dependency 'mechanize', '~> 2.7', '>= 2.7.6'
   s.add_dependency 'rb-readline', '~> 0.5', '>= 0.5.5'
 end

data/test/meeseeker/meeseeker_test.rb

@@ -4,28 +4,273 @@ require 'rake'
 module Meeseeker
   class MeeseekerTest < Meeseeker::Test
     def setup
-      gem_dir = File.expand_path("..", File.dirname(__FILE__))
-      $LOAD_PATH.unshift gem_dir
-
-      pwd = Dir.pwd
-      Dir.chdir(gem_dir)
-      Rake.application.init
-      Rake.application.load_rakefile
-      Dir.chdir(pwd)
+      @max_blocks = 30 # must be at least 15 to get past irreversible
+    end
+    
+    def test_verify_hive_jobs
+      chain = 'hive'
+      check_keys(chain)
+      keys = []
+      
+      begin
+        Rake::Task['verify:block_org'].reenable
+        assert Rake::Task['verify:block_org'].invoke('hive', @max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+        
+      begin
+        Rake::Task['verify:witness:schedule'].reenable
+        assert Rake::Task['verify:witness:schedule'].invoke('hive', @max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+        
+      block_api = Hive::BlockApi.new(url: 'http://anyx.io')
+      keys = Meeseeker.redis.keys('hive:*')
+      data = keys.map do |key|
+        next if key == 'hive:meeseeker:last_block_num'
+        
+        n, b, t, i, o = key.split(':')
+        
+        assert_equal 'hive', n, "expected hive key, got: #{key}"
+        
+        next if t == Meeseeker::VIRTUAL_TRX_ID
+        
+        [b, t]
+      end.compact.sample(10).to_h
       
-      if !!Meeseeker.redis.get(Meeseeker::LAST_BLOCK_NUM_KEY)
-        fail "Found existing keys.  Please use 'rake reset' to enable this test."
+      assert data.any?, 'expect hive data'
+      
+      data.each do |b, t|
+        block_api.get_block(block_num: b) do |result|
+          block = result.block
+          
+          refute_nil block, "did not expect nil block (#{b})"
+          
+          if !!block.transaction_ids
+            assert block.transaction_ids.include?(t), "Could not find hive trx_id (#{t}) in block (#{b})."
+          else
+            puts "Skipped check for hive trx_id (#{t}) in block (#{b}) because API does not support lookup."
+          end
+        end
+      end
+      
+      if keys.any?
+        dropped = Meeseeker.redis.del(*keys)
+        puts "Dropped #{dropped} keys."
+      else
+        fail 'No keys.'
       end
     end
     
-    def test_verify_all_jobs
-      max_blocks = 30 # must be at least 15 to get past irreversible
+    def test_verify_steem_jobs
+      chain = 'steem'
+      check_keys(chain)
+      keys = []
+      
+      begin
+        Rake::Task['verify:block_org'].reenable
+        assert Rake::Task['verify:block_org'].invoke('steem', @max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+      
+      begin
+        Rake::Task['verify:witness:schedule'].reenable
+        assert Rake::Task['verify:witness:schedule'].invoke('steem', @max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+      
+      block_api = Steem::BlockApi.new
+      keys = Meeseeker.redis.keys('steem:*')
+      data = keys.map do |key|
+        next if key == 'steem:meeseeker:last_block_num'
+        
+        n, b, t, i, o = key.split(':')
+        
+        assert_equal 'steem', n, "expected steem key, got: #{key}"
+        
+        next if t == Meeseeker::VIRTUAL_TRX_ID
+        
+        [b, t]
+      end.compact.sample(10).to_h
+      
+      assert data.any?, 'expect steem data'
+      
+      data.each do |b, t|
+        block_api.get_block(block_num: b) do |result|
+          block = result.block
+          
+          refute_nil block, "did not expect nil block (#{b})"
+          
+          if !!block.transaction_ids
+            assert block.transaction_ids.include?(t), "Could not find steem trx_id (#{t}) in block (#{b})."
+          else
+            puts "Skipped check for steem trx_id (#{t}) in block (#{b}) because API does not support lookup."
+          end
+        end
+      end
+      
+      if keys.any?
+        dropped = Meeseeker.redis.del(*keys)
+        puts "Dropped #{dropped} keys."
+      else
+        fail 'No keys.'
+      end
+    end
     
-      assert Rake::Task['verify:block_org'].invoke(max_blocks)
-      assert Rake::Task['verify:steem_engine_block_org'].invoke(max_blocks)
-      assert Rake::Task['verify:witness:schedule'].invoke(max_blocks)
+    def test_verify_steem_engine_jobs
+      chain = 'steem_engine' 
+      check_keys(chain)
+      keys = []
+      
+      begin
+        Rake::Task['verify:steem_engine_block_org'].reenable
+        Rake::Task['verify:engine_block_org'].reenable
+        assert Rake::Task['verify:steem_engine_block_org'].invoke(@max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+      
+      begin
+        Rake::Task['verify:steem_engine_ref_blocks'].reenable
+        Rake::Task['verify:engine_ref_blocks'].reenable
+        assert Rake::Task['verify:steem_engine_ref_blocks'].invoke(@max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
       
-      Rake::Task['reset'].invoke
+      agent = Meeseeker::SteemEngine::Agent.new
+      keys = Meeseeker.redis.keys('steem_engine:*')
+      data = keys.map do |key|
+        next if key == 'steem:meeseeker:last_block_num'
+        
+        n, b, t, i, o = key.split(':')
+        
+        assert_equal chain, n, "expected steem_engine key, got: #{key}"
+        
+        next if t == Meeseeker::VIRTUAL_TRX_ID
+        
+        [b, t]
+      end.compact.sample(10).to_h
+      
+      assert data.any?, 'expect steem_engine data'
+      
+      data.each do |b, t|
+        next if t == 'last_block_num'
+        
+        block = agent.block(b)
+        refute_nil block, "did not expect nil block (#{b})"
+        
+        count = block['transactions'].select do |trx|
+          trx['transactionId'].to_s.include? t
+        end.size
+        
+        assert count > 0, "Could not find steem_engine trx_id (#{t}) in block (#{b})."
+      end
+      
+      agent.shutdown
+      
+      if keys.any?
+        dropped = Meeseeker.redis.del(*keys)
+        puts "Dropped #{dropped} keys."
+      else
+        fail 'No keys.'
+      end
+    end
+    
+    def test_verify_hive_engine_jobs
+      chain = 'hive_engine'
+      check_keys(chain)
+      keys = []
+      
+      begin
+        Rake::Task['verify:hive_engine_block_org'].reenable
+        Rake::Task['verify:engine_block_org'].reenable
+        assert Rake::Task['verify:hive_engine_block_org'].invoke(@max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+      
+      begin
+        Rake::Task['verify:hive_engine_ref_blocks'].reenable
+        Rake::Task['verify:engine_ref_blocks'].reenable
+        assert Rake::Task['verify:hive_engine_ref_blocks'].invoke(@max_blocks)
+      rescue SystemExit => e
+        puts 'Exited.'
+      rescue Redis::TimeoutError => e
+        skip 'Timed out.'
+      end
+      
+      agent = Meeseeker::HiveEngine::Agent.new
+      keys = Meeseeker.redis.keys('hive_engine:*')
+      data = keys.map do |key|
+        n, b, t, i, o = key.split(':')
+        
+        assert_equal chain, n, "expected hive_engine key, got: #{key}"
+        
+        next if t == Meeseeker::VIRTUAL_TRX_ID
+        
+        [b, t]
+      end.compact.sample(10).to_h
+      
+      assert data.any?, 'expect hive_engine data'
+      
+      data.each do |b, t|
+        block = agent.block(b)
+        refute_nil block, "did not expect nil block (#{b})"
+        
+        count = block['transactions'].select do |trx|
+          trx['transactionId'].include? t
+        end.size
+        
+        assert count > 0, "Could not find hive_engine trx_id (#{t}) in block (#{b})."
+      end
+      
+      agent.shutdown
+      
+      if keys.any?
+        dropped = Meeseeker.redis.del(*keys)
+        puts "Dropped #{dropped} keys."
+      else
+        fail 'No keys.'
+      end
+    end
+  private
+    def check_keys(chain)
+      chain = chain_key_prefix = chain.to_s
+      Meeseeker.node_url = case chain.to_sym
+      when :hive_engine then Meeseeker.shuffle_node_url('hive')
+      when :steem_engine then Meeseeker.shuffle_node_url('steem')
+      else
+        Meeseeker.shuffle_node_url(chain.to_s)
+      end
+      
+      begin
+        if !!Meeseeker.redis.get(chain_key_prefix + Meeseeker::LAST_BLOCK_NUM_KEY_SUFFIX)
+          fail "Found existing keys.  Please use 'rake reset' to enable this test."
+        end
+      rescue Redis::CannotConnectError => e
+        warn "Cannot connect to redis, using MockRedis instead."
+        
+        Meeseeker.redis = MockRedis.new  
+      end
     end
   end
 end