meeseeker 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee0984fc73172da1b1186ecaf6efee7301857ef06daa1b140f16cc732e93e97f
-  data.tar.gz: 60c2cad0dc9ddd541323dca737f1a6c017f0dc4b458075c5d2512b960133c863
+  metadata.gz: ad89ce6d8e55cafd5dcca2f0ddc2455c23ece9dab83c2d5e1a3f0a59df6c6a4a
+  data.tar.gz: e709c970eddc2c22c92d6f9482757305b851efc9603ccc725892e1feeb5a966d
 SHA512:
-  metadata.gz: d031ecca3a20d7b4c484baa461bbe35a2405c240ee4a2cb0ba45d6342bbba91a884cc55b347fd7db4fcdd987734e6d163bbf29be302127d5cf9674215333b592
-  data.tar.gz: f4824210cb157c65a80fdb217f9ec30dc55e09dde132ebe62db3896a08f7b50d2b21978ce9721d7b60312fd3c629abb7f2755f235b2f239281039b394578b35e
+  metadata.gz: 2a443c55716a8b6c605cb8cd5e72aee8008ba12f5641df0d1dd717e7db2535b4df9895b3d011ca07ab4c80df8164806243ff41eb82de68aa1fdfd0ce76962356
+  data.tar.gz: 53d277d9d1a8c848691bee1c4ecf6406ca61987a3b5010c657bad7485b87a25fd579fef8d7fc06108c2e06b91b50b97ff4b94e70414b2d2afefb1e2c7390b5dd

data/README.md

@@ -288,13 +288,52 @@ Or to get all ops for a particular transaction:
 redis-cli --scan --pattern 'steem:*:31ecb9c85e9eabd7ca2460fdb4f3ce4a7ca6ec32:*'
 ```
 
----
+### Steem Engine Support
 
-<center>
-  <img src="https://i.imgur.com/Y3Sa2GW.jpg" />
-</center>
+As of `v0.0.6`, meeseeker can also follow the Steem Engine side-chain.  This is optional and requires a separate process.
 
-See some of my previous Ruby How To posts in: [#radiator](https://steemit.com/created/radiator) [#ruby](https://steemit.com/created/ruby)
+To sync Steem Engine to your local redis source (also defaults to `redis://127.0.0.1:6379/0`):
+
+```bash
+meeseeker sync steem_engine
+```
+
+When running `meeseeker sync steem_engine`, the following channels are available:
+
+* `steem_engine:block`
+* `steem_engine:transaction`
+* `steem_engine:market:buy`
+* `steem_engine:market:cancel`
+* `steem_engine:market:sell`
+* `steem_engine:sscstore:buy`
+* `steem_engine:steempegged:buy`
+* `steem_engine:steempegged:removeWithdrawal`
+* `steem_engine:steempegged:withdraw`
+* `steem_engine:tokens:create`
+* `steem_engine:tokens:issue`
+* `steem_engine:tokens:transfer`
+* `steem_engine:tokens:updateMetadata`
+* `steem_engine:tokens:updateUrl`
+
+The above "channel/action" patterns are the ones that are known that the time of writing.  In addition, if a new contract is added or updated, meeseeker will automatically publish to these corresponding channels as they appear, without needing to update or even restart meeseeker.
+
+See main section on [Using `SUBSCRIBE`](#using-subscribe).
+
+Once your SteemEngine sync has started, you can begin doing queries against redis, for example, in the `redis-cli`:
+
+```bash
+redis-cli --scan --pattern 'steem_engine:*:tokens:transfer'
+```
+
+This returns the keys, for example:
+
+```
+steem_engine:18000:d414373db84e6a642f289641ea1433fda22b8a4d:0:tokens:transfer
+steem_engine:18004:c9e06c8449d2d04b4a0a31ec7b80d2f62009a5f0:0:tokens:transfer
+steem_engine:17994:faf097391760ad896b19d5854e2822f62dee284b:0:tokens:transfer
+```
+
+See main section on [Using `SCAN`](#using-scan).
 
 ### Docker
 
@@ -316,6 +355,14 @@ docker run \
 
 Also see: https://hub.docker.com/r/inertia/meeseeker/
 
+---
+
+<center>
+  <img src="https://i.imgur.com/Y3Sa2GW.jpg" />
+</center>
+
+See some of my previous Ruby How To posts in: [#radiator](https://steemit.com/created/radiator) [#ruby](https://steemit.com/created/ruby)
+
 ## Get in touch!
 
 If you're using Radiator, I'd love to hear from you.  Drop me a line and tell me what you think!  I'm @inertia on STEEM.

data/Rakefile

@@ -31,6 +31,16 @@ task :push do
   exec "gem push meeseeker-#{Meeseeker::VERSION}.gem"
 end
 
+desc 'Build a new version of the meeseeker docker image.'
+task :docker_build do
+  exec 'docker build -t inertia/meeseeker:latest .'
+end
+
+desc 'Publish the current version of the meeseeker docker image.'
+task :docker_push do
+  exec 'docker push inertia/meeseeker:latest'
+end
+
 task :check_schema do
   begin
     abort 'Unable to ping redis source.' unless Meeseeker.redis.ping == 'PONG'
@@ -41,8 +51,17 @@ task :check_schema do
   end
 end
 
-task(:sync, [:at_block_num] => [:check_schema]) do |t, args|
-  job = Meeseeker::BlockFollowerJob.new
+task(:sync, [:chain, :at_block_num] => [:check_schema]) do |t, args|
+  chain = (args[:chain] || 'steem').to_sym
+  
+  job = case chain
+  when :steem
+    Meeseeker::BlockFollowerJob.new
+  when :steem_engine
+    Meeseeker::SteemEngine::FollowerJob.new
+  else; abort("Unknown chain: #{chain}")
+  end
+  
   job.perform(at_block_num: args[:at_block_num])
 end
 
@@ -54,11 +73,13 @@ namespace :witness do
   end
 end
 
-task(:find, [:what, :key] => [:check_schema]) do |t, args|
+task(:find, [:what, :key, :chain] => [:check_schema]) do |t, args|
+  chain = (args[:chain] || 'steem').downcase.to_sym
   redis = Meeseeker.redis
+  
   match = case args[:what].downcase.to_sym
-  when :block then "steem:#{args[:key]}:*"
-  when :trx then "steem:*:#{args[:key]}:*"
+  when :block then "#{chain}:#{args[:key]}:*"
+  when :trx then "#{chain}:*:#{args[:key]}:*"
   else; abort "Unknown lookup using #{args}"
   end
 
@@ -74,6 +95,7 @@ end
 task reset: [:check_schema] do
   print 'Dropping keys ...'
   keys = Meeseeker.redis.keys('steem:*')
+  keys += Meeseeker.redis.keys('steem_engine:*')
   
   if keys.any?
     print " found #{keys.size} keys ..."
@@ -221,6 +243,119 @@ namespace :verify do
     end
   end
   
+  desc 'Verifies Steem Engine transactions land where they should.'
+  task :steem_engine_block_org, [:max_blocks] do |t, args|
+    max_blocks = args[:max_blocks]
+    node_url = ENV.fetch('MEESEEKER_STEEM_ENGINE_NODE_URL', 'https://api.steem-engine.com/rpc')
+    agent = Meeseeker::SteemEngine::Agent.new(url: node_url)
+    until_block_num = if !!max_blocks
+      agent.latest_block_info['blockNumber']
+    end
+    
+    Thread.new do
+      job = Meeseeker::SteemEngine::FollowerJob.new
+      
+      loop do
+        begin
+          at_block_num = agent.latest_block_info["blockNumber"] - max_blocks.to_i
+          at_block_num = [at_block_num, 1].max
+          job.perform(at_block_num: at_block_num, until_block_num: until_block_num)
+        rescue => e
+          puts e.inspect
+          sleep 5
+        end
+        
+        break # success
+      end
+      
+      puts 'Background sync finished ...'
+    end
+    
+    begin
+      block_channel = 'steem_engine:block'
+      redis_url = ENV.fetch('MEESEEKER_REDIS_URL', 'redis://127.0.0.1:6379/0')
+      subscription = Redis.new(url: redis_url)
+      ctx = Redis.new(url: redis_url)
+      timeout = (max_blocks).to_i * 3
+      
+      subscribe_mode, subscribe_args = if timeout > 0
+        [:subscribe_with_timeout, [timeout, [block_channel]]]
+      else
+        [:subscribe, [[block_channel]]]
+      end
+      
+      subscription.send(subscribe_mode, *subscribe_args) do |on|
+        on.subscribe do |channel, subscriptions|
+          puts "Subscribed to ##{channel} (subscriptions: #{subscriptions})"
+        end
+        
+        on.message do |channel, message|
+          payload = JSON[message]
+          block_num = payload['block_num']
+          next_block_num = block_num + 1
+          
+          if !!max_blocks
+            if block_num >= until_block_num
+              # We're done trailing blocks.  Typically, this is used by unit
+              # tests so the test can halt.
+              
+              subscription.unsubscribe
+              next
+            end
+          end
+          
+          while ctx.keys("steem_engine:#{next_block_num}:*").size == 0
+            # This ensures at least the next block has been indexed before
+            # proceeding.
+            
+            puts "Waiting for block: #{next_block_num} ..."
+            sleep 6
+          end
+          
+          # In theory, we should have all the keys using this pattern.
+          keys = ctx.keys("steem_engine:#{block_num}:*")
+          
+          # If we have all the keys, we should also have all transaction ids.
+          expected_ids = keys.map { |k| k.split(':')[2] }.uniq
+          actual_ids = nil
+          
+          agent.block(block_num).tap do |block|
+            raise 'Got empty block result.' if block.nil?
+            
+            actual_ids = block['transactions'].map{|trx| trx['transactionId'].split('-').first}.uniq
+          end
+          
+          # We do an intersection to make sure there's no difference between
+          # the two copies, regardless of order, as opposed to just checking that
+          # the lengths match.
+          
+          (actual_ids & expected_ids).tap do |intersection|
+            all_sizes = [intersection.size, expected_ids.size, actual_ids.size]
+            puts 'intersection: %d; expected: %d; actual: %d' % all_sizes
+            
+            if all_sizes.min != all_sizes.max
+              puts "Expected transaction ids:"
+              puts expected_ids
+              puts "Actual transaction ids:"
+              puts actual_ids
+              
+              puts "actual_ids minus expected:"
+              puts actual_ids - expected_ids
+              puts "expected_ids minus actual:"
+              puts expected_ids - actual_ids
+              
+              exit(-1)
+            end
+          end
+        end
+        
+        on.unsubscribe do |channel, subscriptions|
+          puts "Unsubscribed from ##{channel} (subscriptions: #{subscriptions})"
+        end
+      end
+    end
+  end
+    
   namespace :witness do
     desc 'Verifies witnessses in the schedule produced a block.'
     task :schedule, [:max_blocks] do |t, args|

data/bin/meeseeker

@@ -23,7 +23,7 @@ when 'sync', 'witness:schedule'
   max_backoff = 30
   
   loop do; begin
-    Rake::Task[ARGV[0]].invoke(ARGV[1])
+    Rake::Task[ARGV[0]].invoke(*ARGV[1..-1])
   rescue => e
     puts "Error: #{e.inspect}"
     backoff = [backoff, max_backoff].min
@@ -35,7 +35,7 @@ when 'find' then Rake::Task['find'].invoke(*ARGV[1..-1])
 when 'reset' then Rake::Task['reset'].invoke
 else
   puts "\nBegin/resume sync:"
-  puts "\t#{filename} sync\n\n"
+  puts "\t#{filename} sync [chain] [block_num]\n\n"
   puts "Publish witness schedule:"
   puts "\t#{filename} witness:schedule\n\n"
   puts "Start in the ruby console:"

data/lib/meeseeker.rb

@@ -4,13 +4,17 @@ require 'steem'
 require 'meeseeker/version'
 require 'meeseeker/block_follower_job'
 require 'meeseeker/witness_schedule_job'
+require 'meeseeker/steem_engine/agent'
+require 'meeseeker/steem_engine/follower_job'
 
 module Meeseeker
   LAST_BLOCK_NUM_KEY = 'steem:meeseeker:last_block_num'
+  LAST_STEEM_ENGINE_BLOCK_NUM_KEY = 'steem:meeseeker:last_steem_engine_block_num'
   BLOCKS_PER_DAY = 28800
   VIRTUAL_TRX_ID = '0000000000000000000000000000000000000000'
   @redis = Redis.new(url: ENV.fetch('MEESEEKER_REDIS_URL', 'redis://127.0.0.1:6379/0'))
   @node_url = ENV.fetch('MEESEEKER_NODE_URL', 'https://api.steemit.com')
+  @steem_engine_node_url = ENV.fetch('MEESEEKER_STEEM_ENGINE_NODE_URL', 'https://api.steem-engine.com/rpc')
   @stream_mode = ENV.fetch('MEESEEKER_STREAM_MODE', 'head').downcase.to_sym
   @include_virtual = ENV.fetch('MEESEEKER_INCLUDE_VIRTUAL', 'true').downcase == 'true'
   @include_block_header = ENV.fetch('MEESEEKER_INCLUDE_BLOCK_HEADER', 'true').downcase == 'true'
@@ -19,6 +23,6 @@ module Meeseeker
   
   extend self
 
-  attr_accessor :redis, :node_url, :expire_keys, :stream_mode, :include_virtual,
-    :include_block_header, :publish_op_custom_id
+  attr_accessor :redis, :node_url, :steem_engine_node_url, :expire_keys,
+    :stream_mode, :include_virtual, :include_block_header, :publish_op_custom_id
 end

data/lib/meeseeker/steem_engine/agent.rb

@@ -0,0 +1,60 @@
+require 'mechanize'
+
+module Meeseeker::SteemEngine
+  class Agent < Mechanize
+    POST_HEADERS = {
+      'Content-Type' => 'application/json; charset=utf-8',
+      'User-Agent' => Meeseeker::AGENT_ID
+    }
+    
+    def initialize(options = {})
+      super
+      
+      self.user_agent = Meeseeker::AGENT_ID
+      self.max_history = 0
+      self.default_encoding = 'UTF-8'
+      
+      @node_url = options[:url] || Meeseeker::steem_engine_node_url
+    end
+    
+    def blockchain_uri
+      @blockchain_uri ||= URI.parse(@node_url + '/blockchain')
+    end
+    
+    def blockchain_http_post
+      @http_post ||= Net::HTTP::Post.new(blockchain_uri.request_uri, POST_HEADERS)
+    end
+
+    def latest_block_info
+      request_body = {
+        jsonrpc: "2.0",
+        method: :getLatestBlockInfo,
+        id: rpc_id
+      }.to_json
+      
+      response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
+      
+      JSON[response.body]["result"]
+    end
+
+    def block(block_num)
+      request_body = {
+        jsonrpc: "2.0",
+        method: :getBlockInfo,
+        params: {
+          blockNumber: block_num
+        },
+        id: rpc_id
+      }.to_json
+      
+      response = request_with_entity :post, blockchain_uri, request_body, POST_HEADERS
+      
+      JSON[response.body]["result"]
+    end
+  private
+    def rpc_id
+      @rpc_id ||= 0
+      @rpc_id = @rpc_id + 1
+    end
+  end
+end

data/lib/meeseeker/steem_engine/follower_job.rb

@@ -0,0 +1,107 @@
+module Meeseeker::SteemEngine
+  class FollowerJob
+    def perform(options = {})
+      redis = Meeseeker.redis
+      last_key_prefix = nil
+      trx_index = 0
+      current_block_num = nil
+      block_transactions = []
+      
+      stream_transactions(options) do |transaction, block|
+        begin
+          trx_id = transaction['transactionId'].split('-').first
+          block_num = block['blockNumber']
+          current_key_prefix = "steem_engine:#{block_num}:#{trx_id}"
+          contract = transaction['contract']
+          action = transaction['action']
+
+          if current_key_prefix == last_key_prefix
+            trx_index += 1
+          else
+            if !!last_key_prefix
+              n, b, t = last_key_prefix.split(':')
+              transaction_payload = {
+                block_num: b.to_i,
+                transaction_id: t,
+                transaction_num: block_transactions.size
+              }
+              
+              block_transactions << trx_id
+              redis.publish('steem_engine:transaction', transaction_payload.to_json)
+            end
+            
+            last_key_prefix = "steem_engine:#{block_num}:#{trx_id}"
+            trx_index = 0
+          end
+          
+          key = "#{current_key_prefix}:#{trx_index}:#{contract}:#{action}"
+          puts key
+        end
+
+        redis.set(key, transaction.to_json)
+        redis.expire(key, Meeseeker.expire_keys) unless Meeseeker.expire_keys == -1
+        
+        if current_block_num != block_num
+          block_transactions = []
+          block_payload = {
+            block_num: block_num
+          }
+          
+          redis.set(Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY, block_num)
+          redis.publish('steem_engine:block', block_payload.to_json)
+          current_block_num = block_num
+        end
+        
+        redis.publish("steem_engine:#{contract}:#{action}", {key: key}.to_json)
+      end
+    end
+  private
+    def stream_transactions(options = {}, &block)
+      redis = Meeseeker.redis
+      last_block_num = nil
+      agent = Agent.new
+      
+      if !!options[:at_block_num]
+        last_block_num = options[:at_block_num].to_i
+      else
+        last_block_num = redis.get(Meeseeker::LAST_STEEM_ENGINE_BLOCK_NUM_KEY).to_i + 1
+        block_info = agent.latest_block_info
+        block_num = block_info['blockNumber']
+        
+        last_block = agent.block(last_block_num)
+        last_block_timestamp = Time.parse(last_block['timestamp'] + 'Z')
+        
+        if Meeseeker.expire_keys == -1
+          last_block_num = [last_block_num, block_num].max
+          
+          puts "Sync Steem Engine from: #{last_block_num}"
+        elsif Time.now.utc - last_block_timestamp > Meeseeker.expire_keys
+          last_block_num = block_num + 1
+          
+          puts 'Starting new Steem Engine sync.'
+        else
+          puts "Resuming from Steem Engine block #{last_block_num} ..."
+        end
+      end
+      
+      block_num = last_block_num
+      
+      loop do
+        block = agent.block(block_num)
+        
+        if block.nil?
+          sleep 3 # sleep for one mainnet block interval
+          redo
+        end
+        
+        transactions = block['transactions']
+        
+        transactions.each do |transaction|
+          yield transaction.merge(timestamp: block['timestamp']), block
+        end
+        
+        block_num = block_num + 1
+      end
+    end
+  end
+end

data/lib/meeseeker/version.rb

@@ -1,3 +1,4 @@
 module Meeseeker
-  VERSION = '0.0.5'
+  VERSION = '0.0.6'
+  AGENT_ID = "meeseeker/#{VERSION}"
 end

data/meeseeker.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'minitest-proveit', '~> 1.0', '>= 1.0.0'
   s.add_development_dependency 'simplecov', '~> 0.15', '>= 0.15.1'
   s.add_development_dependency 'pry', '~> 0.11', '>= 0.11.3'
+  s.add_development_dependency 'irb', '~> 1.0', '>= 1.0.0'
 
   s.add_dependency 'redis', '~> 4.1', '>= 4.1.0'
   s.add_dependency 'steem-mechanize', '~> 0.0', '>= 0.0.5'

data/test/meeseeker/meeseeker_test.rb

@@ -22,6 +22,7 @@ module Meeseeker
       max_blocks = 30 # must be at least 15 to get past irreversible
     
       assert Rake::Task['verify:block_org'].invoke(max_blocks)
+      assert Rake::Task['verify:steem_engine_block_org'].invoke(max_blocks)
       assert Rake::Task['verify:witness:schedule'].invoke(max_blocks)
       
       Rake::Task['reset'].invoke

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: meeseeker
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Anthony Martin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-11 00:00:00.000000000 Z
+date: 2019-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -130,6 +130,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.11.3
+- !ruby/object:Gem::Dependency
+  name: irb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -207,6 +227,8 @@ files:
 - bin/meeseeker
 - lib/meeseeker.rb
 - lib/meeseeker/block_follower_job.rb
+- lib/meeseeker/steem_engine/agent.rb
+- lib/meeseeker/steem_engine/follower_job.rb
 - lib/meeseeker/version.rb
 - lib/meeseeker/witness_schedule_job.rb
 - meeseeker.gemspec