meeseeker 0.0.3pre1 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '058271f9eb089b43a706275a96a06f4b87ac8bbf96b2404a2bbd7c239c2882fc'
-  data.tar.gz: f5f73c53bc34c485b46d94e887249ce65dc3114ee0cf60189849a28786656a27
+  metadata.gz: 8b01bc12292352c934627c0f6ed373b6eec30dd05fc91ddcfbe83a5aa3da1e95
+  data.tar.gz: '07983e2f5ab16f93e178160140e208636fb5e073e10aac7b5a0212eb50e49f68'
 SHA512:
-  metadata.gz: dfb048376f1a0e17bed91a44116d2718a99127eeba0d770ae50d535eead02d298b1105401385b76e06124e94a3e777b7ecf8bd188a4a4c7c6bc3cff635774835
-  data.tar.gz: 74ac56d7d083c6f1e2d5da9496079bf670f4bd12d15122044ac295fe6b75b05c50402cc0d969ab902f54dc264ffa574ee6a78f89a60fb6c9f2597c6db484e417
+  metadata.gz: 04d532184efc2c4d086e78e25e8e96106bbd0bf15121b2b55e25ef872874ea44da99ed994fe4e96ee6653dd002be7f82d3c1dbe5fa751703a6167487f4989631
+  data.tar.gz: 195cd89d7cdad7f430f6b2b91816c50bca5bafc52551edc8cb8ff253620980ceb872f7ad360a0a25824aefa038a4dbf978a25b843a9c509f62467d717974d684

data/README.md

@@ -107,7 +107,9 @@ When `meeseeker sync` starts for the first time, it initializes from the last ir
 
 For `redis-cli`, please see: https://redis.io/topics/pubsub
 
-Channels available for `meeseeker`:
+##### Sync
+
+When running `meeseeker sync`, the following channels are available:
 
 * `steem:block`
 * `steem:transaction`
@@ -193,6 +195,24 @@ end
 
 Many other clients are supported: https://redis.io/clients
 
+##### Witness Schedule
+
+When running `meeseeker witness:schedule`, the `steem:witness:schedule` channel is available.  This is offered as a separate command because most applications don't need to worry about this level of blockchain logistics.
+
+For example, from `redis-cli`, if we wanted to subscribe to the witness schedule:
+
+```
+$ redis-cli
+127.0.0.1:6379> subscribe steem:witness:schedule
+Reading messages... (press Ctrl-C to quit)
+1) "subscribe"
+2) "steem:witness:schedule"
+3) (integer) 1
+1) "message"
+2) "steem:witness:schedule"
+3) "{\"id\":0,\"current_virtual_time\":\"415293532210075480213212125\",\"next_shuffle_block_num\":30035208,\"current_shuffled_witnesses\":[\"thecryptodrive\",\"timcliff\",\"utopian-io\",\"themarkymark\",\"aggroed\",\"smooth.witness\",\"someguy123\",\"gtg\",\"followbtcnews\",\"yabapmatt\",\"therealwolf\",\"ausbitbank\",\"curie\",\"clayop\",\"drakos\",\"blocktrades\",\"good-karma\",\"roelandp\",\"lukestokes.mhth\",\"liondani\",\"anyx\"],\"num_scheduled_witnesses\":21,\"elected_weight\":1,\"timeshare_weight\":5,\"miner_weight\":1,\"witness_pay_normalization_factor\":25,\"median_props\":{\"account_creation_fee\":{\"amount\":\"3000\",\"precision\":3,\"nai\":\"@@000000021\"},\"maximum_block_size\":65536,\"sbd_interest_rate\":0,\"account_subsidy_budget\":797,\"account_subsidy_decay\":347321},\"majority_version\":\"0.20.8\",\"max_voted_witnesses\":20,\"max_miner_witnesses\":0,\"max_runner_witnesses\":1,\"hardfork_required_witnesses\":17,\"account_subsidy_rd\":{\"resource_unit\":10000,\"budget_per_time_unit\":797,\"pool_eq\":157691079,\"max_pool_size\":157691079,\"decay_params\":{\"decay_per_time_unit\":347321,\"decay_per_time_unit_denom_shift\":36},\"min_decay\":0},\"account_subsidy_witness_rd\":{\"resource_unit\":10000,\"budget_per_time_unit\":996,\"pool_eq\":9384019,\"max_pool_size\":9384019,\"decay_params\":{\"decay_per_time_unit\":7293741,\"decay_per_time_unit_denom_shift\":36},\"min_decay\":257},\"min_witness_account_subsidy_decay\":0}"
+```
+
 #### Using `SCAN`
 
 From the redis manual:
@@ -276,6 +296,26 @@ redis-cli --scan --pattern 'steem:*:31ecb9c85e9eabd7ca2460fdb4f3ce4a7ca6ec32:*'
 
 See some of my previous Ruby How To posts in: [#radiator](https://steemit.com/created/radiator) [#ruby](https://steemit.com/created/ruby)
 
+### Docker
+
+This will launch meeseeker in a docker container, so you can immediately attach to it on port 6380.
+
+```bash
+docker run -d -p 6380:6379 inertia/meeseeker:latest
+redis-cli -p 6380
+```
+
+You can also pass any of the environment variables meeseeker accepts.  For example, this will launch meeseeker with `custom_json.id` channels enabled, but only keeps ops around for 5 minutes:
+
+```bash
+docker run \
+  --env MEESEEKER_PUBLISH_OP_CUSTOM_ID=true \
+  --env MEESEEKER_EXPIRE_KEYS=300 \
+  -d -p 6380:6379 inertia/meeseeker:latest
+```
+
+Also see: https://hub.docker.com/r/inertia/meeseeker/
+
 ## Get in touch!
 
 If you're using Radiator, I'd love to hear from you.  Drop me a line and tell me what you think!  I'm @inertia on STEEM.

data/Rakefile

@@ -2,6 +2,8 @@ require "bundler/gem_tasks"
 require "rake/testtask"
 require 'meeseeker'
 
+defined? Thread.report_on_exception and Thread.report_on_exception = true
+
 Rake::TestTask.new(:test) do |t|
   t.libs << 'test'
   t.libs << 'lib'
@@ -44,6 +46,14 @@ task(:sync, [:at_block_num] => [:check_schema]) do |t, args|
   job.perform(at_block_num: args[:at_block_num])
 end
 
+namespace :witness do
+  desc 'Publish the witness schedule every minute or so (steem:witness:schedule).'
+  task :schedule do
+    job = Meeseeker::WitnessScheduleJob.new
+    job.perform
+  end
+end
+
 task(:find, [:what, :key] => [:check_schema]) do |t, args|
   redis = Meeseeker.redis
   match = case args[:what].downcase.to_sym
@@ -77,25 +87,24 @@ end
 namespace :verify do
   desc 'Verifies transactions land where they should.'
   task :block_org, [:max_blocks] do |t, args|
-    defined? Thread.report_on_exception and Thread.report_on_exception = true
-    
     max_blocks = args[:max_blocks]
     node_url = ENV.fetch('MEESEEKER_NODE_URL', 'https://api.steemit.com')
     database_api = Steem::DatabaseApi.new(url: node_url)
-    until_block_num = nil
+    mode = ENV.fetch('MEESEEKER_STREAM_MODE', 'head').to_sym
+    until_block_num = if !!max_blocks
+      database_api.get_dynamic_global_properties do |dgpo|
+        raise 'Got empty dynamic_global_properties result.' if dgpo.nil?
+        
+        case mode
+        when :head then dgpo.head_block_number
+        when :irreversible then dgpo.last_irreversible_block_num
+        else; abort "Unknown block mode: #{mode}"
+        end
+      end + max_blocks.to_i
+    end
     
     Thread.new do
       job = Meeseeker::BlockFollowerJob.new
-      mode = ENV.fetch('MEESEEKER_STREAM_MODE', 'head').to_sym
-      until_block_num = if !!max_blocks
-        database_api.get_dynamic_global_properties do |dgpo|
-          case mode
-          when :head then dgpo.head_block_number
-          when :irreversible then dgpo.last_irreversible_block_num
-          else; abort "Unknown block mode: #{mode}"
-          end
-        end + max_blocks.to_i
-      end
       
       loop do
         begin
@@ -138,6 +147,9 @@ namespace :verify do
           
           if !!max_blocks
             if block_num >= until_block_num
+              # We're done trailing blocks.  Typically, this is used by unit
+              # tests so the test can halt.
+              
               subscription.unsubscribe
               next
             end
@@ -152,6 +164,8 @@ namespace :verify do
           end
           
           database_api.get_dynamic_global_properties do |dgpo|
+            raise 'Got empty dynamic_global_properties result.' if dgpo.nil?
+            
             (block_num - dgpo.last_irreversible_block_num).tap do |offset|
               # This will block all channel callbacks until the first known block
               # is irreversible.  After that, the offsets should mostly go
@@ -169,6 +183,8 @@ namespace :verify do
           expected_ids -= [Meeseeker::VIRTUAL_TRX_ID]
           
           actual_ids, actual_witness = block_api.get_block(block_num: block_num) do |result|
+            raise 'Got empty block result.' if result.nil? || result.block.nil?
+            
             block = result.block
             [block.transaction_ids, block.witness]
           end
@@ -204,4 +220,156 @@ namespace :verify do
       end
     end
   end
+  
+  namespace :witness do
+    desc 'Verifies witnessses in the schedule produced a block.'
+    task :schedule, [:max_blocks] do |t, args|
+      max_blocks = args[:max_blocks]
+      node_url = ENV.fetch('MEESEEKER_NODE_URL', 'https://api.steemit.com')
+      database_api = Steem::DatabaseApi.new(url: node_url)
+      mode = ENV.fetch('MEESEEKER_STREAM_MODE', 'head').to_sym
+      until_block_num = if !!max_blocks
+        database_api.get_dynamic_global_properties do |dgpo|
+          raise 'Got empty dynamic_global_properties result.' if dgpo.nil?
+          
+          case mode
+          when :head then dgpo.head_block_number
+          when :irreversible then dgpo.last_irreversible_block_num
+          else; abort "Unknown block mode: #{mode}"
+          end
+        end + max_blocks.to_i
+      end
+      
+      Thread.new do
+        job = Meeseeker::WitnessScheduleJob.new
+        
+        loop do
+          begin
+            job.perform(mode: mode, until_block_num: until_block_num)
+          rescue => e
+            puts e.inspect
+            sleep 5
+          end
+          
+          break # success
+        end
+        
+        puts 'Background sync finished ...'
+      end
+    
+      begin
+        block_api = Steem::BlockApi.new(url: node_url)
+        schedule_channel = 'steem:witness:schedule'
+        redis_url = ENV.fetch('MEESEEKER_REDIS_URL', 'redis://127.0.0.1:6379/0')
+        subscription = Redis.new(url: redis_url)
+        ctx = Redis.new(url: redis_url)
+        timeout = (max_blocks).to_i * 3
+        
+        subscribe_mode, subscribe_args = if timeout > 0
+          [:subscribe_with_timeout, [timeout, [schedule_channel]]]
+        else
+          [:subscribe, [[schedule_channel]]]
+        end
+        
+        # Check if the redis context is still available right before we
+        # subscribe.
+        break unless subscription.ping == 'PONG'
+        
+        subscription.send(subscribe_mode, *subscribe_args) do |on|
+          on.subscribe do |channel, subscriptions|
+            puts "Subscribed to ##{channel} (subscriptions: #{subscriptions})"
+          end
+          
+          on.message do |channel, message|
+            payload = JSON[message]
+            next_shuffle_block_num = payload['next_shuffle_block_num']
+            current_shuffled_witnesses = payload['current_shuffled_witnesses']
+            num_witnesses = current_shuffled_witnesses.size
+            from_block_num = next_shuffle_block_num - num_witnesses + 1
+            to_block_num = from_block_num + num_witnesses - 1
+            block_range = from_block_num..to_block_num # typically 21 blocks
+            
+            if !!max_blocks
+              if block_range.include? until_block_num
+                # We're done trailing blocks.  Typically, this is used by unit
+                # tests so the test can halt.
+                
+                subscription.unsubscribe
+              end
+            end
+            
+            begin
+              # We write witnesses to this hash until all 21 produce blocks.
+              actual_witnesses = {}
+              tries = 0
+              
+              while actual_witnesses.size != num_witnesses
+                # Allow the immediate node to catch up in case it's behind by a
+                # block.
+                sleep 3
+                
+                # Typically, nodes will allow up to 50 block headers in one
+                # request, if backed by jussi.  We only need 21, so each
+                # request should only make a single response with the entire
+                # round.  Under normal circumstances, this call happens only
+                # once.  But if the there's additional p2p or cache latency,
+                # it might have missing headers.
+                
+                block_api.get_block_headers(block_range: block_range) do |header, block_num|
+                  unless !!header
+                    # Can happen when there's excess p2p latency and/or jussi
+                    # cache is under load.
+                    puts "Waiting for block header: #{block_num}"
+                    
+                    next
+                  end
+                  
+                  actual_witnesses[header.witness] = block_num
+                end
+                
+                break if (tries += 1) > 5
+              end
+              
+              # If there are multiple tries due to high p2p latency, even though
+              # we got all 21 block headers, seeing this message could be an
+              # early-warning of other problems on the blockchain.
+              
+              # If there's a missing block header, this will always show 5
+              # tries.
+              
+              puts "Tries: #{tries}" if tries > 1
+              
+              missing_witnesses = current_shuffled_witnesses - actual_witnesses.keys
+              extra_witnesses = actual_witnesses.keys - current_shuffled_witnesses
+              
+              if missing_witnesses.any? || extra_witnesses.any?
+                puts "Expected only these witness to produce a block in #{block_range}."
+                puts "Missing witnesses: #{missing_witnesses.join(', ')}"
+                puts "Extra witnesses: #{extra_witnesses.join(', ')}"
+                
+                puts "\nWitnesses and block numbers in range:"
+                actual_witnesses.sort_by{ |k, v| v }.each do |k, v|
+                  puts "#{v}: #{k}"
+                end
+                puts "Count: #{actual_witnesses.size}"
+                
+                # Non-zero exit to notify the shell caller that there's a
+                # problem.
+                
+                exit(-(missing_witnesses.size + extra_witnesses.size))
+              end
+            end
+            
+            # Perfect round.
+            
+            puts "Found all #{num_witnesses} expected witnesses in block range #{block_range}: √"
+          end
+          
+          on.unsubscribe do |channel, subscriptions|
+            puts "Unsubscribed from ##{channel} (subscriptions: #{subscriptions})"
+          end
+        end
+      end
+    end
+  end
 end

data/bin/meeseeker

@@ -18,12 +18,12 @@ filename = __FILE__.split('/').last
 
 case ARGV[0]
 when 'console' then Rake::Task['console'].invoke
-when 'sync'
+when 'sync', 'witness:schedule'
   backoff = 0.01
   max_backoff = 30
   
   loop do; begin
-    Rake::Task['sync'].invoke(ARGV[1])
+    Rake::Task[ARGV[0]].invoke(ARGV[1])
   rescue => e
     puts "Error: #{e.inspect}"
     backoff = [backoff, max_backoff].min
@@ -36,6 +36,8 @@ when 'reset' then Rake::Task['reset'].invoke
 else
   puts "\nBegin/resume sync:"
   puts "\t#{filename} sync\n\n"
+  puts "Publish witness schedule:"
+  puts "\t#{filename} witness:schedule\n\n"
   puts "Start in the ruby console:"
   puts "\t#{filename} console\n\n"
   puts 'Find block or transaction:'

data/lib/meeseeker.rb

@@ -3,6 +3,7 @@ require 'steem'
 
 require 'meeseeker/version'
 require 'meeseeker/block_follower_job'
+require 'meeseeker/witness_schedule_job'
 
 module Meeseeker
   LAST_BLOCK_NUM_KEY = 'steem:meeseeker:last_block_num'

data/lib/meeseeker/block_follower_job.rb

@@ -45,8 +45,12 @@ module Meeseeker
           }
           
           if Meeseeker.include_block_header
-            block_api.get_block_header(block_num: block_num) do |result|
-              block_payload = block_payload.merge(result.header.to_h)
+            catch :block_header do
+              block_api.get_block_header(block_num: block_num) do |result|
+                throw :block_header if result.nil || result.header.nil?
+                
+                block_payload.merge!(result.header.to_h)
+              end
             end
           end
           
@@ -81,28 +85,32 @@ module Meeseeker
         database_api = Steem::DatabaseApi.new(url: Meeseeker.node_url)
         last_block_num = redis.get(LAST_BLOCK_NUM_KEY).to_i + 1
         
-        database_api.get_dynamic_global_properties do |dgpo|
-          block_num = case mode
-          when :head then dgpo.head_block_number
-          when :irreversible then dgpo.last_irreversible_block_num
-          else; abort "Unknown stream mode: #{mode}"
-          end
-          
-          if Meeseeker.expire_keys == -1
-            last_block_num = [last_block_num, block_num].max
-            
-            puts "Sync from: #{last_block_num}"
-          elsif block_num - last_block_num > Meeseeker.expire_keys / 3
-            last_block_num = block_num
+        block_num = catch :dynamic_global_properties do
+          database_api.get_dynamic_global_properties do |dgpo|
+            throw :dynamic_global_properties if dgpo.nil?
             
-            puts 'Starting new sync.'
-          else
-            behind_sec = block_num - last_block_num
-            behind_sec *= 3.0
-            
-            puts "Resuming from #{behind_sec / 60} minutes ago ..."
+            case mode
+            when :head then dgpo.head_block_number
+            when :irreversible then dgpo.last_irreversible_block_num
+            else; abort "Unknown stream mode: #{mode}"
+            end
           end
         end
+          
+        if Meeseeker.expire_keys == -1
+          last_block_num = [last_block_num, block_num].max
+          
+          puts "Sync from: #{last_block_num}"
+        elsif block_num - last_block_num > Meeseeker.expire_keys / 3
+          last_block_num = block_num
+          
+          puts 'Starting new sync.'
+        else
+          behind_sec = block_num - last_block_num
+          behind_sec *= 3.0
+          
+          puts "Resuming from #{behind_sec / 60} minutes ago ..."
+        end
       end
       
       begin

data/lib/meeseeker/version.rb

@@ -1,3 +1,3 @@
 module Meeseeker
-  VERSION = '0.0.3pre1'
+  VERSION = '0.0.4'
 end

data/lib/meeseeker/witness_schedule_job.rb

@@ -0,0 +1,68 @@
+module Meeseeker
+  class WitnessScheduleJob
+    def perform(options = {})
+      database_api = Steem::DatabaseApi.new(url: Meeseeker.node_url)
+      redis = Meeseeker.redis
+      mode = options.delete(:mode) || Meeseeker.stream_mode
+      schedule = nil
+      last_shuffle_block_num = nil
+      
+      loop do
+        # Using hammer assignment will ensure we only request a new schedule
+        # after we've published.
+        
+        schedule ||= catch :witness_schedule do
+          database_api.get_witness_schedule do |result|
+            throw :witness_schedule if result.nil?
+          
+            result
+          end
+        end
+        
+        next_shuffle_block_num = schedule.next_shuffle_block_num
+        block_num = catch :dynamic_global_properties do
+          database_api.get_dynamic_global_properties do |dgpo|
+            throw :dynamic_global_properties if dgpo.nil?
+            
+            case mode
+            when :head then dgpo.head_block_number
+            when :irreversible then dgpo.last_irreversible_block_num
+            else; abort "Unknown stream mode: #{mode}"
+            end
+          end
+        end
+        
+        # Find out how far away we are from the next schedule.
+        
+        remaining_blocks = [next_shuffle_block_num - block_num - 1.5, 0].max
+        
+        # It's better for the schedule to publish a little late than to miss
+        # an entire schedule, so we subtract 1.5 blocks from the total.
+        # Sometimes we check a little early and sometimes we check a little
+        # late.  But it all averages out.
+        
+        if remaining_blocks > 0
+          delay = [remaining_blocks * 3.0, 0.25].max
+          puts "Sleeping for #{delay} seconds (remaining blocks: #{remaining_blocks})."
+          sleep delay
+          next
+        end
+        
+        # Now that we've reached the current schedule, check if we've published
+        # it already.  If not, publish and reset for the next schedule.
+        
+        if next_shuffle_block_num != last_shuffle_block_num
+          puts "next_shuffle_block_num: #{next_shuffle_block_num}; current_shuffled_witnesses: #{schedule.current_shuffled_witnesses.join(', ')}"
+          redis.publish('steem:witness:schedule', schedule.to_json)
+          last_shuffle_block_num = next_shuffle_block_num
+        end
+        
+        schedule = nil # re-enabled hammer assignment
+        
+        if !!options[:until_block_num]
+          break if block_num >= options[:until_block_num].to_i
+        end
+      end
+    end
+  end
+end

data/test/meeseeker/meeseeker_test.rb

@@ -12,16 +12,19 @@ module Meeseeker
       Rake.application.init
       Rake.application.load_rakefile
       Dir.chdir(pwd)
-    end
-    
-    def test_verify_block_org
-      max_blocks = 30 # must be at least 15 to get past irreversible
+      
       if !!Meeseeker.redis.get(Meeseeker::LAST_BLOCK_NUM_KEY)
         fail "Found existing keys.  Please use 'rake reset' to enable this test."
       end
-      
+    end
+    
+    def test_verify_all_jobs
+      max_blocks = 30 # must be at least 15 to get past irreversible
+    
       assert Rake::Task['verify:block_org'].invoke(max_blocks)
-      assert Rake::Task['reset'].invoke
+      assert Rake::Task['verify:witness:schedule'].invoke(max_blocks)
+      
+      Rake::Task['reset'].invoke
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: meeseeker
 version: !ruby/object:Gem::Version
-  version: 0.0.3pre1
+  version: 0.0.4
 platform: ruby
 authors:
 - Anthony Martin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-01 00:00:00.000000000 Z
+date: 2019-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -208,6 +208,7 @@ files:
 - lib/meeseeker.rb
 - lib/meeseeker/block_follower_job.rb
 - lib/meeseeker/version.rb
+- lib/meeseeker/witness_schedule_job.rb
 - meeseeker.gemspec
 - test/meeseeker/meeseeker_test.rb
 - test/test_helper.rb
@@ -227,9 +228,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.7.7