steem-ruby 0.9.1 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d2df68c704478da2f78e2cdee3a423523950f27
-  data.tar.gz: f8e6912e7b808be2bb7c065dfb177d0ef6a98a53
+  metadata.gz: 25b2e7d4eb5445c89b02aac685677e49a0c551c8
+  data.tar.gz: 9b5f21517af8ea96a85c751d0a470463cff09ea7
 SHA512:
-  metadata.gz: 8b93714f563a05179fdbcf1c45af4e2279047343b86d1e932bb5ea306a8c7a508f8769b8a22a9415f4344effd591d0f5f86bcfc02a004bf34478fdd140a599ec
-  data.tar.gz: 12bf547182d8a9abae9d0182cb21f509e176c46f283b3683cebc7b57fc9efb6b3cdabf94538ba1b86550824e5483526257b13d45476d63bf7402bff5b393902e
+  metadata.gz: d960cab6d69f53dc73366084c85bb98ca4534829f225306125ca517b4adb391d108483836be355659d32143058daa19b7fa2685bfa4dd2a8b192677cb786e272
+  data.tar.gz: ba077e084d71823040fb8470aa5793439356f8d7002da3763b60a4c4939d3655ff78a41ae920a98a8b15709a035b1c0ef9f39a296207675f296213201f2fbb6c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    steem-ruby (0.9.1)
+    steem-ruby (0.9.2)
       bitcoin-ruby (~> 0.0, >= 0.0.18)
       ffi (~> 1.9, >= 1.9.23)
       hashie (~> 3.5, >= 3.5.7)
@@ -50,7 +50,7 @@ GEM
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
-    yard (0.9.14)
+    yard (0.9.15)
 
 PLATFORMS
   ruby

data/README.md

@@ -70,6 +70,79 @@ end
 
 *See: [Broadcast](https://www.rubydoc.info/gems/steem-ruby/Steem/Broadcast)*
 
+### Streaming
+
+The value passed to the block is an object, with the keys: `:type` and `:value`.
+
+```ruby
+stream = Steem::Stream.new
+
+stream.operations do |op|
+  puts "#{op.type}: #{op.value}"
+end
+```
+
+To start a stream from a specific block number, pass it as an argument:
+
+```ruby
+stream = Steem::Stream.new
+
+stream.operations(at_block_num: 9001) do |op|
+  puts "#{op.type}: #{op.value}"
+end
+```
+
+You can also grab the related transaction id and block number for each operation:
+
+```ruby
+stream = Steem::Stream.new
+
+stream.operations do |op, trx_id, block_num|
+  puts "#{block_num} :: #{trx_id}"
+  puts "#{op.type}: #{op.value}"
+end
+```
+
+To stream only certain operations:
+
+```ruby
+stream = Steem::Stream.new
+
+stream.operations(types: :vote_operation) do |op|
+  puts "#{op.type}: #{op.value}"
+end
+```
+
+Or pass an array of certain operations:
+
+```ruby
+stream = Steem::Stream.new
+
+stream.operations(types: [:comment_operation, :vote_operation]) do |op|
+  puts "#{op.type}: #{op.value}"
+end
+```
+
+Or (optionally) just pass the operation(s) you want as the only arguments.  This is semantic sugar for when you want specific types and take all of the defaults.
+
+```ruby
+stream = Steem::Stream.new
+
+stream.operations(:vote_operation) do |op|
+  puts "#{op.type}: #{op.value}"
+end
+```
+
+To also include virtual operations:
+
+```ruby
+stream = Steem::Stream.new
+
+stream.operations(include_virtual: true) do |op|
+  puts "#{op.type}: #{op.value}"
+end
+```
+
 ### Multisig
 
 You can use multisignature to broadcast an operation.

data/Rakefile

@@ -138,44 +138,141 @@ end
   
 namespace :stream do
   desc 'Test the ability to stream a block range.'
-  task :block_range do
-    block_api = Steem::BlockApi.new(url: ENV['TEST_NODE'])
+  task :block_range, [:mode, :at_block_num] do |t, args|
+    mode = (args[:mode] || 'irreversible').to_sym
+    first_block_num = args[:at_block_num].to_i if !!args[:at_block_num]
+    stream = Steem::Stream.new(url: ENV['TEST_NODE'], mode: mode)
     api = Steem::Api.new(url: ENV['TEST_NODE'])
     last_block_num = nil
-    first_block_num = nil
     last_timestamp = nil
+    range_complete = false
     
-    loop do
-      api.get_dynamic_global_properties do |properties|
-        current_block_num = properties.last_irreversible_block_num
-        # First pass replays latest a random number of blocks to test chunking.
-        first_block_num ||= current_block_num - (rand * 200).to_i
+    api.get_dynamic_global_properties do |properties|
+      current_block_num = if mode == :head
+        properties.head_block_number
+      else
+        properties.last_irreversible_block_num
+      end
+      
+      # First pass replays latest a random number of blocks to test chunking.
+      first_block_num ||= current_block_num - (rand * 200).to_i
+      
+      range = first_block_num..current_block_num
+      puts "Initial block range: #{range.size}"
+      
+      stream.blocks(at_block_num: range.first) do |block, block_num|
+        current_timestamp = Time.parse(block.timestamp + 'Z')
         
-        if current_block_num >= first_block_num
-          range = first_block_num..current_block_num
-          puts "Got block range: #{range.size}"
-          block_api.get_blocks(block_range: range) do |block, block_num|
-            current_timestamp = Time.parse(block.timestamp + 'Z')
-            
-            if !!last_timestamp && block_num != last_block_num + 1
-              puts "Bug: Last block number was #{last_block_num} then jumped to: #{block_num}"
-              exit
-            end
-            
-            if !!last_timestamp && current_timestamp < last_timestamp
-              puts "Bug: Went back in time.  Last timestamp was #{last_timestamp}, then jumped back to #{current_timestamp}"
-              exit
-            end
-            
-            puts "\t#{block_num} Timestamp: #{current_timestamp}, witness: #{block.witness}"
-            last_block_num = block_num
-            last_timestamp = current_timestamp
-          end
-          
-          first_block_num = range.max + 1
+        if !range_complete && block_num > range.last
+          puts 'Done with initial range.'
+          range_complete = true
         end
         
-        sleep 3
+        if !!last_timestamp && block_num != last_block_num + 1
+          puts "Bug: Last block number was #{last_block_num} then jumped to: #{block_num}"
+          exit
+        end
+        
+        if !!last_timestamp && current_timestamp < last_timestamp
+          puts "Bug: Went back in time.  Last timestamp was #{last_timestamp}, then jumped back to #{current_timestamp}"
+          exit
+        end
+        
+        puts "\t#{block_num} Timestamp: #{current_timestamp}, witness: #{block.witness}"
+        last_block_num = block_num
+        last_timestamp = current_timestamp
+      end
+    end
+  end
+  
+  desc 'Test the ability to stream a block range of transactions.'
+  task :trx_range, [:mode, :at_block_num] do |t, args|
+    mode = (args[:mode] || 'irreversible').to_sym
+    first_block_num = args[:at_block_num].to_i if !!args[:at_block_num]
+    stream = Steem::Stream.new(url: ENV['TEST_NODE'], mode: mode)
+    api = Steem::Api.new(url: ENV['TEST_NODE'])
+    
+    api.get_dynamic_global_properties do |properties|
+      current_block_num = if mode == :head
+        properties.head_block_number
+      else
+        properties.last_irreversible_block_num
+      end
+      
+      # First pass replays latest a random number of blocks to test chunking.
+      first_block_num ||= current_block_num - (rand * 200).to_i
+      
+      stream.transactions(at_block_num: first_block_num) do |trx, trx_id, block_num|
+        puts "#{block_num} :: #{trx_id}; ops: #{trx.operations.map(&:type).join(', ')}"
+      end
+    end
+  end
+  
+  desc 'Test the ability to stream a block range of operations.'
+  task :op_range, [:mode, :at_block_num] do |t, args|
+    mode = (args[:mode] || 'irreversible').to_sym
+    first_block_num = args[:at_block_num].to_i if !!args[:at_block_num]
+    stream = Steem::Stream.new(url: ENV['TEST_NODE'], mode: mode)
+    api = Steem::Api.new(url: ENV['TEST_NODE'])
+    
+    api.get_dynamic_global_properties do |properties|
+      current_block_num = if mode == :head
+        properties.head_block_number
+      else
+        properties.last_irreversible_block_num
+      end
+      
+      # First pass replays latest a random number of blocks to test chunking.
+      first_block_num ||= current_block_num - (rand * 200).to_i
+      
+      stream.operations(at_block_num: first_block_num) do |op, trx_id, block_num|
+        puts "#{block_num} :: #{trx_id}; op: #{op.type}"
+      end
+    end
+  end
+  
+  desc 'Test the ability to stream a block range of virtual operations.'
+  task :vop_range, [:mode, :at_block_num] do |t, args|
+    mode = (args[:mode] || 'irreversible').to_sym
+    first_block_num = args[:at_block_num].to_i if !!args[:at_block_num]
+    stream = Steem::Stream.new(url: ENV['TEST_NODE'], mode: mode)
+    api = Steem::Api.new(url: ENV['TEST_NODE'])
+    
+    api.get_dynamic_global_properties do |properties|
+      current_block_num = if mode == :head
+        properties.head_block_number
+      else
+        properties.last_irreversible_block_num
+      end
+      
+      # First pass replays latest a random number of blocks to test chunking.
+      first_block_num ||= current_block_num - (rand * 200).to_i
+      
+      stream.operations(at_block_num: first_block_num, only_virtual: true) do |op, trx_id, block_num|
+        puts "#{block_num} :: #{trx_id}; op: #{op.type}"
+      end
+    end
+  end
+  
+  desc 'Test the ability to stream a block range of all operations (including virtual).'
+  task :all_op_range, [:mode, :at_block_num] do |t, args|
+    mode = (args[:mode] || 'irreversible').to_sym
+    first_block_num = args[:at_block_num].to_i if !!args[:at_block_num]
+    stream = Steem::Stream.new(url: ENV['TEST_NODE'], mode: mode)
+    api = Steem::Api.new(url: ENV['TEST_NODE'])
+    
+    api.get_dynamic_global_properties do |properties|
+      current_block_num = if mode == :head
+        properties.head_block_number
+      else
+        properties.last_irreversible_block_num
+      end
+      
+      # First pass replays latest a random number of blocks to test chunking.
+      first_block_num ||= current_block_num - (rand * 200).to_i
+      
+      stream.operations(at_block_num: first_block_num, include_virtual: true) do |op, trx_id, block_num|
+        puts "#{block_num} :: #{trx_id}; op: #{op.type}"
       end
     end
   end

data/lib/steem.rb

@@ -19,6 +19,7 @@ require 'steem/jsonrpc'
 require 'steem/block_api'
 require 'steem/formatter'
 require 'steem/broadcast'
+require 'steem/stream'
 
 module Steem
   def self.api_classes

data/lib/steem/api.rb

@@ -1,6 +1,6 @@
 module Steem
   # This ruby API works with
-  # {https://github.com/steemit/steem/releases steemd-0.19.4} and other AppBase
+  # {https://github.com/steemit/steem/releases steemd-0.19.10} and other AppBase
   # compatible upstreams.  To access different API namespaces, use the
   # following:
   #
@@ -36,7 +36,7 @@ module Steem
   #
   # Also see: {https://developers.steem.io/apidefinitions/ Complete API Definitions}
   class Api
-    attr_accessor :chain, :methods
+    attr_accessor :chain, :methods, :rpc_client
     
     # Use this for debugging naive thread handler.
     # DEFAULT_RPC_CLIENT_CLASS = RPC::HttpClient
@@ -57,12 +57,17 @@ module Steem
       @api_name.to_s.split('_').map(&:capitalize).join
     end
     
-    def self.jsonrpc=(jsonrpc)
-      @jsonrpc = jsonrpc
+    def self.jsonrpc=(jsonrpc, url = nil)
+      @jsonrpc ||= {}
+      @jsonrpc[url || jsonrpc.rpc_client.uri.to_s] = jsonrpc
     end
     
-    def self.jsonrpc
-      @jsonrpc
+    def self.jsonrpc(url = nil)
+      if @jsonrpc.size < 2 && url.nil?
+        @jsonrpc.values.first
+      else
+        @jsonrpc[url]
+      end
     end
     
     # Override this if you want to use your own client.
@@ -89,7 +94,7 @@ module Steem
         # have access to instance options until now.
         
         Api::jsonrpc = Jsonrpc.new(options)
-        @methods = Api::jsonrpc.get_api_methods
+        @methods = Api::jsonrpc(rpc_client.uri.to_s).get_api_methods
         
         unless !!@methods[@api_name]
           raise UnknownApiError, "#{@api_name} (known APIs: #{@methods.keys.join(' ')})"
@@ -115,19 +120,22 @@ module Steem
     end
   private
     # @private
-    def self.args_keys_to_s(rpc_method_name)
+    def args_keys_to_s(rpc_method_name)
       args = signature(rpc_method_name).args
       args_keys = JSON[args.to_json]
     end
     
     # @private
-    def self.signature(rpc_method_name)
+    def signature(rpc_method_name)
+      url = rpc_client.uri.to_s
+      
       @@signatures ||= {}
-      @@signatures[rpc_method_name] ||= jsonrpc.get_signature(method: rpc_method_name).result
+      @@signatures[url] ||= {}
+      @@signatures[url][rpc_method_name] ||= Api::jsonrpc(url).get_signature(method: rpc_method_name).result
     end
     
     # @private
-    def self.raise_error_response(rpc_method_name, rpc_args, response)
+    def raise_error_response(rpc_method_name, rpc_args, response)
       raise UnknownError, "#{rpc_method_name}: #{response}" if response.error.nil?
       
       error = response.error
@@ -153,9 +161,9 @@ module Steem
       when :condenser_api then args
       when :jsonrpc then args.first
       else
-        expected_args = Api::signature(rpc_method_name).args || []
+        expected_args = signature(rpc_method_name).args || []
         expected_args_key_string = if expected_args.size > 0
-          " (#{Api::args_keys_to_s(rpc_method_name)})"
+          " (#{args_keys_to_s(rpc_method_name)})"
         end
         expected_args_size = expected_args.size
         
@@ -178,11 +186,11 @@ module Steem
         args
       end
       
-      response = @rpc_client.rpc_execute(@api_name, m, rpc_args)
+      response = rpc_client.rpc_execute(@api_name, m, rpc_args)
       
       if defined?(response.error) && !!response.error
         if !!response.error.message
-          Api::raise_error_response rpc_method_name, rpc_args, response
+          raise_error_response rpc_method_name, rpc_args, response
         else
           raise Steem::ArgumentError, response.error.inspect
         end

data/lib/steem/base_error.rb

@@ -1,16 +1,16 @@
 module Steem
   class BaseError < StandardError
-    def initialize(error, cause = nil)
+    def initialize(error = nil, cause = nil)
       @error = error
       @cause = cause
     end
     
     def to_s
-      if !!@cause
-        JSON[error: @error, cause: @cause] rescue {error: @error, cause: @cause}.to_s
-      else
-        JSON[@error] rescue @error.to_s
-      end
+      detail = {}
+      detail[:error] = @error if !!@error
+      detail[:cause] = @cause if !!@cause
+      
+      JSON[detail] rescue detai.to_s
     end
     
     def self.build_error(error, context)
@@ -22,6 +22,10 @@ module Steem
         raise Steem::RemoteNodeError.new, error.message, build_backtrace(error)
       end
       
+      if error.message.include? 'Server error'
+        raise Steem::RemoteNodeError.new, error.message, build_backtrace(error)
+      end
+      
       if error.message.include? 'plugin not enabled'
         raise Steem::PluginNotEnabledError, error.message, build_backtrace(error)
       end
@@ -30,6 +34,10 @@ module Steem
         raise Steem::ArgumentError, "#{context}: #{error.message}", build_backtrace(error)
       end
       
+      if error.message.include? 'Invalid params'
+        raise Steem::ArgumentError, "#{context}: #{error.message}", build_backtrace(error)
+      end
+      
       if error.message.start_with? 'Bad Cast:'
         raise Steem::ArgumentError, "#{context}: #{error.message}", build_backtrace(error)
       end
@@ -110,6 +118,10 @@ module Steem
         raise Steem::MissingOtherAuthorityError, "#{context}: #{error.message}", build_backtrace(error)
       end
       
+      if error.message.include? 'Upstream response error'
+        raise Steem::UpstreamResponseError, "#{context}: #{error.message}", build_backtrace(error)
+      end
+      
       if error.message.include? 'Bad or missing upstream response'
         raise Steem::BadOrMissingUpstreamResponseError, "#{context}: #{error.message}", build_backtrace(error)
       end
@@ -122,6 +134,10 @@ module Steem
         raise Steem::InvalidAccountError, "#{context}: #{error.message}", build_backtrace(error)
       end
       
+      if error.message.include?('Method') && error.message.include?(' does not exist.')
+        raise Steem::UnknownMethodError, "#{context}: #{error.message}", build_backtrace(error)
+      end
+      
       if error.message.include? 'Invalid operation name'
         raise Steem::UnknownOperationError, "#{context}: #{error.message}", build_backtrace(error)
       end
@@ -162,9 +178,6 @@ module Steem
   
   class UnsupportedChainError < BaseError; end
   class ArgumentError < BaseError; end
-  class RemoteNodeError < BaseError; end
-  class RemoteDatabaseLockError < RemoteNodeError; end
-  class PluginNotEnabledError < RemoteNodeError; end
   class TypeError < BaseError; end
   class EmptyTransactionError < ArgumentError; end
   class InvalidAccountError < ArgumentError; end
@@ -186,12 +199,18 @@ module Steem
   class MissingOtherAuthorityError < MissingAuthorityError; end
   class IncorrectRequestIdError < BaseError; end
   class IncorrectResponseIdError < BaseError; end
-  class BadOrMissingUpstreamResponseError < BaseError; end
+  class RemoteNodeError < BaseError; end
+  class UpstreamResponseError < RemoteNodeError; end
+  class RemoteDatabaseLockError < UpstreamResponseError; end
+  class PluginNotEnabledError < UpstreamResponseError; end
+  class BadOrMissingUpstreamResponseError < UpstreamResponseError; end
   class TransactionIndexDisabledError < BaseError; end
   class NotAppBaseError < BaseError; end
   class UnknownApiError < BaseError; end
+  class UnknownMethodError < BaseError; end
   class UnknownOperationError < BaseError; end
   class JsonRpcBatchMaximumSizeExceededError < BaseError; end
   class TooManyTimeoutsError < BaseError; end
+  class TooManyRetriesError < BaseError; end
   class UnknownError < BaseError; end
 end

data/lib/steem/block_api.rb

@@ -12,11 +12,25 @@ module Steem
       super
     end
     
+    # Uses a batched requst on a range of block headers.
+    #
+    # @param options [Hash] The attributes to get a block range with.
+    # @option options [Range] :block_range starting on one block number and ending on an higher block number.
+    def get_block_headers(options = {block_range: (0..0)}, &block)
+      get_block_objects(options.merge(object: :block_header), block)
+    end
+    
     # Uses a batched requst on a range of blocks.
     #
     # @param options [Hash] The attributes to get a block range with.
     # @option options [Range] :block_range starting on one block number and ending on an higher block number.
     def get_blocks(options = {block_range: (0..0)}, &block)
+      get_block_objects(options.merge(object: :block), block)
+    end
+  private
+    def get_block_objects(options = {block_range: (0..0)}, block = nil)
+      object = options[:object]
+      object_method = "get_#{object}".to_sym
       block_range = options[:block_range] || (0..0)
       
       if (start = block_range.first) < 1
@@ -33,15 +47,21 @@ module Steem
         request_object = []
         
         for i in sub_range do
-          @rpc_client.put(self.class.api_name, :get_block, block_num: i, request_object: request_object)
+          @rpc_client.put(self.class.api_name, object_method, block_num: i, request_object: request_object)
         end
         
         if !!block
           index = 0
           @rpc_client.rpc_batch_execute(request_object: request_object) do |result, error, id|
             block_num = sub_range.to_a[index]
-            index = index += 1
-            yield(result.nil? ? nil : result.block, block_num)
+            index = index + 1
+            
+            case object
+            when :block_header
+              block.call(result.nil? ? nil : result[:header], block_num)
+            else
+              block.call(result.nil? ? nil : result[object], block_num)
+            end
           end
         else
           blocks = []

data/lib/steem/broadcast.rb

@@ -1136,7 +1136,7 @@ module Steem
       end
     end
     
-    # @privats
+    # @private
     def self.database_api(options)
       options[:database_api] ||= if !!options[:app_base]
         Steem::DatabaseApi.new(options)

data/lib/steem/mixins/retriable.rb

@@ -13,24 +13,12 @@ module Steem
       IncorrectResponseIdError, RemoteDatabaseLockError
     ]
     
-    # Expontential backoff.
-    #
-    # @private
-    def backoff
-      @backoff ||= 0.1
-      @backoff *= 2
-      @backoff = 0.1 if @backoff > MAX_BACKOFF
-      
-      sleep @backoff
-    end
-    
     def can_retry?(e = nil)
       @retry_count ||= 0
-      @first_retry_at ||= Time.now.utc
       
       return false if @retry_count >= MAX_RETRY_COUNT
       
-      @retry_count = if Time.now.utc - @first_retry_at > MAX_RETRY_ELAPSE
+      @retry_count = if retry_reset?
         @first_retry_at = nil
       else
         @retry_count + 1
@@ -45,5 +33,26 @@ module Steem
       
       can_retry
     end
+  private
+    # @private
+    def first_retry_at
+      @first_retry_at ||= Time.now.utc
+    end
+    
+    # @private
+    def retry_reset?
+      Time.now.utc - first_retry_at > MAX_RETRY_ELAPSE
+    end
+    
+    # Expontential backoff.
+    #
+    # @private
+    def backoff
+      @backoff ||= 0.1
+      @backoff *= 2
+      @backoff = 0.1 if @backoff > MAX_BACKOFF
+      
+      sleep @backoff
+    end
   end
 end

data/lib/steem/rpc/base_client.rb

@@ -12,8 +12,7 @@ module Steem
       MAX_TIMEOUT_BACKOFF = 30
       
       # @private
-      TIMEOUT_ERRORS = [Net::ReadTimeout, Errno::EBADF, Errno::ECONNREFUSED,
-        IOError]
+      TIMEOUT_ERRORS = [Net::ReadTimeout, Errno::EBADF, IOError]
       
       def initialize(options = {})
         @chain = options[:chain] || :steem

data/lib/steem/rpc/http_client.rb

@@ -17,7 +17,7 @@ module Steem
       # 
       # @private
       TIMEOUT_ERRORS = [Net::OpenTimeout, JSON::ParserError, Net::ReadTimeout,
-        Errno::EBADF, Errno::ECONNREFUSED, IOError]
+        Errno::EBADF, IOError, Errno::ENETDOWN]
       
       # @private
       POST_HEADERS = {

data/lib/steem/stream.rb

@@ -0,0 +1,377 @@
+module Steem
+  # Steem::Stream allows a live view of the STEEM blockchain.
+  # 
+  # Example streaming blocks:
+  # 
+  #     stream = Steem::Stream.new
+  #     
+  #     stream.blocks do |block, block_num|
+  #       puts "#{block_num} :: #{block.witness}"
+  #     end
+  # 
+  # Example streaming transactions:
+  # 
+  #     stream = Steem::Stream.new
+  #     
+  #     stream.transactions do |trx, trx_id, block_num|
+  #       puts "#{block_num} :: #{trx_id} :: operations: #{trx.operations.size}"
+  #     end
+  # 
+  # Example streaming operations:
+  # 
+  #     stream = Steem::Stream.new
+  #    
+  #     stream.operations do |op, trx_id, block_num|
+  #       puts "#{block_num} :: #{trx_id} :: #{op.type}: #{op.value.to_json}"
+  #     end
+  # 
+  # Allows streaming of block headers, full blocks, transactions, operations and
+  # virtual operations.
+  class Stream
+    attr_reader :database_api, :block_api, :account_history_api, :mode
+    
+    BLOCK_INTERVAL = 3
+    MAX_BACKOFF_BLOCK_INTERVAL = 30
+    MAX_RETRY_COUNT = 10
+    
+    VOP_TRX_ID = ('0' * 40).freeze
+
+    # @param options [Hash] additional options
+    # @option options [Steem::DatabaseApi] :database_api
+    # @option options [Steem::BlockApi] :block_api
+    # @option options [Steem::AccountHistoryApi || Steem::CondenserApi] :account_history_api
+    # @option options [Symbol] :mode we have the choice between
+    #   * :head the last block
+    #   * :irreversible the block that is confirmed by 2/3 of all block producers and is thus irreversible!
+    # @option options [Boolean] :no_warn do not generate warnings
+    def initialize(options = {mode: :irreversible})
+      @instance_options = options
+      @database_api = options[:database_api] || Steem::DatabaseApi.new(options)
+      @block_api = options[:block_api] || Steem::BlockApi.new(options)
+      @account_history_api = options[:account_history_api]
+      @mode = options[:mode] || :irreversible
+      @no_warn = !!options[:no_warn]
+    end
+    
+    # Use this method to stream block numbers.  This is significantly faster
+    # than requesting full blocks and even block headers.  Basically, the only
+    # thing this method does is call {Steem::Database#get_dynamic_global_properties} at 3 second
+    # intervals.
+    #
+    # @param options [Hash] additional options
+    # @option options [Integer] :at_block_num Starts the stream at the given block number.  Default: nil.
+    # @option options [Integer] :until_block_num Ends the stream at the given block number.  Default: nil.
+    def block_numbers(options = {}, &block)
+      block_objects(options.merge(object: :block_numbers), block)
+    end
+    
+    # Use this method to stream block headers.  This is quite a bit faster than
+    # requesting full blocks.
+    #
+    # @param options [Hash] additional options
+    # @option options [Integer] :at_block_num Starts the stream at the given block number.  Default: nil.
+    # @option options [Integer] :until_block_num Ends the stream at the given block number.  Default: nil.
+    def block_headers(options = {}, &block)
+      block_objects(options.merge(object: :block_headers), block)
+    end
+    
+    # Use this method to stream full blocks.
+    #
+    # @param options [Hash] additional options
+    # @option options [Integer] :at_block_num Starts the stream at the given block number.  Default: nil.
+    # @option options [Integer] :until_block_num Ends the stream at the given block number.  Default: nil.
+    def blocks(options = {}, &block)
+      block_objects(options.merge(object: :blocks), block)
+    end
+    
+    # Use this method to stream each transaction.
+    #
+    # @param options [Hash] additional options
+    # @option options [Integer] :at_block_num Starts the stream at the given block number.  Default: nil.
+    # @option options [Integer] :until_block_num Ends the stream at the given block number.  Default: nil.
+    def transactions(options = {}, &block)
+      blocks(options) do |block, block_num|
+        block.transactions.each_with_index do |transaction, index|
+          trx_id = block.transaction_ids[index]
+          
+          yield transaction, trx_id, block_num
+        end
+      end
+    end
+    
+    # Returns the latest operations from the blockchain.
+    #
+    #   stream = Steem::Stream.new
+    #   stream.operations do |op|
+    #     puts op.to_json
+    #   end
+    # 
+    # If symbol are passed to `types` option, then only that operation is
+    # returned.  Expected symbols are:
+    #
+    #   account_create_operation
+    #   account_create_with_delegation_operation
+    #   account_update_operation
+    #   account_witness_proxy_operation
+    #   account_witness_vote_operation
+    #   cancel_transfer_from_savings_operation
+    #   change_recovery_account_operation
+    #   claim_reward_balance_operation
+    #   comment_operation
+    #   comment_options_operation
+    #   convert_operation
+    #   custom_operation
+    #   custom_json_operation
+    #   decline_voting_rights_operation
+    #   delegate_vesting_shares_operation
+    #   delete_comment_operation
+    #   escrow_approve_operation
+    #   escrow_dispute_operation
+    #   escrow_release_operation
+    #   escrow_transfer_operation
+    #   feed_publish_operation
+    #   limit_order_cancel_operation
+    #   limit_order_create_operation
+    #   limit_order_create2_operation
+    #   pow_operation
+    #   pow2_operation
+    #   recover_account_operation
+    #   request_account_recovery_operation
+    #   set_withdraw_vesting_route_operation
+    #   transfer_operation
+    #   transfer_from_savings_operation
+    #   transfer_to_savings_operation
+    #   transfer_to_vesting_operation
+    #   vote_operation
+    #   withdraw_vesting_operation
+    #   witness_update_operation
+    #
+    # For example, to stream only votes:
+    #
+    #   stream = Steem::Stream.new
+    #   stream.operations(types: :vote_operation) do |vote|
+    #     puts vote.to_json
+    #   end
+    # 
+    # ... Or ...
+    # 
+    #   stream = Steem::Stream.new
+    #   stream.operations(:vote_operation) do |vote|
+    #     puts vote.to_json
+    #   end
+    #
+    # You can also stream virtual operations:
+    #
+    #   stream = Steem::Stream.new
+    #   stream.operations(types: :author_reward_operation, only_virtual: true) do |vop|
+    #     v = vop.value
+    #     puts "#{v.author} got paid for #{v.permlink}: #{[v.sbd_payout, v.steem_payout, v.vesting_payout]}"
+    #   end
+    #
+    # ... or multiple virtual operation types;
+    #
+    #   stream = Steem::Stream.new
+    #   stream.operations(types: [:producer_reward_operation, :author_reward_operation], only_virtual: true) do |vop|
+    #     puts vop.to_json
+    #   end
+    #
+    # ... or all types, including virtual operation types from the head block number:
+    #
+    #   stream = Steem::Stream.new(mode: :head)
+    #   stream.operations(include_virtual: true) do |op|
+    #     puts op.to_json
+    #   end
+    #
+    # Expected virtual operation types:
+    #
+    #   producer_reward_operation
+    #   author_reward_operation
+    #   curation_reward_operation
+    #   fill_convert_request_operation
+    #   fill_order_operation
+    #   fill_vesting_withdraw_operation
+    #   interest_operation
+    #   shutdown_witness_operation
+    #
+    # @param args [Symbol || Array<Symbol> || Hash] the type(s) of operation or hash of expanded options, optional.
+    # @option args [Integer] :at_block_num Starts the stream at the given block number.  Default: nil.
+    # @option args [Integer] :until_block_num Ends the stream at the given block number.  Default: nil.
+    # @option args [Symbol || Array<Symbol>] :types the type(s) of operation, optional.
+    # @option args [Boolean] :only_virtual Only stream virtual options.  Setting this true will improve performance because the stream only needs block numbers to then retrieve virtual operations.  Default: false.
+    # @option args [Boolean] :include_virtual Also stream virtual options.  Setting this true will impact performance.  Default: false.
+    # @param block the block to execute for each result.  Yields: |op, trx_id, block_num|
+    def operations(*args, &block)
+      options = {}
+      types = []
+      only_virtual = false
+      include_virtual = false
+      last_block_num = nil
+      
+      case args.first
+      when Hash
+        options = args.first
+        types = transform_types(options[:types])
+        only_virtual = !!options[:only_virtual] || false
+        include_virtual = !!options[:include_virtual] || only_virtual || false
+      when Symbol, Array then types = transform_types(args)
+      end
+      
+      if only_virtual
+        block_numbers(options) do |block_num|
+          get_virtual_ops(types, block_num, block)
+        end
+      else
+        transactions(options) do |transaction, trx_id, block_num|
+          transaction.operations.each do |op|
+            yield op, trx_id, block_num if types.none? || types.include?(op.type)
+            
+            next unless last_block_num != block_num
+            
+            last_block_num = block_num
+            
+            get_virtual_ops(types, block_num, block) if include_virtual
+          end
+        end
+      end
+    end
+    
+    def account_history_api
+      @account_history_api ||= begin
+        Steem::AccountHistoryApi.new(@instance_options)
+      rescue Steem::UnknownApiError => e
+        warn "#{e.inspect}, falling back to Steem::CondenserApi." unless @no_warn
+        Steem::CondenserApi.new(@instance_options)
+      end
+    end
+  private
+    # @private
+    def block_objects(options = {}, block)
+      object = options[:object]
+      object_method = "get_#{object}".to_sym
+      block_interval = BLOCK_INTERVAL
+      
+      at_block_num, until_block_num = if !!block_range = options[:block_range]
+        [block_range.first, block_range.last]
+      else
+        [options[:at_block_num], options[:until_block_num]]
+      end
+      
+      loop do
+        break if !!until_block_num && !!at_block_num && until_block_num < at_block_num
+        
+        database_api.get_dynamic_global_properties do |properties|
+          current_block_num = find_block_number(properties)
+          current_block_num = [current_block_num, until_block_num].compact.min
+          at_block_num ||= current_block_num
+          
+          if current_block_num >= at_block_num
+            range = at_block_num..current_block_num
+            
+            if object == :block_numbers
+              range.each do |n|
+                block.call n
+                block_interval = BLOCK_INTERVAL
+              end
+            else
+              block_api.send(object_method, block_range: range) do |b, n|
+                block.call b, n
+                block_interval = BLOCK_INTERVAL
+              end
+            end
+            
+            at_block_num = range.max + 1
+          else
+            # The stream has stalled, so let's back off and let the node sync
+            # up.  We'll catch up with a bigger batch in the next cycle.
+            block_interval = [block_interval * 2, MAX_BACKOFF_BLOCK_INTERVAL].min
+          end
+        end
+        
+        sleep block_interval
+      end
+    end
+    
+    # @private
+    def find_block_number(properties)
+      block_num = case mode
+      when :head then properties.head_block_number
+      when :irreversible then properties.last_irreversible_block_num
+      else; raise Steem::ArgumentError, "Unknown mode: #{mode}"
+      end
+      
+      block_num
+    end
+    
+    # @private
+    def transform_types(types)
+      [types].compact.flatten.map do |type|
+        type = type.to_s
+        
+        unless type.end_with? '_operation'
+          warn "Op type #{type} is deprecated.  Use #{type}_operation instead." unless @no_warn
+          type += '_operation'
+        end
+        
+        type
+      end
+    end
+    
+    # @private
+    def get_virtual_ops(types, block_num, block)
+      retries = 0
+      
+      loop do
+        get_ops_in_block_options = case account_history_api
+        when Steem::CondenserApi
+          [block_num, true]
+        when Steem::AccountHistoryApi
+          {
+            block_num: block_num,
+            only_virtual: true
+          }
+        end
+        
+        response = account_history_api.get_ops_in_block(*get_ops_in_block_options)
+        result = response.result
+        
+        if result.nil?
+          if retries < MAX_RETRY_COUNT
+            warn "Retrying get_ops_in_block on block #{block_num}" unless @no_warn
+            retries = retries + 1
+            sleep 9
+            redo
+          else
+            raise TooManyRetriesError, "unable to get valid result while finding virtual operations for block: #{block_num}"
+          end
+        end
+        
+        ops = case account_history_api
+        when Steem::CondenserApi
+          result.map do |trx|
+            op = {type: trx.op[0] + '_operation', value: trx.op[1]}
+            op = Hashie::Mash.new(op)
+          end
+        when Steem::AccountHistoryApi then result.ops.map { |trx| trx.op }
+        end
+        
+        if ops.empty?
+          if retries < MAX_RETRY_COUNT
+            sleep 3
+            retries = retries + 1
+            redo
+          else
+            raise TooManyRetriesError, "unable to find virtual operations for block: #{block_num}"
+          end
+        end
+        
+        ops.each do |op|
+          next if types.any? && !types.include?(op.type)
+          
+          block.call op, VOP_TRX_ID, block_num
+        end
+        
+        break
+      end
+    end
+  end
+end

data/lib/steem/version.rb

@@ -1,4 +1,4 @@
 module Steem
-  VERSION = '0.9.1'
+  VERSION = '0.9.2'
   AGENT_ID = "steem-ruby/#{VERSION}"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: steem-ruby
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.2
 platform: ruby
 authors:
 - Anthony Martin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-28 00:00:00.000000000 Z
+date: 2018-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -358,6 +358,7 @@ files:
 - lib/steem/rpc/base_client.rb
 - lib/steem/rpc/http_client.rb
 - lib/steem/rpc/thread_safe_http_client.rb
+- lib/steem/stream.rb
 - lib/steem/transaction_builder.rb
 - lib/steem/type/amount.rb
 - lib/steem/type/base_type.rb