radiator 0.3.0dev3 → 0.3.0dev4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 768d130d67520d82f31a14a1ce99236e5b17688b
-  data.tar.gz: 6da4c36f9d54a3de922ac26ec4ed8f70aee0def1
+  metadata.gz: 4b709aac2407041d491d0d08abb766ce404dc0cf
+  data.tar.gz: d007d6f2d7d20b096450f05c196790da6374b76e
 SHA512:
-  metadata.gz: 2c8507502286b7d4f07c6705bbbd8fdd0bfc04d5a6187747b1573a9714bcdf58ecf5dcb12e615fa2eb3218ef91a1b934470c70c0da742f0fdb6fdb448697a12f
-  data.tar.gz: 86d7090a11dbd508db65305bccf71d3c6c1533a962cbdc66c8b4c9b42105470341274d81010fd8aed902e43085fbae3fa7f06fbd9561cfd6800c9f986a39975c
+  metadata.gz: 073339bb35cbffff34c4adedfef58534b0827aac9757f903f94f7a2ae56adb27ef8468dc85271895a725a70eb425015faa3609424fc266d88581e81f0acd023e
+  data.tar.gz: a7cae3da19c410b1fceb01b717de6b4199dea2f7ebbd8018b7f1ef21c07e6da58439c0016508d60c7d1b9c6f8bd9f488b66179519018f7c40afc50a6e2a5f581

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    radiator (0.3.0dev3)
+    radiator (0.3.0dev4)
       bitcoin-ruby (= 0.0.11)
       ffi (= 1.9.18)
       hashie (~> 3.5, >= 3.5.5)

data/lib/radiator/api.rb

@@ -161,6 +161,7 @@ module Radiator
     # @option options [Array<String>] :failover_urls An array that contains one or more full nodes to fall back on.  Default from DEFAULT_FAILOVER_URLS.
     # @option options [Logger] :logger An instance of `Logger` to send debug messages to.
     # @option options [Boolean] :recover_transactions_on_error Have Radiator try to recover transactions that are accepted but could not be confirmed due to an error like network timeout.  Default: `true`
+    # @option options [Integer] :max_requests Maximum number of requests on a connection before it is considered expired and automatically closed.
     def initialize(options = {})
       @user = options[:user]
       @password = options[:password]
@@ -171,6 +172,7 @@ module Radiator
       @debug = !!options[:debug]
       @logger = options[:logger] || Radiator.logger
       @hashie_logger = options[:hashie_logger] || Logger.new(nil)
+      @max_requests = options[:max_requests] || 30
       
       unless @hashie_logger.respond_to? :warn
         @hashie_logger = Logger.new(@hashie_logger)
@@ -304,8 +306,11 @@ module Radiator
     # Stops the persistant http connections.
     #
     def shutdown
+      @http_id = nil
       @http.shutdown if !!@http && defined?(@http.shutdown)
       @http = nil
+      @api.shutdown if !!@api && @api != self
+      @api = nil
     end
     
     # @private
@@ -403,6 +408,9 @@ module Radiator
           @logger.warn "Socket Error (#{e.message}), retrying ..."
         rescue JSON::ParserError => e
           @logger.warn "JSON Parse Error (#{e.message}), retrying ..."
+          response = nil
+        rescue ApiError => e
+          @logger.warn "ApiError (#{e.message}), retrying ..."
         rescue => e
           @logger.warn "Unknown exception from request, retrying ..."
           ap e if defined? ap
@@ -444,13 +452,19 @@ module Radiator
       @uri ||= URI.parse(@url)
     end
     
+    def http_id
+      @http_id ||= "radiator-#{Radiator::VERSION}-#{api_name}-#{SecureRandom.uuid}"
+    end
+    
     def http
-      @http_id ||= "radiator-#{Radiator::VERSION}-#{self.class.name.downcase}"
-      @http ||= Net::HTTP::Persistent.new(@http_id).tap do |http|
-        http.retry_change_requests = true
-        http.max_requests = 30
+      @http ||= Net::HTTP::Persistent.new(http_id).tap do |http|
+        idempotent = api_name != :network_broadcast_api
+        http.keep_alive = 30
         http.read_timeout = 10
         http.open_timeout = 10
+        http.idle_timeout = idempotent ? 10 : nil
+        http.max_requests = @max_requests
+        http.retry_change_requests = idempotent
       end
     end
     
@@ -488,7 +502,7 @@ module Radiator
       # but we also give up once the block time is before the `after` argument.
       
       api.get_blocks(block_range) do |block, block_num|
-        raise "Race condition detected at: #{block_num}" if block.nil?
+        raise ApiError, "Race condition detected on remote node at: #{block_num}" if block.nil?
         
         timestamp = Time.parse(block.timestamp + 'Z')
         break if timestamp < after
@@ -496,9 +510,6 @@ module Radiator
         block.transactions.each_with_index do |tx, index|
           next unless ((tx['signatures'] || []) & signatures).any?
           
-          puts "Found matching signatures in #{(Time.now.utc - now)} seconds: #{signatures}"
-          ap tx
-          
           return {
             id: rpc_id,
             result: {
@@ -510,8 +521,6 @@ module Radiator
           }
         end
       end
-      
-      puts "Took #{(Time.now.utc - now)} seconds to scan for signatures."
     end
     
     def reset_failover

data/lib/radiator/stream.rb

@@ -38,33 +38,42 @@ module Radiator
     # If symbol are passed, then only that operation is returned.  Expected
     # symbols are:
     #
-    #   transfer_to_vesting
-    #   withdraw_vesting
-    #   interest
-    #   transfer
-    #   liquidity_reward
-    #   author_reward
-    #   curation_reward
-    #   transfer_to_savings
-    #   transfer_from_savings
+    #   account_create
+    #   account_create_with_delegation
+    #   account_update
+    #   account_witness_proxy
+    #   account_witness_vote
     #   cancel_transfer_from_savings
-    #   escrow_transfer
+    #   change_recovery_account
+    #   claim_reward_balance
+    #   comment
+    #   comment_options
+    #   convert
+    #   custom
+    #   custom_json
+    #   decline_voting_rights
+    #   delegate_vesting_shares
+    #   delete_comment
     #   escrow_approve
     #   escrow_dispute
     #   escrow_release
-    #   comment
-    #   limit_order_create
+    #   escrow_transfer
+    #   feed_publish
     #   limit_order_cancel
-    #   fill_convert_request
-    #   fill_order
+    #   limit_order_create
+    #   limit_order_create2
+    #   pow
+    #   pow2
+    #   recover_account
+    #   request_account_recovery
+    #   set_withdraw_vesting_route
+    #   transfer
+    #   transfer_from_savings
+    #   transfer_to_savings
+    #   transfer_to_vesting
     #   vote
-    #   account_witness_vote
-    #   account_witness_proxy
-    #   account_create
-    #   account_update
+    #   withdraw_vesting
     #   witness_update
-    #   pow
-    #   custom
     #
     # For example, to stream only votes:
     #
@@ -73,30 +82,93 @@ module Radiator
     #     puts vote.to_json
     #   end
     #
+    # You can also stream virtual operations:
+    #
+    #   stream = Radiator::Stream.new
+    #   stream.operations(:author_reward) do |vop|
+    #       puts "#{vop.author} got paid for #{vop.permlink}: #{[vop.sbd_payout, vop.steem_payout, vop.vesting_payout]}"
+    #   end
+    #
+    # ... or multiple virtual operation types;
+    #
+    #   stream = Radiator::Stream.new
+    #   stream.operations([:produer_reward, :author_reward]) do |vop|
+    #     puts vop
+    #   end
+    #
+    # ... or all types, inluding virtual operation types;
+    #
+    #   stream = Radiator::Stream.new
+    #   stream.operations(nil, nil, :head, include_virtual: true) do |vop|
+    #     puts vop
+    #   end
+    #
+    # Expected virtual operation types:
+    #
+    #   author_reward
+    #   curation_reward
+    #   fill_convert_request
+    #   fill_order
+    #   fill_vesting_withdraw
+    #   interest
+    #   shutdown_witness
+    #
     # @param type [symbol || Array<symbol>] the type(s) of operation, optional.
     # @param start starting block
     # @param mode we have the choice between
     #   * :head the last block
     #   * :irreversible the block that is confirmed by 2/3 of all block producers and is thus irreversible!
     # @param block the block to execute for each result, optional.
+    # @param options [Hash] additional options
+    # @option options [Boollean] :include_virtual Also stream virtual options.  Setting this true will impact performance.  Default: false.
     # @return [Hash]
-    def operations(type = nil, start = nil, mode = :irreversible, &block)
-      transactions(start, mode) do |transaction|
+    def operations(type = nil, start = nil, mode = :irreversible, options = {include_virtual: false}, &block)
+      type = [type].flatten.compact.map(&:to_sym)
+      include_virtual = !!options[:include_virtual]
+      
+      if virtual_op_type?(type)
+        include_virtual = true
+      end
+      
+      latest_block_number = -1
+      
+      transactions(start, mode) do |transaction, trx_id, block_number|
+        virtual_ops_collected = latest_block_number == block_number
+        latest_block_number = block_number
+        
         ops = transaction.operations.map do |t, op|
           t = t.to_sym
-          if type == t
+          if type.size == 1 && type.first == t
             op
-          elsif type.nil? || [type].flatten.include?(t)
+          elsif type.none? || type.include?(t)
             {t => op}
           end
         end.compact
         
+        if include_virtual && !virtual_ops_collected
+          api.get_ops_in_block(block_number, true) do |vops|
+            vops.each do |vtx|
+              next unless defined? vtx.op
+              
+              t = vtx.op.first.to_sym
+              op = vtx.op.last
+              if type.size == 1 && type.first == t
+                ops << op
+              elsif type.none? || type.include?(t)
+                ops << {t => op}
+              end
+            end
+          end
+          
+          virtual_ops_collected = true
+        end
+        
         next if ops.none?
         
         return ops unless !!block
         
         ops.each do |op|
-          yield op
+          yield op, trx_id, block_number
         end
       end
     end
@@ -115,7 +187,7 @@ module Radiator
     # @param block the block to execute for each result, optional.
     # @return [Hash]
     def transactions(start = nil, mode = :irreversible, &block)
-      blocks(start, mode) do |b|
+      blocks(start, mode) do |b, block_number|
         next if (_transactions = b.transactions).nil?
         return _transactions unless !!block
         
@@ -124,7 +196,7 @@ module Radiator
             b['transaction_ids'][index]
           end
           
-          yield transaction, trx_id
+          yield transaction, trx_id, block_number
         end
       end
     end
@@ -145,44 +217,72 @@ module Radiator
     # @return [Hash]
     def blocks(start = nil, mode = :irreversible, max_blocks_per_node = MAX_BLOCKS_PER_NODE, &block)
       counter = 0
-      
-      if start.nil?
-        properties = api.get_dynamic_global_properties.result
-        start = case mode.to_sym
-        when :head then properties.head_block_number
-        when :irreversible then properties.last_irreversible_block_num
-        else; raise StreamError, '"mode" has to be "head" or "irreversible"'
-        end
-      end
+      latest_block_number = -1
+      @api_options[:max_requests] = [max_blocks_per_node * 2, @api_options[:max_requests].to_i].max
       
       loop do
-        properties = api.get_dynamic_global_properties.result
-        
-        head_block = case mode.to_sym
-        when :head then properties.head_block_number
-        when :irreversible then properties.last_irreversible_block_num
-        else; raise StreamError, '"mode" has to be "head" or "irreversible"'
-        end
-        
-        [*(start..(head_block))].each do |n|
-          if (counter += 1) > max_blocks_per_node
-            shutdown
-            counter = 0
+        catch :sequence do; begin
+          head_block = api.get_dynamic_global_properties do |properties|
+            if properties.head_block_number.nil?
+              # This can happen if a reverse proxy is acting up.
+              @logger.warn "Bad block sequence after height: #{latest_block_number}"
+              throw :sequence
+            end
+                
+            case mode.to_sym
+            when :head then properties.head_block_number
+            when :irreversible then properties.last_irreversible_block_num
+            else; raise StreamError, '"mode" has to be "head" or "irreversible"'
+            end
+          end
+            
+          if head_block == latest_block_number
+            # This can when there's a delay in block production.
+            sleep 0.5
+            throw :sequence
+          elsif head_block < latest_block_number
+            # This can happen if a reverse proxy is acting up.
+            @logger.warn "Invalid block sequence at height: #{head_block}"
+            sleep 0.5
+            throw :sequence
           end
+          
+          start ||= head_block
+          range = (start..head_block)
+          
+          if range.size > 400
+            # When the range is 400 blocks, the stream will be behind by about
+            # 20 minutes.  Time to warn.
+            @logger.warn "Stream behind by #{range.size} blocks (about #{(range.size * 3) / 60.0} minutes)."
+          end
+          
+          [*range].each do |n|
+            if (counter += 1) > max_blocks_per_node
+              shutdown
+              counter = 0
+            end
 
-          response = api.send(:get_block, n)
-          raise StreamError, JSON[response.error] if !!response.error
-          result = response.result
-        
-          if !!block
-            yield result, n
-          else
-            return result, n
+            api.get_block(n) do |current_block, error|
+              if !!error
+                @logger.warn "Node responded with: #{error.message || 'unknown error'}"
+                ap error
+                throw :sequence
+              end
+              
+              latest_block_number = n
+              return current_block, n if block.nil?
+              yield current_block, n
+            end
+            
+            start = head_block + 1
+            sleep 3 / range.size
           end
-        end
-        
-        start = head_block + 1
-        sleep 3
+        rescue StreamError; raise
+        rescue => e
+          @logger.warn "Unknown streaming error: #{e.inspect}, retrying ...  "
+          ap e
+          redo
+        end; end
       end
     end
     
@@ -241,10 +341,6 @@ module Radiator
       end
     end
   private
-    def api
-      @api ||= Api.new(@api_options)
-    end
-    
     def method_missing(m, *args, &block)
       super unless respond_to_missing?(m)
       
@@ -292,5 +388,11 @@ module Radiator
       @timeout = INITIAL_TIMEOUT if @timeout > MAX_TIMEOUT
       @timeout
     end
+    
+    def virtual_op_type?(type)
+      type = [type].flatten.compact.map(&:to_sym)
+      
+      (Radiator::OperationTypes::TYPES.keys && type).any?
+    end
   end
 end

data/lib/radiator/transaction.rb

@@ -106,6 +106,11 @@ module Radiator
       # The expiration allows for transactions to expire if they are not
       # included into a block by that time.  Always update it to the current
       # time + EXPIRE_IN_SECS.
+      #
+      # Note, as of #1215, expiration exactly 'now' will be rejected:
+      # https://github.com/steemit/steem/blob/57451b80d2cf480dcce9b399e48e56aa7af1d818/libraries/chain/database.cpp#L2870
+      # https://github.com/steemit/steem/issues/1215
+      
       @expiration = Time.parse(@properties.time + 'Z') + EXPIRE_IN_SECS
       
       self

data/lib/radiator/version.rb

@@ -1,3 +1,3 @@
 module Radiator
-  VERSION = '0.3.0dev3'
+  VERSION = '0.3.0dev4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: radiator
 version: !ruby/object:Gem::Version
-  version: 0.3.0dev3
+  version: 0.3.0dev4
 platform: ruby
 authors:
 - Anthony Martin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-30 00:00:00.000000000 Z
+date: 2017-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -410,7 +410,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: STEEM RPC Ruby Client