hive-ruby 1.0.0.pre.1

data/lib/hive/rpc/http_client.rb

@@ -0,0 +1,143 @@
+module Hive
+  module RPC
+    # {HttpClient} is intended for single-threaded applications.  For
+    # multi-threaded apps, use {ThreadSafeHttpClient}.
+    class HttpClient < BaseClient
+      # Timeouts are lower level errors, related in that retrying them is
+      # trivial, unlike, for example TransactionExpiredError, that *requires*
+      # the client to do something before retrying.
+      # 
+      # These situations are hopefully momentary interruptions or rate limiting
+      # but they might indicate a bigger problem with the node, so they are not
+      # retried forever, only up to MAX_TIMEOUT_RETRY_COUNT and then we give up.
+      # 
+      # *Note:* {JSON::ParserError} is included in this list because under
+      # certain timeout conditions, a web server may respond with a generic
+      # http status code of 200 and HTML page.
+      # 
+      # @private
+      TIMEOUT_ERRORS = [Net::OpenTimeout, JSON::ParserError, Net::ReadTimeout,
+        Errno::EBADF, IOError, Errno::ENETDOWN, Hive::RemoteDatabaseLockError]
+      
+      # @private
+      POST_HEADERS = {
+        'Content-Type' => 'application/json; charset=utf-8',
+        'User-Agent' => Hive::AGENT_ID
+      }
+      
+      JSON_RPC_BATCH_SIZE_MAXIMUM = 50
+      
+      def http
+        @http ||= Net::HTTP.new(uri.host, uri.port).tap do |http|
+          http.use_ssl = true if uri.to_s =~ /^https/i
+          http.keep_alive_timeout = 150 # seconds
+          
+          # WARNING This method opens a serious security hole. Never use this
+          # method in production code.
+          # http.set_debug_output(STDOUT) if !!ENV['DEBUG']
+        end
+      end
+      
+      def http_post
+        @http_post ||= Net::HTTP::Post.new(uri.request_uri, POST_HEADERS)
+      end
+      
+      def http_request(request)
+        http.request(request)
+      end
+      
+      # This is the main method used by API instances to actually fetch data
+      # from the remote node.  It abstracts the api namespace, method name, and
+      # parameters so that the API instance can be decoupled from the protocol.
+      # 
+      # @param api_name [String] API namespace of the method being called.
+      # @param api_method [String] API method name being called.
+      # @param options [Hash] options
+      # @option options [Object] :request_object Hash or Array to become json in request body.
+      def rpc_execute(api_name = @api_name, api_method = nil, options = {}, &block)
+        reset_timeout
+        
+        catch :tota_cera_pila do; begin
+          request = http_post
+          
+          request_object = if !!api_name && !!api_method
+            put(api_name, api_method, options)
+          elsif !!options && defined?(options.delete)
+            options.delete(:request_object)
+          end
+          
+          if request_object.size > JSON_RPC_BATCH_SIZE_MAXIMUM
+            raise JsonRpcBatchMaximumSizeExceededError, "Maximum json-rpc-batch is #{JSON_RPC_BATCH_SIZE_MAXIMUM} elements."
+          end
+          
+          request.body = if request_object.class == Hash
+            request_object
+          elsif request_object.size == 1
+            request_object.first
+          else
+            request_object
+          end.to_json
+          
+          response = catch :http_request do; begin; http_request(request)
+          rescue *TIMEOUT_ERRORS => e
+            throw retry_timeout(:http_request, e)
+          end; end
+          
+          if response.nil?
+            throw retry_timeout(:tota_cera_pila, 'response was nil')
+          end
+          
+          case response.code
+          when '200'
+            response = catch :parse_json do; begin; JSON[response.body]
+            rescue *TIMEOUT_ERRORS => e
+              throw retry_timeout(:parse_json, e)
+            end; end
+            
+            response = case response
+            when Hash
+              Hashie::Mash.new(response).tap do |r|
+                evaluate_id(request: request_object.first, response: r, api_method: api_method)
+              end
+            when Array
+              Hashie::Array.new(response).tap do |r|
+                request_object.each_with_index do |req, index|
+                  evaluate_id(request: req, response: r[index], api_method: api_method)
+                end
+              end
+            else; response
+            end
+            
+            [response].flatten.each_with_index do |r, i|
+              if defined?(r.error) && !!r.error
+                if !!r.error.message
+                  begin
+                    rpc_method_name = "#{api_name}.#{api_method}"
+                    rpc_args = [request_object].flatten[i]
+                    raise_error_response rpc_method_name, rpc_args, r
+                  rescue *TIMEOUT_ERRORS => e
+                    throw retry_timeout(:tota_cera_pila, e)
+                  end
+                else
+                  raise Hive::ArgumentError, r.error.inspect
+                end
+              end
+            end
+            
+            yield_response response, &block
+          when '504' # Gateway Timeout
+            throw retry_timeout(:tota_cera_pila, response.body)
+          when '502' # Bad Gateway
+            throw retry_timeout(:tota_cera_pila, response.body)
+          else
+            raise UnknownError, "#{api_name}.#{api_method}: #{response.body}"
+          end
+        end; end
+      end
+      
+      def rpc_batch_execute(options = {}, &block)
+        yield_response rpc_execute(nil, nil, options), &block
+      end
+    end
+  end
+end

data/lib/hive/rpc/thread_safe_http_client.rb

@@ -0,0 +1,35 @@
+module Hive
+  module RPC
+    # {ThreadSafeHttpClient} is the default RPC Client used by `hive-ruby.`
+    # It's perfect for simple requests.  But for higher performance, it's better
+    # to override {HttpClient} and implement something other than {Net::HTTP}.
+    # 
+    # It performs http requests in a {Mutex} critical section because {Net::HTTP}
+    # is not thread safe.  This is the very minimum level thread safety
+    # available.
+    class ThreadSafeHttpClient < HttpClient
+      SEMAPHORE = Mutex.new.freeze
+      
+      # Same as #{HttpClient#http_post}, but scoped to each thread so it is
+      # thread safe.
+      def http_post
+        thread = Thread.current
+        http_post = thread.thread_variable_get(:http_post)
+        http_post ||= Net::HTTP::Post.new(uri.request_uri, POST_HEADERS)
+        thread.thread_variable_set(:http_post, http_post)
+      end
+      
+      def http_request(request); SEMAPHORE.synchronize{super}; end
+      
+      # Same as #{BaseClient#rpc_id}, auto-increment, but scoped to each thread
+      # so it is thread safe.
+      def rpc_id
+        thread = Thread.current
+        rpc_id = thread.thread_variable_get(:rpc_id)
+        rpc_id ||= 0
+        rpc_id += 1
+        thread.thread_variable_set(:rpc_id, rpc_id)
+      end
+    end
+  end
+end

data/lib/hive/stream.rb

@@ -0,0 +1,385 @@
+module Hive
+  # Hive::Stream allows a live view of the HIVE blockchain.
+  # 
+  # Example streaming blocks:
+  # 
+  #     stream = Hive::Stream.new
+  #     
+  #     stream.blocks do |block, block_num|
+  #       puts "#{block_num} :: #{block.witness}"
+  #     end
+  # 
+  # Example streaming transactions:
+  # 
+  #     stream = Hive::Stream.new
+  #     
+  #     stream.transactions do |trx, trx_id, block_num|
+  #       puts "#{block_num} :: #{trx_id} :: operations: #{trx.operations.size}"
+  #     end
+  # 
+  # Example streaming operations:
+  # 
+  #     stream = Hive::Stream.new
+  #    
+  #     stream.operations do |op, trx_id, block_num|
+  #       puts "#{block_num} :: #{trx_id} :: #{op.type}: #{op.value.to_json}"
+  #     end
+  # 
+  # Allows streaming of block headers, full blocks, transactions, operations and
+  # virtual operations.
+  class Stream
+    attr_reader :database_api, :block_api, :account_history_api, :mode
+    
+    BLOCK_INTERVAL = 3
+    MAX_BACKOFF_BLOCK_INTERVAL = 30
+    MAX_RETRY_COUNT = 10
+    
+    VOP_TRX_ID = ('0' * 40).freeze
+
+    # @param options [Hash] additional options
+    # @option options [Hive::DatabaseApi] :database_api
+    # @option options [Hive::BlockApi] :block_api
+    # @option options [Hive::AccountHistoryApi || Hive::CondenserApi] :account_history_api
+    # @option options [Symbol] :mode we have the choice between
+    #   * :head the last block
+    #   * :irreversible the block that is confirmed by 2/3 of all block producers and is thus irreversible!
+    # @option options [Boolean] :no_warn do not generate warnings
+    def initialize(options = {mode: :irreversible})
+      @instance_options = options
+      @database_api = options[:database_api] || Hive::DatabaseApi.new(options)
+      @block_api = options[:block_api] || Hive::BlockApi.new(options)
+      @account_history_api = options[:account_history_api]
+      @mode = options[:mode] || :irreversible
+      @no_warn = !!options[:no_warn]
+    end
+    
+    # Use this method to stream block numbers.  This is significantly faster
+    # than requesting full blocks and even block headers.  Basically, the only
+    # thing this method does is call {Hive::Database#get_dynamic_global_properties} at 3 second
+    # intervals.
+    #
+    # @param options [Hash] additional options
+    # @option options [Integer] :at_block_num Starts the stream at the given block number.  Default: nil.
+    # @option options [Integer] :until_block_num Ends the stream at the given block number.  Default: nil.
+    def block_numbers(options = {}, &block)
+      block_objects(options.merge(object: :block_numbers), block)
+    end
+    
+    # Use this method to stream block headers.  This is quite a bit faster than
+    # requesting full blocks.
+    #
+    # @param options [Hash] additional options
+    # @option options [Integer] :at_block_num Starts the stream at the given block number.  Default: nil.
+    # @option options [Integer] :until_block_num Ends the stream at the given block number.  Default: nil.
+    def block_headers(options = {}, &block)
+      block_objects(options.merge(object: :block_headers), block)
+    end
+    
+    # Use this method to stream full blocks.
+    #
+    # @param options [Hash] additional options
+    # @option options [Integer] :at_block_num Starts the stream at the given block number.  Default: nil.
+    # @option options [Integer] :until_block_num Ends the stream at the given block number.  Default: nil.
+    def blocks(options = {}, &block)
+      block_objects(options.merge(object: :blocks), block)
+    end
+    
+    # Use this method to stream each transaction.
+    #
+    # @param options [Hash] additional options
+    # @option options [Integer] :at_block_num Starts the stream at the given block number.  Default: nil.
+    # @option options [Integer] :until_block_num Ends the stream at the given block number.  Default: nil.
+    def transactions(options = {}, &block)
+      blocks(options) do |block, block_num|
+        if block.nil?
+          warn "Batch missing block_num: #{block_num}, retrying ..."
+          
+          block = block_api.get_block(block_num: block_num) do |result|
+            result.block
+          end
+        end
+        
+        block.transactions.each_with_index do |transaction, index|
+          trx_id = block.transaction_ids[index]
+          
+          yield transaction, trx_id, block_num
+        end
+      end
+    end
+    
+    # Returns the latest operations from the blockchain.
+    #
+    #   stream = Hive::Stream.new
+    #   stream.operations do |op|
+    #     puts op.to_json
+    #   end
+    # 
+    # If symbol are passed to `types` option, then only that operation is
+    # returned.  Expected symbols are:
+    #
+    #   account_create_operation
+    #   account_create_with_delegation_operation
+    #   account_update_operation
+    #   account_witness_proxy_operation
+    #   account_witness_vote_operation
+    #   cancel_transfer_from_savings_operation
+    #   change_recovery_account_operation
+    #   claim_reward_balance_operation
+    #   comment_operation
+    #   comment_options_operation
+    #   convert_operation
+    #   custom_operation
+    #   custom_json_operation
+    #   decline_voting_rights_operation
+    #   delegate_vesting_shares_operation
+    #   delete_comment_operation
+    #   escrow_approve_operation
+    #   escrow_dispute_operation
+    #   escrow_release_operation
+    #   escrow_transfer_operation
+    #   feed_publish_operation
+    #   limit_order_cancel_operation
+    #   limit_order_create_operation
+    #   limit_order_create2_operation
+    #   pow_operation
+    #   pow2_operation
+    #   recover_account_operation
+    #   request_account_recovery_operation
+    #   set_withdraw_vesting_route_operation
+    #   transfer_operation
+    #   transfer_from_savings_operation
+    #   transfer_to_savings_operation
+    #   transfer_to_vesting_operation
+    #   vote_operation
+    #   withdraw_vesting_operation
+    #   witness_update_operation
+    #
+    # For example, to stream only votes:
+    #
+    #   stream = Hive::Stream.new
+    #   stream.operations(types: :vote_operation) do |vote|
+    #     puts vote.to_json
+    #   end
+    # 
+    # ... Or ...
+    # 
+    #   stream = Hive::Stream.new
+    #   stream.operations(:vote_operation) do |vote|
+    #     puts vote.to_json
+    #   end
+    #
+    # You can also stream virtual operations:
+    #
+    #   stream = Hive::Stream.new
+    #   stream.operations(types: :author_reward_operation, only_virtual: true) do |vop|
+    #     v = vop.value
+    #     puts "#{v.author} got paid for #{v.permlink}: #{[v.sbd_payout, v.steem_payout, v.vesting_payout]}"
+    #   end
+    #
+    # ... or multiple virtual operation types;
+    #
+    #   stream = Hive::Stream.new
+    #   stream.operations(types: [:producer_reward_operation, :author_reward_operation], only_virtual: true) do |vop|
+    #     puts vop.to_json
+    #   end
+    #
+    # ... or all types, including virtual operation types from the head block number:
+    #
+    #   stream = Hive::Stream.new(mode: :head)
+    #   stream.operations(include_virtual: true) do |op|
+    #     puts op.to_json
+    #   end
+    #
+    # Expected virtual operation types:
+    #
+    #   producer_reward_operation
+    #   author_reward_operation
+    #   curation_reward_operation
+    #   fill_convert_request_operation
+    #   fill_order_operation
+    #   fill_vesting_withdraw_operation
+    #   interest_operation
+    #   shutdown_witness_operation
+    #
+    # @param args [Symbol || Array<Symbol> || Hash] the type(s) of operation or hash of expanded options, optional.
+    # @option args [Integer] :at_block_num Starts the stream at the given block number.  Default: nil.
+    # @option args [Integer] :until_block_num Ends the stream at the given block number.  Default: nil.
+    # @option args [Symbol || Array<Symbol>] :types the type(s) of operation, optional.
+    # @option args [Boolean] :only_virtual Only stream virtual options.  Setting this true will improve performance because the stream only needs block numbers to then retrieve virtual operations.  Default: false.
+    # @option args [Boolean] :include_virtual Also stream virtual options.  Setting this true will impact performance.  Default: false.
+    # @param block the block to execute for each result.  Yields: |op, trx_id, block_num|
+    def operations(*args, &block)
+      options = {}
+      types = []
+      only_virtual = false
+      include_virtual = false
+      last_block_num = nil
+      
+      case args.first
+      when Hash
+        options = args.first
+        types = transform_types(options[:types])
+        only_virtual = !!options[:only_virtual] || false
+        include_virtual = !!options[:include_virtual] || only_virtual || false
+      when Symbol, Array then types = transform_types(args)
+      end
+      
+      if only_virtual
+        block_numbers(options) do |block_num|
+          get_virtual_ops(types, block_num, block)
+        end
+      else
+        transactions(options) do |transaction, trx_id, block_num|
+          transaction.operations.each do |op|
+            yield op, trx_id, block_num if types.none? || types.include?(op.type)
+            
+            next unless last_block_num != block_num
+            
+            last_block_num = block_num
+            
+            get_virtual_ops(types, block_num, block) if include_virtual
+          end
+        end
+      end
+    end
+    
+    def account_history_api
+      @account_history_api ||= begin
+        Hive::AccountHistoryApi.new(@instance_options)
+      rescue Hive::UnknownApiError => e
+        warn "#{e.inspect}, falling back to Hive::CondenserApi." unless @no_warn
+        Hive::CondenserApi.new(@instance_options)
+      end
+    end
+  private
+    # @private
+    def block_objects(options = {}, block)
+      object = options[:object]
+      object_method = "get_#{object}".to_sym
+      block_interval = BLOCK_INTERVAL
+      
+      at_block_num, until_block_num = if !!block_range = options[:block_range]
+        [block_range.first, block_range.last]
+      else
+        [options[:at_block_num], options[:until_block_num]]
+      end
+      
+      loop do
+        break if !!until_block_num && !!at_block_num && until_block_num < at_block_num
+        
+        database_api.get_dynamic_global_properties do |properties|
+          current_block_num = find_block_number(properties)
+          current_block_num = [current_block_num, until_block_num].compact.min
+          at_block_num ||= current_block_num
+          
+          if current_block_num >= at_block_num
+            range = at_block_num..current_block_num
+            
+            if object == :block_numbers
+              range.each do |n|
+                block.call n
+                block_interval = BLOCK_INTERVAL
+              end
+            else
+              block_api.send(object_method, block_range: range) do |b, n|
+                block.call b, n
+                block_interval = BLOCK_INTERVAL
+              end
+            end
+            
+            at_block_num = range.max + 1
+          else
+            # The stream has stalled, so let's back off and let the node sync
+            # up.  We'll catch up with a bigger batch in the next cycle.
+            block_interval = [block_interval * 2, MAX_BACKOFF_BLOCK_INTERVAL].min
+          end
+        end
+        
+        sleep block_interval
+      end
+    end
+    
+    # @private
+    def find_block_number(properties)
+      block_num = case mode
+      when :head then properties.head_block_number
+      when :irreversible then properties.last_irreversible_block_num
+      else; raise Hive::ArgumentError, "Unknown mode: #{mode}"
+      end
+      
+      block_num
+    end
+    
+    # @private
+    def transform_types(types)
+      [types].compact.flatten.map do |type|
+        type = type.to_s
+        
+        unless type.end_with? '_operation'
+          warn "Op type #{type} is deprecated.  Use #{type}_operation instead." unless @no_warn
+          type += '_operation'
+        end
+        
+        type
+      end
+    end
+    
+    # @private
+    def get_virtual_ops(types, block_num, block)
+      retries = 0
+      
+      loop do
+        get_ops_in_block_options = case account_history_api
+        when Hive::CondenserApi
+          [block_num, true]
+        when Hive::AccountHistoryApi
+          {
+            block_num: block_num,
+            only_virtual: true
+          }
+        end
+        
+        response = account_history_api.get_ops_in_block(*get_ops_in_block_options)
+        
+        if response.nil? || (result = response.result).nil?
+          if retries < MAX_RETRY_COUNT
+            warn "Retrying get_ops_in_block on block #{block_num}" unless @no_warn
+            retries = retries + 1
+            sleep 9
+            redo
+          else
+            raise TooManyRetriesError, "unable to get valid result while finding virtual operations for block: #{block_num}"
+          end
+        end
+        
+        ops = case account_history_api
+        when Hive::CondenserApi
+          result.map do |trx|
+            op = {type: trx.op[0] + '_operation', value: trx.op[1]}
+            op = Hashie::Mash.new(op)
+          end
+        when Hive::AccountHistoryApi then result.ops.map { |trx| trx.op }
+        end
+        
+        if ops.empty?
+          if retries < MAX_RETRY_COUNT
+            sleep 3
+            retries = retries + 1
+            redo
+          else
+            warn "unable to find virtual operations for block: #{block_num}"
+            # raise TooManyRetriesError, "unable to find virtual operations for block: #{block_num}"
+          end
+        end
+        
+        ops.each do |op|
+          next if types.any? && !types.include?(op.type)
+          
+          block.call op, VOP_TRX_ID, block_num
+        end
+        
+        break
+      end
+    end
+  end
+end