radiator 0.4.0pre1 → 0.4.0pre2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b0f7aaa5e12224ae48fdfe1535182583e298d97f
-  data.tar.gz: 5ec823c95e96ad1a44ceb0b77bc08b72aede7f8e
+  metadata.gz: ec41122ec4298e304c921f89355e437830c9fd78
+  data.tar.gz: 5aec651358eea83415241b2c6295122951d833c0
 SHA512:
-  metadata.gz: 6b7736813af94b8ef03e1b80e0f43aa593b194c48f71094e672798d3791eab2f0925ee313d1ea5b8b749cc8b753e0209f9298205038c5ff71b43015289eedc27
-  data.tar.gz: 55248de7d7142432b2d8b3a11a9e536c34915ba52bea098e38988e4c3c180ace704cfe852aa9a65914b6afc702f6bd94a6a831c2e5a1a7b6ee623b31c5e8aa53
+  metadata.gz: 33e83e99337f5e9146a39e65057bcd88693f9d208fbcef91acba1f0b5d3741fed47a2d98e70110e3a326c3d771bf4b3ec68c5cd9dceb3271c35676d676f46b3c
+  data.tar.gz: 170b5bec982a6f359cbdc64d0d89e7e695e13687c70a54867edd33424190ec216934502631e493e54494b42d395a22a1819b2ce63402df9fa8570f53c234b5e9

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    radiator (0.4.0pre1)
+    radiator (0.4.0pre2)
       awesome_print (~> 1.7, >= 1.7.0)
       bitcoin-ruby (~> 0.0, >= 0.0.11)
       ffi (~> 1.9, >= 1.9.18)
@@ -16,7 +16,7 @@ GEM
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
     awesome_print (1.8.0)
-    bitcoin-ruby (0.0.17)
+    bitcoin-ruby (0.0.18)
     connection_pool (2.2.1)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)

data/Rakefile

@@ -57,14 +57,13 @@ task :test_live_stream, [:chain, :persist] do |t, args|
   chain = args[:chain] || 'steem'
   persist = (args[:persist] || 'true') == 'true'
   last_block_number = 0
-  options = {chain: chain, persist: persist}
-  api = Radiator::Api.new(options)
+  options = {chain: chain, persist: persist, url: 'https://api.steemitstage.com'}
   total_ops = 0.0
   total_vops = 0.0
   elapsed = 0
   count = 0
   
-  Radiator::Stream.new(options).blocks do |b, n|
+  Radiator::Stream.new(options).blocks do |b, n, api|
     start = Time.now.utc
     
     if last_block_number == 0
@@ -75,6 +74,7 @@ task :test_live_stream, [:chain, :persist] do |t, args|
       o = t.map(&:operations)
       op_size = o.map(&:size).reduce(0, :+)
       total_ops += op_size
+      
       api.get_ops_in_block(n, true) do |vops, error|
         if !!error
           puts "Error on get_ops_in_block for block #{n}"

data/lib/radiator/api.rb

@@ -205,11 +205,25 @@ module Radiator
       @preferred_url = @url.dup
       @failover_urls = options[:failover_urls]
       @debug = !!options[:debug]
-      @logger = options[:logger] || Radiator.logger
-      @hashie_logger = options[:hashie_logger] || Logger.new(nil)
       @max_requests = options[:max_requests] || 30
       @ssl_verify_mode = options[:ssl_verify_mode] || OpenSSL::SSL::VERIFY_PEER
       @ssl_version = options[:ssl_version]
+
+      @self_logger = false
+      @logger = if options[:logger].nil?
+        @self_logger = true
+        Radiator.logger
+      else
+        options[:logger]
+      end
+      
+      @self_hashie_logger = false
+      @hashie_logger = if options[:hashie_logger].nil?
+        @self_hashie_logger = true
+        Logger.new(nil)
+      else
+        options[:hashie_logger]
+      end
       
       if @failover_urls.nil?
         @failover_urls = Api::default_failover_urls(@chain) - [@url]
@@ -306,15 +320,19 @@ module Radiator
       @block_api.shutdown if !!@block_api && @block_api != self
       @block_api = nil
       
-      if !!@logger && defined?(@logger.close)
-        if defined?(@logger.closed?)
-          @logger.close unless @logger.closed?
+      if @self_logger
+        if !!@logger && defined?(@logger.close)
+          if defined?(@logger.closed?)
+            @logger.close unless @logger.closed?
+          end
         end
       end
       
-      if !!@hashie_logger && defined?(@hashie_logger.close)
-        if defined?(@hashie_logger.closed?)
-          @hashie_logger.close unless @hashie_logger.closed?
+      if @self_hashie_logger
+        if !!@hashie_logger && defined?(@hashie_logger.close)
+          if defined?(@hashie_logger.closed?)
+            @hashie_logger.close unless @hashie_logger.closed?
+          end
         end
       end
     end
@@ -747,6 +765,7 @@ module Radiator
       @backoff_sleep ||= 0.01
       
       @backoff_sleep *= 2
+      GC.start
       sleep @backoff_sleep
     ensure
       if !!@backoff_at && Time.now.utc - @backoff_at > 300

data/lib/radiator/chain.rb

@@ -35,6 +35,19 @@ module Radiator
     ).map(&:to_sym)
     VALID_OPTIONS.each { |option| attr_accessor option }
     
+    def self.parse_slug(*args)
+      args = [args].flatten
+      
+      if args.size == 1
+        case args[0]
+        when String then split_slug(args[0])
+        when Hash then [args[0]['author'], args[0]['permlink']]
+        end
+      else
+        args
+      end
+    end
+    
     def initialize(options = {})
       options = options.dup
       options.each do |k, v|
@@ -98,7 +111,7 @@ module Radiator
     # @param args [String || Array<String>] Slug or author, permlink of comment.
     # @return [Hash]
     def find_comment(*args)
-      author, permlink = normalize_author_permlink(args)
+      author, permlink = Chain.parse_slug(args)
       
       api.get_content(author, permlink) do |comment, err|
         raise ChainError, ErrorParser.new(err) if !!err
@@ -241,6 +254,15 @@ module Radiator
       end
     end
   private
+    def self.split_slug(slug)
+      slug = slug.split('@').last
+      author = slug.split('/')[0]
+      permlink = slug.split('/')[1..-1].join('/')
+      permlink = permlink.split('#')[0]
+      
+      [author, permlink]
+    end
+    
     def build_options
       {
         chain: chain,
@@ -262,23 +284,6 @@ module Radiator
       @follow_api ||= FollowApi.new(build_options)
     end
     
-    def parse_slug(slug)
-      slug = slug.split('@').last
-      author = slug.split('/')[0]
-      [author, slug.split('/')[1..-1].join('/')]
-    end
-    
-    def normalize_author_permlink(args)
-      if args.size == 1
-        case args[0]
-        when String then parse_slug(args[0])
-        when Hash then [args[0]['author'], args[0]['permlink']]
-        end
-      else
-        args
-      end
-    end
-    
     def default_max_acepted_payout
       "1000000.000 #{default_debt_asset}"
     end

data/lib/radiator/error_parser.rb

@@ -58,6 +58,14 @@ module Radiator
       end
 
       begin
+        if @error['data'].nil?
+          @expiry = false
+          @can_retry = false
+          @can_reprepare = false
+          
+          return
+        end
+
         @error_code = @error['data']['code']
         stacks = @error['data']['stack']
         stack_formats = stacks.map { |s| s['format'] }

data/lib/radiator/stream.rb

@@ -21,7 +21,7 @@ module Radiator
     MAX_TIMEOUT = 80
     
     # @private
-    MAX_BLOCKS_PER_NODE = 1000
+    MAX_BLOCKS_PER_NODE = 10000
     
     RANGE_BEHIND_WARNING = 400
     
@@ -120,7 +120,7 @@ module Radiator
     # @param mode we have the choice between
     #   * :head the last block
     #   * :irreversible the block that is confirmed by 2/3 of all block producers and is thus irreversible!
-    # @param block the block to execute for each result, optional.
+    # @param block the block to execute for each result, optional.  Yields: |op, trx_id, block_num, api|
     # @param options [Hash] additional options
     # @option options [Boollean] :include_virtual Also stream virtual options.  Setting this true will impact performance.  Default: false.
     # @return [Hash]
@@ -148,19 +148,28 @@ module Radiator
         end.compact
         
         if include_virtual && !virtual_ops_collected
-          api.get_ops_in_block(block_number, true) do |vops|
-            vops.each do |vtx|
-              next unless defined? vtx.op
+          catch :pop_vops do; begin
+            api.get_ops_in_block(block_number, true) do |vops, error|
+              if !!error
+                standby "Node responded with: #{error.message || 'unknown error'}, retrying ...", {
+                  error: error,
+                  and: {throw: :pop_vops}
+                }
+              end
               
-              t = vtx.op.first.to_sym
-              op = vtx.op.last
-              if type.size == 1 && type.first == t
-                ops << op
-              elsif type.none? || type.include?(t)
-                ops << {t => op}
+              vops.each do |vtx|
+                next unless defined? vtx.op
+                
+                t = vtx.op.first.to_sym
+                op = vtx.op.last
+                if type.size == 1 && type.first == t
+                  ops << op
+                elsif type.none? || type.include?(t)
+                  ops << {t => op}
+                end
               end
             end
-          end
+          end; end
           
           virtual_ops_collected = true
         end
@@ -170,7 +179,7 @@ module Radiator
         return ops unless !!block
         
         ops.each do |op|
-          yield op, trx_id, block_number
+          yield op, trx_id, block_number, api
         end
       end
     end
@@ -186,7 +195,7 @@ module Radiator
     # @param mode we have the choice between
     #   * :head the last block
     #   * :irreversible the block that is confirmed by 2/3 of all block producers and is thus irreversible!
-    # @param block the block to execute for each result, optional.
+    # @param block the block to execute for each result, optional.  Yields: |tx, trx_id, api|
     # @return [Hash]
     def transactions(start = nil, mode = :irreversible, &block)
       blocks(start, mode) do |b, block_number|
@@ -198,7 +207,7 @@ module Radiator
             b['transaction_ids'][index]
           end
           
-          yield transaction, trx_id, block_number
+          yield transaction, trx_id, block_number, api
         end
       end
     end
@@ -209,13 +218,27 @@ module Radiator
     #   stream.blocks do |bk, num|
     #     puts "[#{num}] #{bk.to_json}"
     #   end
-    # 
+    #
+    # For convenience and memory management, the api used to poll the current
+    # block data is also available inside the block, e.g.:
+    #
+    #   stream = Radiator::Stream.new
+    #   stream.blocks do |bk, num, api|
+    #     puts "[#{num}] #{bk.to_json}"
+    #     
+    #     api.get_ops_in_block(num, true) do |vops, error|
+    #       puts vops
+    #     end
+    #   end
+    #
+    # This idiom is useful for very long running scripts.
+    #
     # @param start starting block
     # @param mode we have the choice between
     #   * :head the last block
     #   * :irreversible the block that is confirmed by 2/3 of all block producers and is thus irreversible!
     # @param max_blocks_per_node the number of blocks to read before trying a new node
-    # @param block the block to execute for each result, optional.
+    # @param block the block to execute for each result, optional.  Yields: |bk, num, api|
     # @return [Hash]
     def blocks(start = nil, mode = :irreversible, max_blocks_per_node = MAX_BLOCKS_PER_NODE, &block)
       reset_api
@@ -229,7 +252,14 @@ module Radiator
         break if stop?
         
         catch :sequence do; begin
-          head_block = api.get_dynamic_global_properties do |properties|
+          head_block = api.get_dynamic_global_properties do |properties, error|
+            if !!error
+              standby "Node responded with: #{error.message || 'unknown error'}, retrying ...", {
+                error: error,
+                and: {throw: :sequence}
+              }
+            end
+            
             break if stop?
             
             if properties.head_block_number.nil?
@@ -295,7 +325,7 @@ module Radiator
               
               latest_block_number = n
               return current_block, n if block.nil?
-              yield current_block, n
+              yield current_block, n, api
             end
             
             start = head_block + 1
@@ -331,6 +361,7 @@ module Radiator
       
       @api = nil
       @block_api = nil
+      GC.start
     end
     
     # @private

data/lib/radiator/transaction.rb

@@ -25,13 +25,20 @@ module Radiator
         end
       end
 
-      @logger = options[:logger] || Radiator.logger
       @chain ||= :steem
       @chain = @chain.to_sym
       @chain_id = chain_id options[:chain_id]
       @url = options[:url] || url
       @operations = options[:operations] || []
-
+      
+      @self_logger = false
+      @logger = if options[:logger].nil?
+        @self_logger = true
+        Radiator.logger
+      else
+        options[:logger]
+      end
+      
       unless NETWORK_CHAIN_IDS.include? @chain_id
         warning "Unknown chain id: #{@chain_id}"
       end
@@ -60,7 +67,7 @@ module Radiator
       @api = Api.new(options)
       @network_broadcast_api = NetworkBroadcastApi.new(options)
 
-      ObjectSpace.define_finalizer(self, self.class.finalize(@api, @network_broadcast_api, @logger))
+      ObjectSpace.define_finalizer(self, self.class.finalize(@api, @network_broadcast_api, @self_logger, @logger))
     end
 
     def chain_id(chain_id = nil)
@@ -124,9 +131,11 @@ module Radiator
       @api.shutdown if !!@api
       @network_broadcast_api.shutdown if !!@network_broadcast_api
 
-      if !!@logger && defined?(@logger.close)
-        if defined?(@logger.closed?)
-          @logger.close unless @logger.closed?
+      if @self_logger
+        if !!@logger && defined?(@logger.close)
+          if defined?(@logger.closed?)
+            @logger.close unless @logger.closed?
+          end
         end
       end
     end
@@ -251,7 +260,7 @@ module Radiator
       )
     end
 
-    def self.finalize(api, network_broadcast_api, logger)
+    def self.finalize(api, network_broadcast_api, self_logger, logger)
       proc {
         if !!api && !api.stopped?
           puts "DESTROY: #{api.inspect}" if ENV['LOG'] == 'TRACE'
@@ -266,9 +275,11 @@ module Radiator
         end
 
         begin
-          if !!logger && defined?(logger.close)
-            if defined?(logger.closed?)
-              logger.close unless logger.closed?
+          if self_logger
+            if !!logger && defined?(logger.close)
+              if defined?(logger.closed?)
+                logger.close unless logger.closed?
+              end
             end
           end
         rescue IOError, NoMethodError => _; end

data/lib/radiator/version.rb

@@ -1,4 +1,4 @@
 module Radiator
-  VERSION = '0.4.0pre1'
+  VERSION = '0.4.0pre2'
   AGENT_ID = "radiator/#{VERSION}"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: radiator
 version: !ruby/object:Gem::Version
-  version: 0.4.0pre1
+  version: 0.4.0pre2
 platform: ruby
 authors:
 - Anthony Martin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-17 00:00:00.000000000 Z
+date: 2018-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler