datahen 1.0.0 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea12b1c12b5a5db4a650b35869de91b9b2ccc8c0c5b4e35da904fc77bfee5ebc
-  data.tar.gz: bd96345cc669816cc281d76065cf64d150268aa8f14659e6395796d2aebd52ec
+  metadata.gz: c325fe2b8125d70ee73a9c66511cd0f2f0db047b9ea387d73bec94d666ace153
+  data.tar.gz: 545c5cb81a714bedc4ce264868383a722338f3a4b7a9cd955ae88a579193faa3
 SHA512:
-  metadata.gz: 763c11bb6d96fdd92c8d2eb8c7965729b3812dbc0dfa9abb47151a61175f695870369d98cac0663ebdf2c644eda028833be313fb8e7924a353f82049c6430c22
-  data.tar.gz: 43e074b6acde5a0367fc11f74c0a3dab0c7e1aecfc781c1e927c8e55bb6e367701ec0dcec2aa90d63c988eca16af90577a63e5f8191a5c7c055e9d0fb9e5bbea
+  metadata.gz: 8d4856fc36ab96ccee541628ff6082d602c1f8ce102d91939f557bdabff94d4bc46b4effb2bc5a8037c336e845cdabefc80bd1dc74d059afb40d8cd12be4a847
+  data.tar.gz: 4c70928e362e65afc3f1173f5a4b7ab0ff018c1a76acedfcb80bb5bbb2ba451620c6df28bb909c2186913030e954ab54d1e965bcdebbd6a11763a8e1284c91e6

data/lib/datahen/client/base.rb

@@ -7,6 +7,12 @@ module Datahen
 
       default_timeout 60
 
+      DEFAULT_RETRY_LIMIT = {
+        seeder: nil,
+        parser: nil,
+        finisher: nil
+      }
+
       def self.env_auth_token
         ENV['DATAHEN_TOKEN']
       end
@@ -33,6 +39,42 @@ module Datahen
         @auth_token = value
       end
 
+      def default_retry_limit
+        @default_retry_limit ||= DEFAULT_RETRY_LIMIT.dup
+      end
+
+      def left_merge target, source
+        # validate source and target
+        return {} if target.nil? || !target.is_a?(Hash)
+        return target if source.nil? || !source.is_a?(Hash)
+
+        # left merge source into target
+        target.merge(source.select{|k,v|target.has_key?(k)})
+      end
+
+      def retry times, delay = nil, err_msg = nil
+        limit = times.nil? ? nil : times.to_i
+        delay = delay.nil? ? 5 : delay.to_i
+        count = 0
+        begin
+          yield
+        rescue StandardError => e
+          STDERR.puts(e.inspect)
+
+          # wait before retry (default 5 sec)
+          sleep(delay) if delay > 0
+
+          # raise error when retry limit is reached
+          raise e unless limit.nil? || count < limit
+
+          # retry with a 100+ failsafe to prevent overflow error due integer limit
+          should_aprox = limit.nil? && count > 99
+          count += 1 unless should_aprox
+          puts "#{err_msg.nil? ? '' : "#{err_msg} "}Retry \##{count}#{should_aprox ? '+' : ''}..."
+          retry
+        end
+      end
+
       def initialize(opts={})
         @ignore_ssl = opts[:ignore_ssl]
         self.class.base_uri(env_api_url)
@@ -45,6 +87,9 @@ module Datahen
           verify: !ignore_ssl
         }
 
+        # extract and merge retry limits
+        @default_retry_limit = self.left_merge(DEFAULT_RETRY_LIMIT, opts[:retry_limit])
+
         query = {}
         query[:p] = opts[:page] if opts[:page]
         query[:pp] = opts[:per_page] if opts[:per_page]

data/lib/datahen/client/job.rb

@@ -55,7 +55,10 @@ module Datahen
 
         params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}/seeding_update", params)
+        limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:seeder]
+        self.retry(limit, 5, "Error while updating the seeder.") do
+          self.class.put("/jobs/#{job_id}/seeding_update", params)
+        end
       end
 
       def finisher_update(job_id, opts={})
@@ -66,7 +69,10 @@ module Datahen
 
         params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}/finisher_update", params)
+        limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:finisher]
+        self.retry(limit, 5, "Error while updating the finisher.") do
+          self.class.put("/jobs/#{job_id}/finisher_update", params)
+        end
       end
 
       def profile(job_id, opts={})

data/lib/datahen/client/job_output.rb

@@ -5,9 +5,11 @@ module Datahen
         self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records/#{id}", @options)
       end
 
-      def all(job_id, collection = 'default')
-
-        self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records", @options)
+      def all(job_id, collection = 'default', opts = {})
+        limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : 0
+        self.retry(limit, 10, "Error while updating the seeder.") do
+          self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records", @options)
+        end
       end
 
       def collections(job_id)
@@ -16,4 +18,3 @@ module Datahen
     end
   end
 end
-

data/lib/datahen/client/job_page.rb

@@ -68,7 +68,10 @@ module Datahen
 
         params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
+        limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:parser]
+        self.retry(limit, 5, "Error while updating the parser.") do
+          self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
+        end
       end
 
       def find_content(job_id, gid)

data/lib/datahen/scraper/batch_parser.rb

@@ -152,7 +152,7 @@ module Datahen
         @page_types = []
         @parsers = Concurrent::Hash.new
         @config = YAML.load_file(config_file)
-        self.config['parsers'].each do |v|
+        (self.config['parsers'] || []).each do |v|
           next if !v['disabled'].nil? && !!v['disabled']
           @page_types << v['page_type']
           self.parsers[v['page_type']] = v['file']

data/lib/datahen/scraper/executor.rb

@@ -5,6 +5,7 @@ module Datahen
     class Executor
       # Max allowed page size when query outputs (see #find_outputs).
       MAX_FIND_OUTPUTS_PER_PAGE = 500
+      FIND_OUTPUTS_RETRY_LIMIT = 0
 
       attr_accessor :filename, :page, :gid, :job_id
 
@@ -159,13 +160,18 @@ module Datahen
         options = {
           query: query,
           page: page,
-          per_page: per_page}
+          per_page: per_page
+        }
 
         # Get job_id
         query_job_id = opts[:job_id] || get_job_id(opts[:scraper_name], self.job_id)
 
+        # find outputs
+        retry_limit = opts.has_key?(:retry_limit) ? opts[:retry_limit] : self.class::FIND_OUTPUTS_RETRY_LIMIT
         client = Client::JobOutput.new(options)
-        response = client.all(query_job_id, collection)
+        response = client.all(query_job_id, collection, {
+          retry_limit: retry_limit
+        })
 
         if response.code != 200
           raise "response_code: #{response.code}|#{response.parsed_response}"
@@ -304,6 +310,7 @@ module Datahen
           end
 
           # saving to server
+
           response = update_to_server(
             job_id: job_id,
             gid: gid,

data/lib/datahen/scraper/ruby_finisher_executor.rb

@@ -3,6 +3,8 @@ module Datahen
     class RubyFinisherExecutor < Executor
       attr_accessor :save
 
+      FIND_OUTPUTS_RETRY_LIMIT = nil
+
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
         @job_id = options[:job_id]
@@ -13,7 +15,8 @@ module Datahen
           :outputs,
           :save_outputs,
           :find_output,
-          :find_outputs
+          :find_outputs,
+          :finish
         ].freeze
       end
 

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -14,6 +14,8 @@ module Datahen
       # @return [Boollean]
       attr_accessor :limbo_self
 
+      FIND_OUTPUTS_RETRY_LIMIT = nil
+
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
         @page = options.fetch(:page) { nil }
@@ -36,7 +38,8 @@ module Datahen
           :find_outputs,
           :refetch,
           :reparse,
-          :limbo
+          :limbo,
+          :finish
         ].freeze
       end
 

data/lib/datahen/scraper/ruby_seeder_executor.rb

@@ -3,6 +3,8 @@ module Datahen
     class RubySeederExecutor < Executor
       attr_accessor :save
 
+      FIND_OUTPUTS_RETRY_LIMIT = nil
+
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
         @job_id = options[:job_id]
@@ -16,7 +18,8 @@ module Datahen
           :save_pages,
           :save_outputs,
           :find_output,
-          :find_outputs
+          :find_outputs,
+          :finish
         ].freeze
       end
 

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "1.0.0"
+  VERSION = "1.0.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.3
 platform: ruby
 authors:
 - Parama Danoesubroto
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-06-03 00:00:00.000000000 Z
+date: 2022-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -277,7 +277,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://datahen.com
   source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -293,7 +293,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key:
+signing_key: 
 specification_version: 4
 summary: DataHen toolbelt for developers
 test_files: []