datahen 0.20.0 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11c43658f61faff627da448abff393b5cc658c15e1dfd6765c6cfdee96958d01
-  data.tar.gz: 5dc4481c9755d33dcee2539a24fe1ba2b0336fc4320beaa54204d1636067ee75
+  metadata.gz: 4aa3927b9865f2815f64463f1d48b2dadddeaa73d2bd446a98ea9eb9ecb3ff5f
+  data.tar.gz: 91b7a370e740721202e9f14c043cc5c28cb9e6102dcd701c5121506042ba753b
 SHA512:
-  metadata.gz: e25db9321dfb26cb88d38aa9adf353c9955c3932553fb87affe235bd58f0182e7043775d102ce2bb5e7d6d0764cd76bd71f3be024ced38eadd17c05cfa0efd69
-  data.tar.gz: b07e2655df12424db7f859da29f0921fa6417dfbfa6b72537615089d6fabac7451e3e61e23ba4f4e8cfff0c0555ef50bf751dd8b91b8be8ae51b23d855145586
+  metadata.gz: c643100e60ea20686d882377b7e982829f93b4d4d8750342d47370f9d649688e94517462041b08ec1387901fcc8f33a0b0105e5f4ab43cd378dca5768cc190c4
+  data.tar.gz: ba3bae8b462aa6894520dc115452a27fcb3d64571820970ad7b69592670549824ec53daa1c8194392750f746a14ced4baf2e552eafc2684735dc185c57af8e8e

data/lib/datahen/cli/scraper.rb

@@ -24,8 +24,9 @@ module Datahen
       option :freshness_type, :aliases => :t, desc: 'Set how fresh the page cache is. Possible values: day, week, month, year. Default: any'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
-      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
-      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
+      option :parsers, :aliases => :pw, type: :numeric, desc: 'Set how many parser workers to use. Default: 1'
+      option :fetchers, :aliases => :fw, type: :numeric, desc: 'Set how many fetcher workers to use. Default: 1'
+      option :browsers, :aliases => :bw, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :disable_scheduler, type: :boolean, desc: 'Set true to disable scheduler. Default: false'
       option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
       option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
@@ -52,8 +53,9 @@ module Datahen
       option :freshness_type, :aliases => :t, desc: 'Set how fresh the page cache is. Possible values: day, week, month, year. Default: any'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
-      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
-      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
+      option :parsers, :aliases => :pw, type: :numeric, desc: 'Set how many parser workers to use. Default: 1'
+      option :fetchers, :aliases => :fw, type: :numeric, desc: 'Set how many fetcher workers to use. Default: 1'
+      option :browsers, :aliases => :bw, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :disable_scheduler, type: :boolean, desc: 'Set true to disable scheduler. Default: false'
       option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
       option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
@@ -97,8 +99,9 @@ module Datahen
       long_desc <<-LONGDESC
           Starts a scraper by creating an active scrape job\x5
           LONGDESC
-      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
-      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
+      option :parsers, :aliases => :pw, type: :numeric, desc: 'Set how many parser workers to use. Default: 1'
+      option :fetchers, :aliases => :fw, type: :numeric, desc: 'Set how many fetcher workers to use. Default: 1'
+      option :browsers, :aliases => :bw, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: [{"name":"foo", "value":"bar", "secret":false}] '
       option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'

data/lib/datahen/cli/scraper_job.rb

@@ -99,8 +99,9 @@ module Datahen
       long_desc <<-LONGDESC
         Updates a scraper's current job.
       LONGDESC
-      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 1. '
-      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 0. '
+      option :parsers, :aliases => :pw, type: :numeric, desc: 'Set how many parser workers to use. Scraper job must be restarted (paused then resumed) for it to take effect. Default: 1. '
+      option :fetchers, :aliases => :fw, type: :numeric, desc: 'Set how many fetcher workers to use. Scraper job must be restarted (paused then resumed) for it to take effect. Default: 1. '
+      option :browsers, :aliases => :bw, type: :numeric, desc: 'Set how many browser workers to use. Scraper job must be restarted (paused then resumed) for it to take effect. Default: 0. '
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'

data/lib/datahen/client/base.rb

@@ -7,6 +7,12 @@ module Datahen
 
       default_timeout 60
 
+      DEFAULT_RETRY_LIMIT = {
+        seeder: nil,
+        parser: nil,
+        finisher: nil
+      }
+
       def self.env_auth_token
         ENV['DATAHEN_TOKEN']
       end
@@ -33,6 +39,42 @@ module Datahen
         @auth_token = value
       end
 
+      def default_retry_limit
+        @default_retry_limit ||= DEFAULT_RETRY_LIMIT.dup
+      end
+
+      def left_merge target, source
+        # validate source and target
+        return {} if target.nil? || !target.is_a?(Hash)
+        return target if source.nil? || !source.is_a?(Hash)
+
+        # left merge source into target
+        target.merge(source.select{|k,v|target.has_key?(k)})
+      end
+
+      def retry times, delay = nil, err_msg = nil
+        limit = times.nil? ? nil : times.to_i
+        delay = delay.nil? ? 5 : delay.to_i
+        count = 0
+        begin
+          yield
+        rescue StandardError => e
+          STDERR.puts(e.inspect)
+
+          # wait before retry (default 5 sec)
+          sleep(delay) if delay > 0
+
+          # raise error when retry limit is reached
+          raise e unless limit.nil? || count < limit
+
+          # retry with a 100+ failsafe to prevent overflow error due integer limit
+          should_aprox = limit.nil? && count > 99
+          count += 1 unless should_aprox
+          puts "#{err_msg.nil? ? '' : "#{err_msg} "}Retry \##{count}#{should_aprox ? '+' : ''}..."
+          retry
+        end
+      end
+
       def initialize(opts={})
         @ignore_ssl = opts[:ignore_ssl]
         self.class.base_uri(env_api_url)
@@ -45,6 +87,9 @@ module Datahen
           verify: !ignore_ssl
         }
 
+        # extract and merge retry limits
+        @default_retry_limit = self.left_merge(DEFAULT_RETRY_LIMIT, opts[:retry_limit])
+
         query = {}
         query[:p] = opts[:page] if opts[:page]
         query[:pp] = opts[:per_page] if opts[:per_page]

data/lib/datahen/client/job.rb

@@ -17,7 +17,8 @@ module Datahen
       def update(job_id, opts={})
         body = {}
         body[:status] = opts[:status] if opts[:status]
-        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:parser_worker_count] = opts[:parsers] if opts[:parsers]
+        body[:fetcher_worker_count] = opts[:fetchers] if opts[:fetchers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:profile] = opts[:profile] if opts[:profile]
@@ -54,7 +55,10 @@ module Datahen
 
         params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}/seeding_update", params)
+        limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:seeder]
+        self.retry(limit, 5, "Error while updating the seeder.") do
+          self.class.put("/jobs/#{job_id}/seeding_update", params)
+        end
       end
 
       def finisher_update(job_id, opts={})
@@ -65,7 +69,10 @@ module Datahen
 
         params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}/finisher_update", params)
+        limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:finisher]
+        self.retry(limit, 5, "Error while updating the finisher.") do
+          self.class.put("/jobs/#{job_id}/finisher_update", params)
+        end
       end
 
       def profile(job_id, opts={})

data/lib/datahen/client/job_output.rb

@@ -5,9 +5,11 @@ module Datahen
         self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records/#{id}", @options)
       end
 
-      def all(job_id, collection = 'default')
-
-        self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records", @options)
+      def all(job_id, collection = 'default', opts = {})
+        limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : 0
+        self.retry(limit, 10, "Error while updating the seeder.") do
+          self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records", @options)
+        end
       end
 
       def collections(job_id)
@@ -16,4 +18,3 @@ module Datahen
     end
   end
 end
-

data/lib/datahen/client/job_page.rb

@@ -68,7 +68,10 @@ module Datahen
 
         params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
+        limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:parser]
+        self.retry(limit, 5, "Error while updating the parser.") do
+          self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
+        end
       end
 
       def find_content(job_id, gid)

data/lib/datahen/client/scraper.rb

@@ -18,7 +18,8 @@ module Datahen
         body[:git_branch] = opts[:branch] || opts[:git_branch] || "master" if opts[:branch] || opts[:git_branch]
         body[:freshness_type] = opts[:freshness_type] if opts[:freshness_type]
         body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
-        body[:standard_worker_count] = opts[:workers] || opts[:standard_worker_count] if opts[:workers] || opts[:standard_worker_count]
+        body[:parser_worker_count] = opts[:parsers] || opts[:parser_worker_count] if opts[:parsers] || opts[:parser_worker_count]
+        body[:fetcher_worker_count] = opts[:fetchers] || opts[:fetcher_worker_count] if opts[:fetchers] || opts[:fetcher_worker_count]
         body[:browser_worker_count] = opts[:browsers] || opts[:browser_worker_count] if opts[:browsers] || opts[:browser_worker_count]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:disable_scheduler] = opts[:disable_scheduler] if opts[:disable_scheduler]
@@ -42,7 +43,8 @@ module Datahen
         body[:git_branch] = opts[:branch] || opts[:git_branch] if opts[:branch] || opts[:git_branch]
         body[:freshness_type] = opts[:freshness_type] if opts[:freshness_type]
         body[:force_fetch] = opts[:force_fetch] if opts.has_key?("force_fetch") || opts.has_key?(:force_fetch)
-        body[:standard_worker_count] = opts[:workers] || opts[:standard_worker_count] if opts[:workers] || opts[:standard_worker_count]
+        body[:parser_worker_count] = opts[:parsers] || opts[:parser_worker_count] if opts[:parsers] || opts[:parser_worker_count]
+        body[:fetcher_worker_count] = opts[:fetchers] || opts[:fetcher_worker_count] if opts[:fetchers] || opts[:fetcher_worker_count]
         body[:browser_worker_count] = opts[:browsers] || opts[:browser_worker_count] if opts[:browsers] || opts[:browser_worker_count]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:disable_scheduler] = opts[:disable_scheduler] if opts.has_key?("disable_scheduler") || opts.has_key?(:disable_scheduler)

data/lib/datahen/client/scraper_job.rb

@@ -8,7 +8,8 @@ module Datahen
 
       def create(scraper_name, opts={})
         body = {}
-        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:parser_worker_count] = opts[:parsers] if opts[:parsers]
+        body[:fetcher_worker_count] = opts[:fetchers] if opts[:fetchers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
@@ -36,7 +37,8 @@ module Datahen
       def update(scraper_name, opts={})
         body = {}
         body[:status] = opts[:status] if opts[:status]
-        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:parser_worker_count] = opts[:parsers] if opts[:parsers]
+        body[:fetcher_worker_count] = opts[:fetchers] if opts[:fetchers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:profile] = opts[:profile] if opts[:profile]

data/lib/datahen/scraper/batch_parser.rb

@@ -152,7 +152,7 @@ module Datahen
         @page_types = []
         @parsers = Concurrent::Hash.new
         @config = YAML.load_file(config_file)
-        self.config['parsers'].each do |v|
+        (self.config['parsers'] || []).each do |v|
           next if !v['disabled'].nil? && !!v['disabled']
           @page_types << v['page_type']
           self.parsers[v['page_type']] = v['file']

data/lib/datahen/scraper/executor.rb

@@ -5,6 +5,7 @@ module Datahen
     class Executor
       # Max allowed page size when query outputs (see #find_outputs).
       MAX_FIND_OUTPUTS_PER_PAGE = 500
+      FIND_OUTPUTS_RETRY_LIMIT = 0
 
       attr_accessor :filename, :page, :gid, :job_id
 
@@ -159,13 +160,18 @@ module Datahen
         options = {
           query: query,
           page: page,
-          per_page: per_page}
+          per_page: per_page
+        }
 
         # Get job_id
         query_job_id = opts[:job_id] || get_job_id(opts[:scraper_name], self.job_id)
 
+        # find outputs
+        retry_limit = opts.has_key?(:retry_limit) ? opts[:retry_limit] : self.class::FIND_OUTPUTS_RETRY_LIMIT
         client = Client::JobOutput.new(options)
-        response = client.all(query_job_id, collection)
+        response = client.all(query_job_id, collection, {
+          retry_limit: retry_limit
+        })
 
         if response.code != 200
           raise "response_code: #{response.code}|#{response.parsed_response}"
@@ -304,6 +310,7 @@ module Datahen
           end
 
           # saving to server
+
           response = update_to_server(
             job_id: job_id,
             gid: gid,

data/lib/datahen/scraper/ruby_finisher_executor.rb

@@ -3,6 +3,8 @@ module Datahen
     class RubyFinisherExecutor < Executor
       attr_accessor :save
 
+      FIND_OUTPUTS_RETRY_LIMIT = nil
+
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
         @job_id = options[:job_id]

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -14,6 +14,8 @@ module Datahen
       # @return [Boollean]
       attr_accessor :limbo_self
 
+      FIND_OUTPUTS_RETRY_LIMIT = nil
+
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
         @page = options.fetch(:page) { nil }

data/lib/datahen/scraper/ruby_seeder_executor.rb

@@ -3,6 +3,8 @@ module Datahen
     class RubySeederExecutor < Executor
       attr_accessor :save
 
+      FIND_OUTPUTS_RETRY_LIMIT = nil
+
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
         @job_id = options[:job_id]

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.20.0"
+  VERSION = "1.0.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.20.0
+  version: 1.0.2
 platform: ruby
 authors:
 - Parama Danoesubroto
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-11-29 00:00:00.000000000 Z
+date: 2022-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -277,7 +277,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://datahen.com
   source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -293,7 +293,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key:
+signing_key: 
 specification_version: 4
 summary: DataHen toolbelt for developers
 test_files: []