datahen 0.19.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 382b0aa0e8191e8ba977e9e17c8f4db7b49d71e3e5791681d0c22c6ab3234851
-  data.tar.gz: 50ded716d3f79419735e84e2d8edd55ddb845ca9b09b7a24820e09d912b0f52b
+  metadata.gz: d90c6eca445a5ffc51a59c784e7a297801938864dfd4b9f22984ebb1917028de
+  data.tar.gz: e2b68ac1b025f8c24efbed1ba01d8b0e87edbfe3630de2f01d01bffe258f0bf1
 SHA512:
-  metadata.gz: e9f280b1857ac04e918a9179ffcc2afae27e8f36e54613a6fc3b2a446b52be9cb81f8ea74d23634fcd7c0cd87292fefe1c851c32c5b19b77227dfd7ef2e6ba99
-  data.tar.gz: 58eaa407c775945303f33f6fa46185b0486c2871664cf29e6eb513fbbe76d7386b926675e8bef6e43f34e28995ed13f973e4d0ca2a5843cf21b6d2ca96877c98
+  metadata.gz: 60c0c0013d454e3c805f67ae3450ee9229c0da00d0ce4e4fca4bb716ffd2b6a45da234024d12e04f5ce0952b553063e21f9d144e24c7b99875e623c0f5f924e7
+  data.tar.gz: 9a30a97aaa2a6e5d07e45cc6616fdca80a9f99ab04c308d82d06c346654c28cfdf3e111a0d9ebcb74d7dacc0b3d8e1e2e403512887ec04bbfd123c906dbda868

data/lib/datahen/cli/account.rb

@@ -11,6 +11,9 @@ module Datahen
         puts "#{client.profile()}"
       end
 
+      desc "deploy_key SUBCOMMAND ...ARGS", "manage deploy key"
+      subcommand "deploy_key", AccountDeployKey
+
     end
   end
 

data/lib/datahen/cli/account_deploy_key.rb

@@ -0,0 +1,26 @@
+module Datahen
+  class CLI < Thor
+    class AccountDeployKey < Thor
+      package_name "account deploy_key"
+      def self.banner(command, namespace = nil, subcommand = false)
+        "#{basename} #{@package_name} #{command.usage}"
+      end
+
+      desc "show", "Show public deploy key"
+      def show()
+        client = Client::DeployKey.new()
+        puts "#{client.find()}"
+      end
+
+      desc "recreate", "Recreate public deploy key"
+      long_desc <<-LONGDESC
+        Recreate public deploy key.
+      LONGDESC
+      def recreate()
+        client = Client::DeployKey.new()
+        puts "#{client.create()}"
+      end
+    end
+  end
+
+end

data/lib/datahen/cli/scraper.rb

@@ -24,8 +24,9 @@ module Datahen
       option :freshness_type, :aliases => :t, desc: 'Set how fresh the page cache is. Possible values: day, week, month, year. Default: any'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
-      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
-      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
+      option :parsers, :aliases => :pw, type: :numeric, desc: 'Set how many parser workers to use. Default: 1'
+      option :fetchers, :aliases => :fw, type: :numeric, desc: 'Set how many fetcher workers to use. Default: 1'
+      option :browsers, :aliases => :bw, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :disable_scheduler, type: :boolean, desc: 'Set true to disable scheduler. Default: false'
       option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
       option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
@@ -52,8 +53,9 @@ module Datahen
       option :freshness_type, :aliases => :t, desc: 'Set how fresh the page cache is. Possible values: day, week, month, year. Default: any'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :force_fetch, :aliases => :f, type: :boolean, desc: 'Set true to force fetch page that is not within freshness criteria. Default: false'
-      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
-      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
+      option :parsers, :aliases => :pw, type: :numeric, desc: 'Set how many parser workers to use. Default: 1'
+      option :fetchers, :aliases => :fw, type: :numeric, desc: 'Set how many fetcher workers to use. Default: 1'
+      option :browsers, :aliases => :bw, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :disable_scheduler, type: :boolean, desc: 'Set true to disable scheduler. Default: false'
       option :cancel_current_job, type: :boolean, desc: 'Set true to cancel currently active job if scheduler starts. Default: false'
       option :schedule, type: :string, desc: 'Set the schedule of the scraper to run. Must be in CRON format.'
@@ -97,8 +99,9 @@ module Datahen
       long_desc <<-LONGDESC
           Starts a scraper by creating an active scrape job\x5
           LONGDESC
-      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
-      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
+      option :parsers, :aliases => :pw, type: :numeric, desc: 'Set how many parser workers to use. Default: 1'
+      option :fetchers, :aliases => :fw, type: :numeric, desc: 'Set how many fetcher workers to use. Default: 1'
+      option :browsers, :aliases => :bw, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: [{"name":"foo", "value":"bar", "secret":false}] '
       option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'

data/lib/datahen/cli/scraper_job.rb

@@ -99,8 +99,9 @@ module Datahen
       long_desc <<-LONGDESC
         Updates a scraper's current job.
       LONGDESC
-      option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 1. '
-      option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Scraper job must be restarted(paused then resumed, or cancelled then resumed) for it to take effect. Default: 0. '
+      option :parsers, :aliases => :pw, type: :numeric, desc: 'Set how many parser workers to use. Scraper job must be restarted (paused then resumed) for it to take effect. Default: 1. '
+      option :fetchers, :aliases => :fw, type: :numeric, desc: 'Set how many fetcher workers to use. Scraper job must be restarted (paused then resumed) for it to take effect. Default: 1. '
+      option :browsers, :aliases => :bw, type: :numeric, desc: 'Set how many browser workers to use. Scraper job must be restarted (paused then resumed) for it to take effect. Default: 0. '
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'

data/lib/datahen/cli.rb

@@ -16,10 +16,9 @@ require 'datahen/cli/parser'
 require 'datahen/cli/seeder'
 require 'datahen/cli/finisher'
 require 'datahen/cli/env_var'
+require 'datahen/cli/account_deploy_key'
 require 'datahen/cli/account'
 
-
-
 module Datahen
   class CLI < Thor
     desc "scraper SUBCOMMAND ...ARGS", "manage scrapers"

data/lib/datahen/client/base.rb

@@ -7,6 +7,12 @@ module Datahen
 
       default_timeout 60
 
+      DEFAULT_RETRY_LIMIT = {
+        seeder: nil,
+        parser: 2,
+        finisher: nil
+      }
+
       def self.env_auth_token
         ENV['DATAHEN_TOKEN']
       end
@@ -33,6 +39,42 @@ module Datahen
         @auth_token = value
       end
 
+      def default_retry_limit
+        @default_retry_limit ||= DEFAULT_RETRY_LIMIT.dup
+      end
+
+      def left_merge target, source
+        # validate source and target
+        return {} if target.nil? || !target.is_a?(Hash)
+        return target if source.nil? || !source.is_a?(Hash)
+
+        # left merge source into target
+        target.merge(source.select{|k,v|target.has_key?(k)})
+      end
+
+      def retry times, delay = nil, err_msg = nil
+        limit = times.nil? ? nil : times.to_i
+        delay = delay.nil? ? 5 : delay.to_i
+        count = 0
+        begin
+          yield
+        rescue StandardError => e
+          STDERR.puts(e.inspect)
+
+          # wait before retry (default 5 sec)
+          sleep(delay) if delay > 0
+
+          # raise error when retry limit is reached
+          raise e unless limit.nil? || count < limit
+
+          # retry with a 100+ failsafe to prevent overflow error due integer limit
+          should_aprox = limit.nil? && count > 99
+          count += 1 unless should_aprox
+          puts "#{err_msg.nil? ? '' : "#{err_msg} "}Retry \##{count}#{should_aprox ? '+' : ''}..."
+          retry
+        end
+      end
+
       def initialize(opts={})
         @ignore_ssl = opts[:ignore_ssl]
         self.class.base_uri(env_api_url)
@@ -45,6 +87,9 @@ module Datahen
           verify: !ignore_ssl
         }
 
+        # extract and merge retry limits
+        @default_retry_limit = self.left_merge(DEFAULT_RETRY_LIMIT, opts[:retry_limit])
+
         query = {}
         query[:p] = opts[:page] if opts[:page]
         query[:pp] = opts[:per_page] if opts[:per_page]

data/lib/datahen/client/job.rb

@@ -17,7 +17,8 @@ module Datahen
       def update(job_id, opts={})
         body = {}
         body[:status] = opts[:status] if opts[:status]
-        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:parser_worker_count] = opts[:parsers] if opts[:parsers]
+        body[:fetcher_worker_count] = opts[:fetchers] if opts[:fetchers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:profile] = opts[:profile] if opts[:profile]
@@ -54,7 +55,10 @@ module Datahen
 
         params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}/seeding_update", params)
+        limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:seeder]
+        self.retry(limit, 5, "Error while updating the seeder.") do
+          self.class.put("/jobs/#{job_id}/seeding_update", params)
+        end
       end
 
       def finisher_update(job_id, opts={})
@@ -65,7 +69,10 @@ module Datahen
 
         params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}/finisher_update", params)
+        limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:finisher]
+        self.retry(limit, 5, "Error while updating the finisher.") do
+          self.class.put("/jobs/#{job_id}/finisher_update", params)
+        end
       end
 
       def profile(job_id, opts={})

data/lib/datahen/client/job_output.rb

@@ -5,9 +5,11 @@ module Datahen
         self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records/#{id}", @options)
       end
 
-      def all(job_id, collection = 'default')
-
-        self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records", @options)
+      def all(job_id, collection = 'default', opts = {})
+        limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : 0
+        self.retry(limit, 10, "Error while updating the seeder.") do
+          self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records", @options)
+        end
       end
 
       def collections(job_id)
@@ -16,4 +18,3 @@ module Datahen
     end
   end
 end
-

data/lib/datahen/client/job_page.rb

@@ -68,7 +68,10 @@ module Datahen
 
         params = @options.merge({body: body.to_json})
 
-        self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
+        limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:parser]
+        self.retry(limit, 5, "Error while updating the parser.") do
+          self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
+        end
       end
 
       def find_content(job_id, gid)

data/lib/datahen/client/scraper.rb

@@ -18,7 +18,8 @@ module Datahen
         body[:git_branch] = opts[:branch] || opts[:git_branch] || "master" if opts[:branch] || opts[:git_branch]
         body[:freshness_type] = opts[:freshness_type] if opts[:freshness_type]
         body[:force_fetch] = opts[:force_fetch] if opts[:force_fetch]
-        body[:standard_worker_count] = opts[:workers] || opts[:standard_worker_count] if opts[:workers] || opts[:standard_worker_count]
+        body[:parser_worker_count] = opts[:parsers] || opts[:parser_worker_count] if opts[:parsers] || opts[:parser_worker_count]
+        body[:fetcher_worker_count] = opts[:fetchers] || opts[:fetcher_worker_count] if opts[:fetchers] || opts[:fetcher_worker_count]
         body[:browser_worker_count] = opts[:browsers] || opts[:browser_worker_count] if opts[:browsers] || opts[:browser_worker_count]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:disable_scheduler] = opts[:disable_scheduler] if opts[:disable_scheduler]
@@ -42,7 +43,8 @@ module Datahen
         body[:git_branch] = opts[:branch] || opts[:git_branch] if opts[:branch] || opts[:git_branch]
         body[:freshness_type] = opts[:freshness_type] if opts[:freshness_type]
         body[:force_fetch] = opts[:force_fetch] if opts.has_key?("force_fetch") || opts.has_key?(:force_fetch)
-        body[:standard_worker_count] = opts[:workers] || opts[:standard_worker_count] if opts[:workers] || opts[:standard_worker_count]
+        body[:parser_worker_count] = opts[:parsers] || opts[:parser_worker_count] if opts[:parsers] || opts[:parser_worker_count]
+        body[:fetcher_worker_count] = opts[:fetchers] || opts[:fetcher_worker_count] if opts[:fetchers] || opts[:fetcher_worker_count]
         body[:browser_worker_count] = opts[:browsers] || opts[:browser_worker_count] if opts[:browsers] || opts[:browser_worker_count]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:disable_scheduler] = opts[:disable_scheduler] if opts.has_key?("disable_scheduler") || opts.has_key?(:disable_scheduler)

data/lib/datahen/client/scraper_job.rb

@@ -8,7 +8,8 @@ module Datahen
 
       def create(scraper_name, opts={})
         body = {}
-        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:parser_worker_count] = opts[:parsers] if opts[:parsers]
+        body[:fetcher_worker_count] = opts[:fetchers] if opts[:fetchers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
@@ -36,7 +37,8 @@ module Datahen
       def update(scraper_name, opts={})
         body = {}
         body[:status] = opts[:status] if opts[:status]
-        body[:standard_worker_count] = opts[:workers] if opts[:workers]
+        body[:parser_worker_count] = opts[:parsers] if opts[:parsers]
+        body[:fetcher_worker_count] = opts[:fetchers] if opts[:fetchers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:profile] = opts[:profile] if opts[:profile]

data/lib/datahen/scraper/batch_parser.rb

@@ -152,7 +152,7 @@ module Datahen
         @page_types = []
         @parsers = Concurrent::Hash.new
         @config = YAML.load_file(config_file)
-        self.config['parsers'].each do |v|
+        (self.config['parsers'] || []).each do |v|
           next if !v['disabled'].nil? && !!v['disabled']
           @page_types << v['page_type']
           self.parsers[v['page_type']] = v['file']

data/lib/datahen/scraper/executor.rb

@@ -5,6 +5,7 @@ module Datahen
     class Executor
       # Max allowed page size when query outputs (see #find_outputs).
       MAX_FIND_OUTPUTS_PER_PAGE = 500
+      FIND_OUTPUTS_RETRY_LIMIT = 0
 
       attr_accessor :filename, :page, :gid, :job_id
 
@@ -159,13 +160,18 @@ module Datahen
         options = {
           query: query,
           page: page,
-          per_page: per_page}
+          per_page: per_page
+        }
 
         # Get job_id
         query_job_id = opts[:job_id] || get_job_id(opts[:scraper_name], self.job_id)
 
+        # find outputs
+        retry_limit = opts.has_key?(:retry_limit) ? opts[:retry_limit] : self.class::FIND_OUTPUTS_RETRY_LIMIT
         client = Client::JobOutput.new(options)
-        response = client.all(query_job_id, collection)
+        response = client.all(query_job_id, collection, {
+          retry_limit: retry_limit
+        })
 
         if response.code != 200
           raise "response_code: #{response.code}|#{response.parsed_response}"
@@ -304,6 +310,7 @@ module Datahen
           end
 
           # saving to server
+
           response = update_to_server(
             job_id: job_id,
             gid: gid,

data/lib/datahen/scraper/ruby_finisher_executor.rb

@@ -3,6 +3,8 @@ module Datahen
     class RubyFinisherExecutor < Executor
       attr_accessor :save
 
+      FIND_OUTPUTS_RETRY_LIMIT = nil
+
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
         @job_id = options[:job_id]

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -14,6 +14,8 @@ module Datahen
       # @return [Boollean]
       attr_accessor :limbo_self
 
+      FIND_OUTPUTS_RETRY_LIMIT = 2
+
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
         @page = options.fetch(:page) { nil }

data/lib/datahen/scraper/ruby_seeder_executor.rb

@@ -3,6 +3,8 @@ module Datahen
     class RubySeederExecutor < Executor
       attr_accessor :save
 
+      FIND_OUTPUTS_RETRY_LIMIT = nil
+
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
         @job_id = options[:job_id]

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.19.0"
+  VERSION = "1.0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Parama Danoesubroto
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-09-17 00:00:00.000000000 Z
+date: 2022-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -212,6 +212,7 @@ files:
 - lib/datahen.rb
 - lib/datahen/cli.rb
 - lib/datahen/cli/account.rb
+- lib/datahen/cli/account_deploy_key.rb
 - lib/datahen/cli/env_var.rb
 - lib/datahen/cli/finisher.rb
 - lib/datahen/cli/global_page.rb
@@ -276,7 +277,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://datahen.com
   source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -292,7 +293,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key:
+signing_key: 
 specification_version: 4
 summary: DataHen toolbelt for developers
 test_files: []