datahen 0.16.2 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8945724e5d11f40eba22a9ffca7ca7d024b8565dae4ba1d3da9e486eac262575
-  data.tar.gz: 867319a0c6358c593951e6241b24d9c3fd9dcb759ce1287eeb640ade0c23e69a
+  metadata.gz: 11c43658f61faff627da448abff393b5cc658c15e1dfd6765c6cfdee96958d01
+  data.tar.gz: 5dc4481c9755d33dcee2539a24fe1ba2b0336fc4320beaa54204d1636067ee75
 SHA512:
-  metadata.gz: bad8dea41951df061c84934fa63a0594bc8caf22b8665c3d6746bb29ca2821103ecd3814a9a9eba020009e8165390900b1c45a1fd0a7175fb7ea52f7a077fdab
-  data.tar.gz: 74342e01eaa21a590ef998219282fb40342cde4bf8db58617d24583bfa00b967df3f8c38e41c9b92868e90113b7cc5f6346fb2062b446edd3bb4612e053c5da7
+  metadata.gz: e25db9321dfb26cb88d38aa9adf353c9955c3932553fb87affe235bd58f0182e7043775d102ce2bb5e7d6d0764cd76bd71f3be024ced38eadd17c05cfa0efd69
+  data.tar.gz: b07e2655df12424db7f859da29f0921fa6417dfbfa6b72537615089d6fabac7451e3e61e23ba4f4e8cfff0c0555ef50bf751dd8b91b8be8ae51b23d855145586

data/lib/datahen/cli/account.rb

@@ -11,6 +11,9 @@ module Datahen
         puts "#{client.profile()}"
       end
 
+      desc "deploy_key SUBCOMMAND ...ARGS", "manage deploy key"
+      subcommand "deploy_key", AccountDeployKey
+
     end
   end
 

data/lib/datahen/cli/account_deploy_key.rb

@@ -0,0 +1,26 @@
+module Datahen
+  class CLI < Thor
+    class AccountDeployKey < Thor
+      package_name "account deploy_key"
+      def self.banner(command, namespace = nil, subcommand = false)
+        "#{basename} #{@package_name} #{command.usage}"
+      end
+
+      desc "show", "Show public deploy key"
+      def show()
+        client = Client::DeployKey.new()
+        puts "#{client.find()}"
+      end
+
+      desc "recreate", "Recreate public deploy key"
+      long_desc <<-LONGDESC
+        Recreate public deploy key.
+      LONGDESC
+      def recreate()
+        client = Client::DeployKey.new()
+        puts "#{client.create()}"
+      end
+    end
+  end
+
+end

data/lib/datahen/cli/scraper.rb

@@ -8,6 +8,7 @@ module Datahen
       LONGDESC
       option :page, :aliases => :p, type: :numeric, desc: 'Get the next set of records by page.'
       option :per_page, :aliases => :P, type: :numeric, desc: 'Number of records per page. Max 500 per page.'
+      option :status, :aliases => :s, type: :string, desc: 'Scraper status. Status can be: done, cancelled, paused, finishing.'
       def list
         client = Client::Scraper.new(options)
         puts "#{client.all}"
@@ -33,6 +34,8 @@ module Datahen
       option :multiple_jobs, type: :boolean, desc: 'Set true to enable multiple jobs. Default: false'
       option :max_job_count, type: :numeric, desc: 'Set a value to set max number of jobs available. Set -1 for unlimited. Default: 3'
       option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
+      option :enable_global_cache, type: :boolean, desc: 'Set true to enable page cache. Default: false'
+      option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
       def create(scraper_name, git_repository)
         # puts "options #{options}"
         client = Client::Scraper.new(options)
@@ -59,6 +62,8 @@ module Datahen
       option :multiple_jobs, type: :boolean, desc: 'Set true to enable multiple jobs. Default: false'
       option :max_job_count, type: :numeric, desc: 'Set a value to set max number of jobs available. Set -1 for unlimited. Default: 3'
       option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
+      option :enable_global_cache, type: :boolean, desc: 'Set true to enable page cache. Default: false'
+      option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
       def update(scraper_name)
         client = Client::Scraper.new(options)
         puts "#{client.update(scraper_name, options)}"
@@ -97,6 +102,7 @@ module Datahen
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
       option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: [{"name":"foo", "value":"bar", "secret":false}] '
       option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
+      option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
       def start(scraper_name)
         client = Client::ScraperJob.new(options)
         puts "Starting a scrape job..."

data/lib/datahen/cli/scraper_job.rb

@@ -105,6 +105,8 @@ module Datahen
       option :profile, type: :string, desc: 'Set the profiles (comma separated) to apply to the job. Default: default'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       option :max_page_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
+      option :enable_global_cache, type: :boolean, desc: 'Set true to enable page cache. Default: false'
+      option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
       def update(scraper_name)
         if options[:job]
           client = Client::Job.new(options)

data/lib/datahen/cli/scraper_page.rb

@@ -46,6 +46,7 @@ module Datahen
       option :ua_type, :aliases => :u, desc: 'Set user agent type. Default: desktop'
       option :no_redirect, :aliases => :n, type: :boolean, desc: 'Set true to not follow redirect. Default: false'
       option :max_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
+      option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
       def add(scraper_name, url)
         begin
           options[:headers] = JSON.parse(options[:headers]) if options[:headers]
@@ -80,6 +81,7 @@ module Datahen
       option :priority, type: :numeric, desc: 'Set fetch priority. The higher the value, the sooner the page gets fetched. Default: 0'
       option :vars, :aliases => :v, type: :string, desc: 'Set user-defined page variables. Must be in json format. i.e: {"Foo":"bar"}'
       option :max_size, type: :numeric, desc: 'Set a value to set max page size when fetching a page. Set a value grather than 0 to set it as limit, 0 means any size. Default: 0'
+      option :retry_interval, type: :numeric, desc: 'Set a value to set retry time interval on seconds when refetching a page. Set a value grather than 0 to set it as new time to refetch, 0 means default time. Default: 0'
       def update(scraper_name, gid)
         begin
           options[:vars] = JSON.parse(options[:vars]) if options[:vars]

data/lib/datahen/cli.rb

@@ -16,10 +16,9 @@ require 'datahen/cli/parser'
 require 'datahen/cli/seeder'
 require 'datahen/cli/finisher'
 require 'datahen/cli/env_var'
+require 'datahen/cli/account_deploy_key'
 require 'datahen/cli/account'
 
-
-
 module Datahen
   class CLI < Thor
     desc "scraper SUBCOMMAND ...ARGS", "manage scrapers"

data/lib/datahen/client/job.rb

@@ -22,6 +22,8 @@ module Datahen
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:profile] = opts[:profile] if opts[:profile]
         body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
+        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
+        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/jobs/#{job_id}", params)

data/lib/datahen/client/job_page.rb

@@ -16,6 +16,8 @@ module Datahen
         body[:priority] = opts[:priority] if opts[:priority]
         body[:vars] = opts[:vars] if opts[:vars]
         body[:max_size] = opts[:max_size] if opts[:max_size]
+        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
+        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
 
         params = @options.merge({body: body.to_json})
 
@@ -38,6 +40,8 @@ module Datahen
         body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
         body[:cookie] = opts[:cookie] if opts[:cookie]
         body[:max_size] = opts[:max_size] if opts[:max_size]
+        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
+        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
 
         params = @options.merge({body: body.to_json})
 

data/lib/datahen/client/scraper.rb

@@ -29,6 +29,8 @@ module Datahen
         body[:multiple_jobs] = opts[:multiple_jobs] if opts[:multiple_jobs]
         body[:max_job_count] = opts[:max_job_count] if opts[:max_job_count]
         body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
+        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
+        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
         params = @options.merge({body: body.to_json})
         self.class.post("/scrapers", params)
       end
@@ -51,6 +53,8 @@ module Datahen
         body[:multiple_jobs] = opts[:multiple_jobs] if opts.has_key?("multiple_jobs") || opts.has_key?(:multiple_jobs)
         body[:max_job_count] = opts[:max_job_count] if opts.has_key?("max_job_count") || opts.has_key?(:max_job_count)
         body[:max_page_size] = opts[:max_page_size] if opts.has_key?("max_page_size") || opts.has_key?(:max_page_size)
+        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
+        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}", params)

data/lib/datahen/client/scraper_job.rb

@@ -12,6 +12,8 @@ module Datahen
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
+        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
+        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
         if opts[:vars]
           if opts[:vars].is_a?(Array)
             body[:vars] = opts[:vars]
@@ -39,6 +41,8 @@ module Datahen
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
         body[:profile] = opts[:profile] if opts[:profile]
         body[:max_page_size] = opts[:max_page_size] if opts[:max_page_size]
+        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
+        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
         params = @options.merge({body: body.to_json})
 
         self.class.put("/scrapers/#{scraper_name}/current_job", params)

data/lib/datahen/client/scraper_job_page.rb

@@ -16,6 +16,8 @@ module Datahen
         body[:priority] = opts[:priority] if opts[:priority]
         body[:vars] = opts[:vars] if opts[:vars]
         body[:max_size] = opts[:max_size] if opts[:max_size]
+        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
+        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
 
         params = @options.merge({body: body.to_json})
 
@@ -61,6 +63,8 @@ module Datahen
         body[:no_redirect] = opts[:no_redirect] if opts[:no_redirect]
         body[:cookie] = opts[:cookie] if opts[:cookie]
         body[:max_size] = opts[:max_size] if opts[:max_size]
+        body[:enable_global_cache] = opts[:enable_global_cache] if opts.has_key?("enable_global_cache") || opts.has_key?(:enable_global_cache)
+        body[:retry_interval] = opts[:retry_interval] if opts[:retry_interval]
 
         params = @options.merge({body: body.to_json})
 

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -8,7 +8,11 @@ module Datahen
       attr_accessor :refetch_self
       # Reparse self page flag.
       # @return [Boollean]
+      # @note It is stronger than #limbo_self flag.
       attr_accessor :reparse_self
+      # Limbo self page flag.
+      # @return [Boollean]
+      attr_accessor :limbo_self
 
       def initialize(options={})
         @filename = options.fetch(:filename) { raise "Filename is required"}
@@ -31,7 +35,8 @@ module Datahen
           :find_output,
           :find_outputs,
           :refetch,
-          :reparse
+          :reparse,
+          :limbo
         ].freeze
       end
 
@@ -104,7 +109,7 @@ module Datahen
 
       def refetch_page gid
         if save
-          Client::ScraperJobPage.new({gid: gid}).refetch_by_job(self.job_id)
+          Client::JobPage.new({gid: gid}).refetch(self.job_id)
           puts "Refetch page #{gid}"
         else
           puts "Would have refetch page #{gid}"
@@ -122,7 +127,7 @@ module Datahen
 
       def reparse_page gid
         if save
-          Client::ScraperJobPage.new({gid: gid}).reparse_by_job(self.job_id)
+          Client::JobPage.new({gid: gid}).reparse(self.job_id)
           puts "Reparse page #{gid}"
         else
           puts "Would have reparse page #{gid}"
@@ -138,6 +143,24 @@ module Datahen
         reparse_page page_gid
       end
 
+      def limbo_page gid
+        if save
+          Client::JobPage.new({gid: gid}).limbo(self.job_id)
+          puts "Limbo page #{gid}"
+        else
+          puts "Would have limbo page #{gid}"
+        end
+      end
+
+      def limbo page_gid
+        raise ArgumentError.new("page_gid needs to be a String.") unless page_gid.is_a?(String)
+        if page_gid == gid
+          self.limbo_self = true
+          raise Error::SafeTerminateError
+        end
+        limbo_page page_gid
+      end
+
       def eval_parser_script(save=false)
         update_parsing_starting_status
 
@@ -148,6 +171,7 @@ module Datahen
           page = init_page_vars(page)
           self.refetch_self = false
           self.reparse_self = false
+          self.limbo_self = false
 
           begin
             context = isolated_binding({
@@ -178,6 +202,8 @@ module Datahen
             refetch_page gid
           elsif reparse_self
             reparse_page gid
+          elsif limbo_self
+            limbo_page gid
           else
             update_parsing_done_status
           end

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.16.2"
+  VERSION = "0.20.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.16.2
+  version: 0.20.0
 platform: ruby
 authors:
 - Parama Danoesubroto
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-11 00:00:00.000000000 Z
+date: 2021-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -212,6 +212,7 @@ files:
 - lib/datahen.rb
 - lib/datahen/cli.rb
 - lib/datahen/cli/account.rb
+- lib/datahen/cli/account_deploy_key.rb
 - lib/datahen/cli/env_var.rb
 - lib/datahen/cli/finisher.rb
 - lib/datahen/cli/global_page.rb
@@ -276,7 +277,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://datahen.com
   source_code_uri: https://github.com/DataHenOfficial/datahen-ruby
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -292,7 +293,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: DataHen toolbelt for developers
 test_files: []