datahen 0.14.17 → 0.14.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3cf1bf50610ab9ba523ca8f9ae1fdb307831cb56018036076f31353b357edfa
-  data.tar.gz: '0199ea98a2f171675168699adc523932418b9822ff156c71786458c6362b6cdb'
+  metadata.gz: 25eb02598ac32462a53995a4b9e72b3bc466b54c2d74be02516f8d04f178a7b8
+  data.tar.gz: 5f6fcedfa7f4a477e18fc1a0ee80126b1a646a3ecefdd8258d1982bf7d7fe06f
 SHA512:
-  metadata.gz: 8d66226573dbd9bd3ef795ce021eb9ee202b21cce1d0b79211093bb9abda6f1982a721bf8ae1fbc9bf84aae6e50d1878bf9d389ee67d685e4d1c440fd88cceeb
-  data.tar.gz: 6ea1a7748cf77ae1cb8f750da71df86120f68e671c84e535e8536ee3324ed71026034a042662d081108ff7b72ccb3f0ef3ab3f049bdb45c92fc0e86ba2e10d46
+  metadata.gz: 69a71d740f9078a5a4c2a77211587c0099a4064cabda690cd6fb4803c153975e3e91f1c05f98278f2852a0bacf8cb444bba8f29f56c3cfbd0fba12cece39b9cd
+  data.tar.gz: df131c11592d2b6192fa74d26fc0e8d823b99f8073b907c82c8e9f04622c7d28aa5e1145419ac0377a99f6efcd3f46ff8fcef88fc436e802d51afc014fd4383a

data/lib/datahen/cli/scraper.rb

@@ -93,6 +93,7 @@ module Datahen
       option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
       option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
+      option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: [{"name":"foo", "value":"bar", "secret":false}] '
       def start(scraper_name)
         client = Client::ScraperJob.new(options)
         puts "Starting a scrape job..."

data/lib/datahen/cli/scraper_page.rb

@@ -99,16 +99,17 @@ module Datahen
 
       desc "refetch <scraper_name>", "Refetch Pages on a scraper's current job"
       long_desc <<-LONGDESC
-        Refetch pages in a scraper's current job. You need to specify either a --gid or --fetch-fail or --parse-fail or --status.\x5
+        Refetch pages in a scraper's current job. You need to specify either a --gid or --fetch-fail or --parse-fail or --status or --page-type.\x5
       LONGDESC
       option :gid, :aliases => :g, type: :string, desc: 'Refetch a specific GID'
       option :fetch_fail, type: :boolean, desc: 'Refetches only pages that fails fetching.'
       option :parse_fail, type: :boolean, desc: 'Refetches only pages that fails parsing.'
       option :status, type: :string, desc: 'Refetches only pages with a specific status.'
+      option :page_type, type: :string, desc: 'Refetches only pages with a specific page type.'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def refetch(scraper_name)
-        if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status)
-          puts "Must specify either a --gid, --fetch-fail, --parse-fail or --status"
+        if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type)
+          puts "Must specify either a --gid, --fetch-fail, --parse-fail, --status or --page-type"
           return
         end
 
@@ -123,15 +124,16 @@ module Datahen
 
       desc "reparse <scraper_name>", "Reparse Pages on a scraper's current job"
       long_desc <<-LONGDESC
-        Reparse pages in a scraper's current job. You need to specify either a --gid or --parse-fail or --status.\x5
+        Reparse pages in a scraper's current job. You need to specify either a --gid or --parse-fail or --status or --page-type.\x5
       LONGDESC
       option :gid, :aliases => :g, type: :string, desc: 'Reparse a specific GID'
       option :parse_fail, type: :boolean, desc: 'Reparse only pages that fails parsing.'
       option :status, type: :string, desc: 'Reparse only pages with a specific status.'
+      option :page_type, type: :string, desc: 'Refetches only pages with a specific page type.'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def reparse(scraper_name)
-        if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status)
-          puts "Must specify either a --gid, --parse-fail or --status"
+        if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type)
+          puts "Must specify either a --gid, --parse-fail, --status or --page-type"
           return
         end
 

data/lib/datahen/client/scraper_job.rb

@@ -11,6 +11,13 @@ module Datahen
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        if opts[:vars]
+          if opts[:vars].is_a?(Array)
+            body[:vars] = opts[:vars]
+          elsif opts[:vars].is_a?(String)
+            body[:vars] = JSON.parse(opts[:vars])
+          end
+        end
         params = @options.merge({body: body.to_json})
         self.class.post("/scrapers/#{scraper_name}/jobs", params)
       end

data/lib/datahen/error.rb

@@ -0,0 +1,6 @@
+require 'datahen/error/safe_terminate_error'
+
+module Datahen
+  module Error
+  end
+end

data/lib/datahen/error/safe_terminate_error.rb

@@ -0,0 +1,6 @@
+module Datahen
+  module Error
+    class SafeTerminateError < Exception
+    end
+  end
+end

data/lib/datahen/scraper.rb

@@ -1,3 +1,4 @@
+require "datahen/error"
 require "datahen/plugin"
 require "datahen/scraper/parser"
 require "datahen/scraper/seeder"

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -112,7 +112,7 @@ module Datahen
         raise ArgumentError.new("page_gid needs to be a String.") unless page_gid.is_a?(String)
         if page_gid == gid
           self.refetch_self = true
-          return
+          raise Error::SafeTerminateError
         end
         refetch_page page_gid
       end
@@ -130,7 +130,7 @@ module Datahen
         raise ArgumentError.new("page_gid needs to be a String.") unless page_gid.is_a?(String)
         if page_gid == gid
           self.reparse_self = true
-          return
+          raise Error::SafeTerminateError
         end
         reparse_page page_gid
       end
@@ -153,6 +153,8 @@ module Datahen
               page: page
             })
             eval_with_context filename, context
+          rescue Error::SafeTerminateError => e
+            # do nothing, this is fine
           rescue SyntaxError => e
             handle_error(e) if save
             raise e
@@ -163,7 +165,7 @@ module Datahen
 
           puts "=========== Parsing Executed ==========="
           begin
-            save_pages_and_outputs(pages, outputs, :parsing)
+            save_pages_and_outputs(pages, outputs, :parsing) unless refetch_self
           rescue => e
             handle_error(e) if save
             raise e

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.17"
+  VERSION = "0.14.22"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.17
+  version: 0.14.22
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-20 00:00:00.000000000 Z
+date: 2020-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -233,6 +233,8 @@ files:
 - lib/datahen/client/scraper_job_page.rb
 - lib/datahen/client/scraper_job_var.rb
 - lib/datahen/client/scraper_var.rb
+- lib/datahen/error.rb
+- lib/datahen/error/safe_terminate_error.rb
 - lib/datahen/plugin.rb
 - lib/datahen/plugin/context_exposer.rb
 - lib/datahen/scraper.rb