datahen 0.14.15 → 0.14.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3dc2db62e50655c78bb70aceac56a3f1fb3f8ec87d155c65bbaf4e521d10fc5
-  data.tar.gz: 57de5a7b0e9271dcfbb05d7ac22df37a127cd3c427e03d93bb448359b7d30edc
+  metadata.gz: 4a2f75a84728b7e8c228c578e4a4af6253dcd19db445f509ba44f9866847936c
+  data.tar.gz: c37744322cc3e3035a31dd69bca340397ac906d27a3497604e3d98321187d972
 SHA512:
-  metadata.gz: 73888c3cabdb2c4f20371fff2f73294a71eb105a282ffff9e85ab675865fe1a7ffd8efb768cd8f0e82667158b6c249bbfef0f883088538dfc1c2c1fc4cdab527
-  data.tar.gz: e15616500c7994d3c537b42c38dd4965035f7ef825181053ad05ece46a42194d8574c322ec3f3e24121ca0c7aac2c4a9351cd93dff64814855500b04a536d61d
+  metadata.gz: fcf774827b35bf23048b47241da29b1a17e2b1d691a360d2afcb5b34c206c26b7ae316c457e26b946ae3144b4d9075dda5db90a2fd0fb7e56b9f53b0cd612d13
+  data.tar.gz: 56fe24f850c0b695b87c61629ab6939f856ecf8f0ea6b6557d59646214b6ffd87557a6ade6bf59e03b993626663cc1af3071ca10858802272d75d47cf34de56a

data/lib/datahen/cli/global_page.rb

@@ -15,7 +15,10 @@ module Datahen
 
         if result['available'] == true
           puts "Preview content url: \"#{result['preview_url']}\""
-          `open "#{result['preview_url']}"`
+          begin
+            `open "#{result['preview_url']}"`
+          rescue
+          end
         else
           puts "Content does not exist"
         end

data/lib/datahen/cli/scraper.rb

@@ -93,6 +93,7 @@ module Datahen
       option :workers, :aliases => :w, type: :numeric, desc: 'Set how many standard workers to use. Default: 1'
       option :browsers, type: :numeric, desc: 'Set how many browser workers to use. Default: 0'
       option :proxy_type, desc: 'Set the Proxy type. Default: standard'
+      option :vars, type: :string, banner: :JSON, desc: 'Set input vars. Must be in json format. i.e: {"name":"foo", "value":"bar", "secret":false} '
       def start(scraper_name)
         client = Client::ScraperJob.new(options)
         puts "Starting a scrape job..."

data/lib/datahen/cli/scraper_export.rb

@@ -36,7 +36,10 @@ module Datahen
 
         if result['signed_url']
           puts "Download url: \"#{result['signed_url']}\""
-          `open "#{result['signed_url']}"`
+          begin
+            `open "#{result['signed_url']}"`
+          rescue
+          end
         else
           puts "Exported file does not exist"
         end

data/lib/datahen/cli/scraper_page.rb

@@ -99,18 +99,20 @@ module Datahen
 
       desc "refetch <scraper_name>", "Refetch Pages on a scraper's current job"
       long_desc <<-LONGDESC
-        Refetch pages in a scraper's current job. You need to specify either a --gid or --fetch-fail or --parse-fail or --status.\x5
+        Refetch pages in a scraper's current job. You need to specify either a --gid or --fetch-fail or --parse-fail or --status or --page-type.\x5
       LONGDESC
       option :gid, :aliases => :g, type: :string, desc: 'Refetch a specific GID'
       option :fetch_fail, type: :boolean, desc: 'Refetches only pages that fails fetching.'
       option :parse_fail, type: :boolean, desc: 'Refetches only pages that fails parsing.'
       option :status, type: :string, desc: 'Refetches only pages with a specific status.'
+      option :page_type, type: :string, desc: 'Refetches only pages with a specific page type.'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def refetch(scraper_name)
-        if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status)
-          puts "Must specify either a --gid, --fetch-fail, --parse-fail or --status"
+        if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type)
+          puts "Must specify either a --gid, --fetch-fail, --parse-fail, --status or --page-type"
           return
         end
+
         if options[:job]
           client = Client::JobPage.new(options)
           puts "#{client.refetch(options[:job])}"
@@ -122,33 +124,47 @@ module Datahen
 
       desc "reparse <scraper_name>", "Reparse Pages on a scraper's current job"
       long_desc <<-LONGDESC
-        Reparse pages in a scraper's current job. You need to specify either a --gid or --parse-fail or --status.\x5
+        Reparse pages in a scraper's current job. You need to specify either a --gid or --parse-fail or --status or --page-type.\x5
       LONGDESC
       option :gid, :aliases => :g, type: :string, desc: 'Reparse a specific GID'
       option :parse_fail, type: :boolean, desc: 'Reparse only pages that fails parsing.'
       option :status, type: :string, desc: 'Reparse only pages with a specific status.'
+      option :page_type, type: :string, desc: 'Refetches only pages with a specific page type.'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def reparse(scraper_name)
-        begin
-          options[:vars] = JSON.parse(options[:vars]) if options[:vars]
+        if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type)
+          puts "Must specify either a --gid, --parse-fail, --status or --page-type"
+          return
+        end
 
-          if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status)
-            puts "Must specify either a --gid, --parse-fail or --status"
-            return
-          end
+        if options[:job]
+          client = Client::JobPage.new(options)
+          puts "#{client.reparse(options[:job])}"
+        else
+          client = Client::ScraperJobPage.new(options)
+          puts "#{client.reparse(scraper_name)}"
+        end
+      end
 
-          if options[:job]
-            client = Client::JobPage.new(options)
-            puts "#{client.reparse(options[:job])}"
-          else
-            client = Client::ScraperJobPage.new(options)
-            puts "#{client.reparse(scraper_name)}"
-          end
+      desc "limbo <scraper_name>", "Move pages on a scraper's current job to limbo"
+      long_desc <<-LONGDESC
+        Move pages in a scraper's current job to limbo. You need to specify either a --gid or --status.\x5
+      LONGDESC
+      option :gid, :aliases => :g, type: :string, desc: 'Move a specific GID to limbo'
+      option :status, type: :string, desc: 'Move pages with a specific status to limbo.'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def limbo(scraper_name)
+        if !options.key?(:gid) && !options.key?(:status)
+          puts "Must specify either a --gid or --status"
+          return
+        end
 
-        rescue JSON::ParserError
-          if options[:vars]
-            puts "Error: #{options[:vars]} on vars is not a valid JSON"
-          end
+        if options[:job]
+          client = Client::JobPage.new(options)
+          puts "#{client.limbo(options[:job])}"
+        else
+          client = Client::ScraperJobPage.new(options)
+          puts "#{client.limbo(scraper_name)}"
         end
       end
 
@@ -224,7 +240,10 @@ module Datahen
 
         if result['available'] == true
           puts "Preview content url: \"#{result['preview_url']}\""
-          `open "#{result['preview_url']}"`
+          begin
+            `open "#{result['preview_url']}"`
+          rescue
+          end
         else
           puts "Content does not exist"
         end
@@ -244,7 +263,10 @@ module Datahen
 
         if result['available'] == true
           puts "Preview failed content url: \"#{result['preview_url']}\""
-          `open "#{result['preview_url']}"`
+          begin
+            `open "#{result['preview_url']}"`
+          rescue
+          end
         else
           puts "Failed Content does not exist"
         end

data/lib/datahen/client/job_page.rb

@@ -72,6 +72,11 @@ module Datahen
         params = @options.merge(opts)
         self.class.put("/jobs/#{job_id}/pages/refetch", params)
       end
+
+      def limbo(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/limbo", params)
+      end
     end
   end
 end

data/lib/datahen/client/scraper_job.rb

@@ -11,6 +11,13 @@ module Datahen
         body[:standard_worker_count] = opts[:workers] if opts[:workers]
         body[:browser_worker_count] = opts[:browsers] if opts[:browsers]
         body[:proxy_type] = opts[:proxy_type] if opts[:proxy_type]
+        if opts[:vars]
+          if opts[:vars].is_a?(Array)
+            body[:vars] = opts[:vars]
+          elsif opts[:vars].is_a?(String)
+            body[:vars] = JSON.parse(opts[:vars])
+          end
+        end
         params = @options.merge({body: body.to_json})
         self.class.post("/scrapers/#{scraper_name}/jobs", params)
       end

data/lib/datahen/client/scraper_job_page.rb

@@ -26,7 +26,7 @@ module Datahen
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", params)
       end
 
-      # Deprecated, please use Datahen::Client::JobVar#refetch instead.
+      # Deprecated, please use Datahen::Client::JobPage#refetch instead.
       #
       # @note This method will be removed at some point in the future.
       def refetch_by_job(job_id, opts={})
@@ -39,6 +39,11 @@ module Datahen
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", params)
       end
 
+      def limbo(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.put("/scrapers/#{scraper_name}/current_job/pages/limbo", params)
+      end
+
       def enqueue(scraper_name, method, url, opts={})
         body = {}
         body[:method] =  method != "" ? method : "GET"

data/lib/datahen/error.rb

@@ -0,0 +1,6 @@
+require 'datahen/error/safe_terminate_error'
+
+module Datahen
+  module Error
+  end
+end

data/lib/datahen/error/safe_terminate_error.rb

@@ -0,0 +1,6 @@
+module Datahen
+  module Error
+    class SafeTerminateError < Exception
+    end
+  end
+end

data/lib/datahen/scraper.rb

@@ -1,3 +1,4 @@
+require "datahen/error"
 require "datahen/plugin"
 require "datahen/scraper/parser"
 require "datahen/scraper/seeder"

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -112,7 +112,7 @@ module Datahen
         raise ArgumentError.new("page_gid needs to be a String.") unless page_gid.is_a?(String)
         if page_gid == gid
           self.refetch_self = true
-          return
+          raise Error::SafeTerminateError
         end
         refetch_page page_gid
       end
@@ -130,7 +130,7 @@ module Datahen
         raise ArgumentError.new("page_gid needs to be a String.") unless page_gid.is_a?(String)
         if page_gid == gid
           self.reparse_self = true
-          return
+          raise Error::SafeTerminateError
         end
         reparse_page page_gid
       end
@@ -153,6 +153,8 @@ module Datahen
               page: page
             })
             eval_with_context filename, context
+          rescue Error::SafeTerminateError => e
+            # do nothing, this is fine
           rescue SyntaxError => e
             handle_error(e) if save
             raise e
@@ -163,7 +165,7 @@ module Datahen
 
           puts "=========== Parsing Executed ==========="
           begin
-            save_pages_and_outputs(pages, outputs, :parsing)
+            save_pages_and_outputs(pages, outputs, :parsing) unless refetch_self
           rescue => e
             handle_error(e) if save
             raise e

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.15"
+  VERSION = "0.14.20"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.15
+  version: 0.14.20
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-07 00:00:00.000000000 Z
+date: 2020-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -233,6 +233,8 @@ files:
 - lib/datahen/client/scraper_job_page.rb
 - lib/datahen/client/scraper_job_var.rb
 - lib/datahen/client/scraper_var.rb
+- lib/datahen/error.rb
+- lib/datahen/error/safe_terminate_error.rb
 - lib/datahen/plugin.rb
 - lib/datahen/plugin/context_exposer.rb
 - lib/datahen/scraper.rb