datahen 0.14.16 → 0.14.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 206ac0cf7cc5dc64deecff24f1d731f353951390c4c96d3cc6721a4dd0ecb258
-  data.tar.gz: 00d8e528649b2296d585d5008ebbf2de62d100449e5841f4f220147fcebc2484
+  metadata.gz: a3cf1bf50610ab9ba523ca8f9ae1fdb307831cb56018036076f31353b357edfa
+  data.tar.gz: '0199ea98a2f171675168699adc523932418b9822ff156c71786458c6362b6cdb'
 SHA512:
-  metadata.gz: 2c66c9527f38db20a7f0ed589c560a2dbafdd038e1834b8552305cd665fec8063fab3c6dcd31eab9df07f2bc04ce5682272bea61772f0dfac20cf84f05b4be35
-  data.tar.gz: 6c327375e3e1eba2235194f0b58ab280b5ecbbb36fd0cf4808d9cf89b539240d0fead54ee8a4019bd0f196ddd2c7621b2ace8c1dd2da80ba6b665b1ca41a015f
+  metadata.gz: 8d66226573dbd9bd3ef795ce021eb9ee202b21cce1d0b79211093bb9abda6f1982a721bf8ae1fbc9bf84aae6e50d1878bf9d389ee67d685e4d1c440fd88cceeb
+  data.tar.gz: 6ea1a7748cf77ae1cb8f750da71df86120f68e671c84e535e8536ee3324ed71026034a042662d081108ff7b72ccb3f0ef3ab3f049bdb45c92fc0e86ba2e10d46

data/lib/datahen/cli/scraper_page.rb

@@ -111,6 +111,7 @@ module Datahen
           puts "Must specify either a --gid, --fetch-fail, --parse-fail or --status"
           return
         end
+
         if options[:job]
           client = Client::JobPage.new(options)
           puts "#{client.refetch(options[:job])}"
@@ -129,26 +130,39 @@ module Datahen
       option :status, type: :string, desc: 'Reparse only pages with a specific status.'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def reparse(scraper_name)
-        begin
-          options[:vars] = JSON.parse(options[:vars]) if options[:vars]
+        if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status)
+          puts "Must specify either a --gid, --parse-fail or --status"
+          return
+        end
 
-          if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status)
-            puts "Must specify either a --gid, --parse-fail or --status"
-            return
-          end
+        if options[:job]
+          client = Client::JobPage.new(options)
+          puts "#{client.reparse(options[:job])}"
+        else
+          client = Client::ScraperJobPage.new(options)
+          puts "#{client.reparse(scraper_name)}"
+        end
+      end
 
-          if options[:job]
-            client = Client::JobPage.new(options)
-            puts "#{client.reparse(options[:job])}"
-          else
-            client = Client::ScraperJobPage.new(options)
-            puts "#{client.reparse(scraper_name)}"
-          end
+      desc "limbo <scraper_name>", "Move pages on a scraper's current job to limbo"
+      long_desc <<-LONGDESC
+        Move pages in a scraper's current job to limbo. You need to specify either a --gid or --status.\x5
+      LONGDESC
+      option :gid, :aliases => :g, type: :string, desc: 'Move a specific GID to limbo'
+      option :status, type: :string, desc: 'Move pages with a specific status to limbo.'
+      option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
+      def limbo(scraper_name)
+        if !options.key?(:gid) && !options.key?(:status)
+          puts "Must specify either a --gid or --status"
+          return
+        end
 
-        rescue JSON::ParserError
-          if options[:vars]
-            puts "Error: #{options[:vars]} on vars is not a valid JSON"
-          end
+        if options[:job]
+          client = Client::JobPage.new(options)
+          puts "#{client.limbo(options[:job])}"
+        else
+          client = Client::ScraperJobPage.new(options)
+          puts "#{client.limbo(scraper_name)}"
         end
       end
 

data/lib/datahen/client/job_page.rb

@@ -72,6 +72,11 @@ module Datahen
         params = @options.merge(opts)
         self.class.put("/jobs/#{job_id}/pages/refetch", params)
       end
+
+      def limbo(job_id, opts={})
+        params = @options.merge(opts)
+        self.class.put("/jobs/#{job_id}/pages/limbo", params)
+      end
     end
   end
 end

data/lib/datahen/client/scraper_job_page.rb

@@ -26,7 +26,7 @@ module Datahen
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/refetch", params)
       end
 
-      # Deprecated, please use Datahen::Client::JobVar#refetch instead.
+      # Deprecated, please use Datahen::Client::JobPage#refetch instead.
       #
       # @note This method will be removed at some point in the future.
       def refetch_by_job(job_id, opts={})
@@ -39,6 +39,11 @@ module Datahen
         self.class.put("/scrapers/#{scraper_name}/current_job/pages/reparse", params)
       end
 
+      def limbo(scraper_name, opts={})
+        params = @options.merge(opts)
+        self.class.put("/scrapers/#{scraper_name}/current_job/pages/limbo", params)
+      end
+
       def enqueue(scraper_name, method, url, opts={})
         body = {}
         body[:method] =  method != "" ? method : "GET"

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "0.14.16"
+  VERSION = "0.14.17"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 0.14.16
+  version: 0.14.17
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-14 00:00:00.000000000 Z
+date: 2020-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -266,7 +266,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: DataHen toolbelt for developers