datahen 1.0.2 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4aa3927b9865f2815f64463f1d48b2dadddeaa73d2bd446a98ea9eb9ecb3ff5f
-  data.tar.gz: 91b7a370e740721202e9f14c043cc5c28cb9e6102dcd701c5121506042ba753b
+  metadata.gz: affc49d8b9ed853c138aba0ec2af224b7f3065de8d06f9395ab49ff91332408b
+  data.tar.gz: b2668aa8fddba8c71b1b05b5b3e492deca25e82cd8f5dd9b53428a16cd3b4d1f
 SHA512:
-  metadata.gz: c643100e60ea20686d882377b7e982829f93b4d4d8750342d47370f9d649688e94517462041b08ec1387901fcc8f33a0b0105e5f4ab43cd378dca5768cc190c4
-  data.tar.gz: ba3bae8b462aa6894520dc115452a27fcb3d64571820970ad7b69592670549824ec53daa1c8194392750f746a14ced4baf2e552eafc2684735dc185c57af8e8e
+  metadata.gz: d1c22a97ad04ce972f5c41dc1749e3613a45da9f43f31c8512703e67f2aaa9862bdb1b373eed6e8bfd2150471598e44054424b93612fd18241db1f78398230d4
+  data.tar.gz: e27c597ddc83e250bcb5edc7db8379f5498ea5636cd599fa7bfa4de8aabd6925e664ffd11aabb4389e5906e45bbca4dbf807db48094070e1dd99abffab1bd874

data/lib/datahen/cli/scraper_page.rb

@@ -131,13 +131,14 @@ module Datahen
         Reparse pages in a scraper's current job. You need to specify either a --gid or --parse-fail or --status or --page-type.\x5
       LONGDESC
       option :gid, :aliases => :g, type: :string, desc: 'Reparse a specific GID'
+      option :fetch_fail, type: :boolean, desc: 'Reparse only pages that fails fetching.'
       option :parse_fail, type: :boolean, desc: 'Reparse only pages that fails parsing.'
       option :status, type: :string, desc: 'Reparse only pages with a specific status.'
       option :page_type, type: :string, desc: 'Refetches only pages with a specific page type.'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def reparse(scraper_name)
-        if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type)
-          puts "Must specify either a --gid, --parse-fail, --status or --page-type"
+        if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type)
+          puts "Must specify either a --gid, --fetch-fail, --parse-fail, --status or --page-type"
           return
         end
 
@@ -155,11 +156,14 @@ module Datahen
         Move pages in a scraper's current job to limbo. You need to specify either a --gid or --status.\x5
       LONGDESC
       option :gid, :aliases => :g, type: :string, desc: 'Move a specific GID to limbo'
+      option :fetch_fail, type: :boolean, desc: 'Move pages that fails fetching to limbo.'
+      option :parse_fail, type: :boolean, desc: 'Move pages that fails parsing to limbo.'
       option :status, type: :string, desc: 'Move pages with a specific status to limbo.'
+      option :page_type, type: :string, desc: 'Move pages with a specific page type to limbo.'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def limbo(scraper_name)
-        if !options.key?(:gid) && !options.key?(:status)
-          puts "Must specify either a --gid or --status"
+        if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type)
+          puts "Must specify either a --gid, --fetch-fail, --parse-fail, --status or --page-type"
           return
         end
 

data/lib/datahen/scraper/ruby_finisher_executor.rb

@@ -12,10 +12,13 @@ module Datahen
 
       def self.exposed_methods
         [
+          :get_content,
+          :get_failed_content,
           :outputs,
           :save_outputs,
           :find_output,
-          :find_outputs
+          :find_outputs,
+          :finish
         ].freeze
       end
 

data/lib/datahen/scraper/ruby_parser_executor.rb

@@ -27,6 +27,8 @@ module Datahen
 
       def self.exposed_methods
         [
+          :get_content,
+          :get_failed_content,
           :content,
           :failed_content,
           :outputs,
@@ -38,7 +40,8 @@ module Datahen
           :find_outputs,
           :refetch,
           :reparse,
-          :limbo
+          :limbo,
+          :finish
         ].freeze
       end
 

data/lib/datahen/scraper/ruby_seeder_executor.rb

@@ -18,7 +18,8 @@ module Datahen
           :save_pages,
           :save_outputs,
           :find_output,
-          :find_outputs
+          :find_outputs,
+          :finish
         ].freeze
       end
 

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "1.0.2"
+  VERSION = "1.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.2
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-08-02 00:00:00.000000000 Z
+date: 2022-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor