datahen 1.1.1 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14cf0c4f60c5104527a963f2cb8ee0112db9f939cc2bb95ba8e9a1931f1781c0
-  data.tar.gz: ca0a1662af06f3ed3344524aa3f1ed2a8e7c32f7dc3d204ae344e4657ecc1b73
+  metadata.gz: 7f50f72a9ba32a9da0ecf43c684baa35c6ed84302f705ef8f87272a3ea0c9d0d
+  data.tar.gz: 5451f07252a2ba798313c02f2a3183e80d321e95d4b8eacc800b70217dd03077
 SHA512:
-  metadata.gz: 4c94feb022e23fc300fa9b7ff90e262d3425f2433ac2cb0935ff6020f96cd75a11c1f04766c7584f94921b2ea0f999f84494cb12daf22c1fe79642918210772a
-  data.tar.gz: a3591cf025d044f0f28e479f8fa883b20a91c77ecf9bdf92bec944ec2a6b61b1a14941da3e9e7964070f7b32a097874a9f9da3702de488adc87d5def1e4b9769
+  metadata.gz: 52d5f2c71f63379503b2f054b3ef2ba587d43b35d5dfa5f20c0a1c0547f62a09a1c805ce7452a37e9d064f5cec1ba3aaf833ce09eb1d6583220fcf9ae88c10a3
+  data.tar.gz: 55d3564529cc1c1a3815b936f1b0184d5e62c95af8e2c85e051430b6c38e206a27622d0c4d00e250b4db1f8cdf7c9a0276b1252335d337a888cc614bc3242790

data/exe/hen

@@ -1,3 +1,4 @@
 #!/usr/bin/env ruby
+require 'datahen'
 require 'datahen/cli'
 Datahen::CLI.start

data/lib/datahen/cli/scraper_page.rb

@@ -131,13 +131,14 @@ module Datahen
         Reparse pages in a scraper's current job. You need to specify either a --gid or --parse-fail or --status or --page-type.\x5
       LONGDESC
       option :gid, :aliases => :g, type: :string, desc: 'Reparse a specific GID'
+      option :fetch_fail, type: :boolean, desc: 'Reparse only pages that fails fetching.'
       option :parse_fail, type: :boolean, desc: 'Reparse only pages that fails parsing.'
       option :status, type: :string, desc: 'Reparse only pages with a specific status.'
       option :page_type, type: :string, desc: 'Refetches only pages with a specific page type.'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def reparse(scraper_name)
-        if !options.key?(:gid) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type)
-          puts "Must specify either a --gid, --parse-fail, --status or --page-type"
+        if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type)
+          puts "Must specify either a --gid, --fetch-fail, --parse-fail, --status or --page-type"
           return
         end
 
@@ -155,11 +156,14 @@ module Datahen
         Move pages in a scraper's current job to limbo. You need to specify either a --gid or --status.\x5
       LONGDESC
       option :gid, :aliases => :g, type: :string, desc: 'Move a specific GID to limbo'
+      option :fetch_fail, type: :boolean, desc: 'Move pages that fails fetching to limbo.'
+      option :parse_fail, type: :boolean, desc: 'Move pages that fails parsing to limbo.'
       option :status, type: :string, desc: 'Move pages with a specific status to limbo.'
+      option :page_type, type: :string, desc: 'Move pages with a specific page type to limbo.'
       option :job, :aliases => :j, type: :numeric, desc: 'Set a specific job ID'
       def limbo(scraper_name)
-        if !options.key?(:gid) && !options.key?(:status)
-          puts "Must specify either a --gid or --status"
+        if !options.key?(:gid) && !options.key?(:fetch_fail) && !options.key?(:parse_fail) && !options.key?(:status) && !options.key?(:page_type)
+          puts "Must specify either a --gid, --fetch-fail, --parse-fail, --status or --page-type"
           return
         end
 

data/lib/datahen/cli.rb

@@ -1,5 +1,5 @@
 require 'thor'
-require 'datahen/scraper'
+require 'datahen'
 require 'datahen/cli/scraper_var'
 require 'datahen/cli/scraper_exporter'
 require 'datahen/cli/scraper_export'
@@ -44,5 +44,10 @@ module Datahen
 
     desc "account SUBCOMMAND ...ARGS", "for account related activities"
     subcommand "account", Account
+
+    desc "version", "Shows the gem version"
+    def version()
+      puts "#{VERSION}"
+    end
   end
 end

data/lib/datahen/client/base.rb

@@ -21,6 +21,10 @@ module Datahen
         ENV['DATAHEN_IGNORE_SSL'].to_s.strip == '1'
       end
 
+      def self.random_delay max_seconds = 2
+        (rand * max_seconds * 1000.0).to_i / 1000.0
+      end
+
       def env_api_url
         ENV['DATAHEN_API_URL'].nil? ? 'https://app.datahen.com/api/v1' : ENV['DATAHEN_API_URL']
       end
@@ -58,11 +62,15 @@ module Datahen
         count = 0
         begin
           yield
-        rescue StandardError => e
-          STDERR.puts(e.inspect)
+        rescue Error::CustomRetryError, StandardError => e
+          is_custom_retry = e.is_a? Error::CustomRetryError
+          real_delay = is_custom_retry ? e.delay : delay
+          err_msg = is_custom_retry ? e.error : e.inspect
+          
+          STDERR.puts(err_msg)
 
           # wait before retry (default 5 sec)
-          sleep(delay) if delay > 0
+          sleep(delay) if real_delay > 0
 
           # raise error when retry limit is reached
           raise e unless limit.nil? || count < limit

data/lib/datahen/client/job.rb

@@ -57,7 +57,11 @@ module Datahen
 
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:seeder]
         self.retry(limit, 5, "Error while updating the seeder.") do
-          self.class.put("/jobs/#{job_id}/seeding_update", params)
+          response = self.class.put("/jobs/#{job_id}/seeding_update", params)
+          if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/i
+            raise CustomRetryError.new(self.class.random_delay(5), response.body.to_s)
+          end
+          response
         end
       end
 
@@ -71,7 +75,11 @@ module Datahen
 
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:finisher]
         self.retry(limit, 5, "Error while updating the finisher.") do
-          self.class.put("/jobs/#{job_id}/finisher_update", params)
+          response = self.class.put("/jobs/#{job_id}/finisher_update", params)
+          if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/
+            raise CustomRetryError.new(self.class.random_delay(5), response.body.to_s)
+          end
+          response
         end
       end
 

data/lib/datahen/client/job_page.rb

@@ -70,7 +70,11 @@ module Datahen
 
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:parser]
         self.retry(limit, 5, "Error while updating the parser.") do
-          self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
+          response = self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
+          if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/i
+            raise Error::CustomRetryError.new(self.class.random_delay(5), response.body.to_s)
+          end
+          response
         end
       end
 

data/lib/datahen/error/custom_retry_error.rb

@@ -0,0 +1,12 @@
+module Datahen
+  module Error
+    class CustomRetryError < Exception
+      attr_accessor :error, :delay
+
+      def initialize delay, error = nil
+        self.error = error
+        self.delay = delay
+      end
+    end
+  end
+end

data/lib/datahen/error.rb

@@ -1,4 +1,5 @@
 require 'datahen/error/safe_terminate_error'
+require 'datahen/error/custom_retry_error'
 
 module Datahen
   module Error

data/lib/datahen/scraper.rb

@@ -1,5 +1,3 @@
-require "datahen/error"
-require "datahen/plugin"
 require "datahen/scraper/parser"
 require "datahen/scraper/batch_parser"
 require "datahen/scraper/seeder"
@@ -8,7 +6,6 @@ require "datahen/scraper/executor"
 require "datahen/scraper/ruby_parser_executor"
 require "datahen/scraper/ruby_seeder_executor"
 require "datahen/scraper/ruby_finisher_executor"
-require "datahen/client"
 
 module Datahen
   module Scraper

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "1.1.1"
+  VERSION = "1.2.1"
 end

data/lib/datahen.rb

@@ -1,5 +1,8 @@
 require "datahen/version"
+require "datahen/error"
+require "datahen/plugin"
 require "datahen/scraper"
+require "datahen/client"
 
 module Datahen
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.1
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-08-11 00:00:00.000000000 Z
+date: 2022-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -257,6 +257,7 @@ files:
 - lib/datahen/client/scraper_job_var.rb
 - lib/datahen/client/scraper_var.rb
 - lib/datahen/error.rb
+- lib/datahen/error/custom_retry_error.rb
 - lib/datahen/error/safe_terminate_error.rb
 - lib/datahen/plugin.rb
 - lib/datahen/plugin/context_exposer.rb