datahen 1.1.2 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: affc49d8b9ed853c138aba0ec2af224b7f3065de8d06f9395ab49ff91332408b
-  data.tar.gz: b2668aa8fddba8c71b1b05b5b3e492deca25e82cd8f5dd9b53428a16cd3b4d1f
+  metadata.gz: bcc40ebf4e65038d925d3b234a42dce05d66e38ac98c254bc1bcf0800475a266
+  data.tar.gz: d9d021b4f17b6f409fed11de2f64092396df4adf3b382a910be3336d8f74e661
 SHA512:
-  metadata.gz: d1c22a97ad04ce972f5c41dc1749e3613a45da9f43f31c8512703e67f2aaa9862bdb1b373eed6e8bfd2150471598e44054424b93612fd18241db1f78398230d4
-  data.tar.gz: e27c597ddc83e250bcb5edc7db8379f5498ea5636cd599fa7bfa4de8aabd6925e664ffd11aabb4389e5906e45bbca4dbf807db48094070e1dd99abffab1bd874
+  metadata.gz: 05b969a63bdd90b1cfe944a303fe203e6def8c5307f072c0eb152e050c5717bec8ac464a25916fbe07ccf890abc7af6befebef831d39c968c591a630915b47b6
+  data.tar.gz: 632c4ad6a55b201bd6fae08830449958558f2d220b570b5e14aee7285b433b2e5439b46e489bfeb696417af6537b3eb636216e1eb8ecc400dbaa0a85f5f8dd8d

data/exe/hen

@@ -1,3 +1,4 @@
 #!/usr/bin/env ruby
+require 'datahen'
 require 'datahen/cli'
 Datahen::CLI.start

data/lib/datahen/cli/seeder.rb

@@ -34,7 +34,7 @@ module Datahen
           job_id = job['id']
         end
 
-        puts Datahen::Scraper::Seeder.exec_seeder(seeder_file, job_id, true)
+        puts Datahen::Scraper::Seeder.exec_seeder(seeder_file, job_id, true, options[:"keep-outputs"])
       end
     end
   end

data/lib/datahen/cli.rb

@@ -1,5 +1,5 @@
 require 'thor'
-require 'datahen/scraper'
+require 'datahen'
 require 'datahen/cli/scraper_var'
 require 'datahen/cli/scraper_exporter'
 require 'datahen/cli/scraper_export'
@@ -44,5 +44,10 @@ module Datahen
 
     desc "account SUBCOMMAND ...ARGS", "for account related activities"
     subcommand "account", Account
+
+    desc "version", "Shows the gem version"
+    def version()
+      puts "#{VERSION}"
+    end
   end
 end

data/lib/datahen/client/base.rb

@@ -21,6 +21,10 @@ module Datahen
         ENV['DATAHEN_IGNORE_SSL'].to_s.strip == '1'
       end
 
+      def self.random_delay max_seconds = 2
+        (rand * max_seconds * 1000.0).to_i / 1000.0
+      end
+
       def env_api_url
         ENV['DATAHEN_API_URL'].nil? ? 'https://app.datahen.com/api/v1' : ENV['DATAHEN_API_URL']
       end
@@ -58,11 +62,15 @@ module Datahen
         count = 0
         begin
           yield
-        rescue StandardError => e
-          STDERR.puts(e.inspect)
+        rescue Error::CustomRetryError, StandardError => e
+          is_custom_retry = e.is_a? Error::CustomRetryError
+          real_delay = is_custom_retry ? e.delay : delay
+          err_msg = is_custom_retry ? e.error : e.inspect
+          
+          STDERR.puts(err_msg)
 
           # wait before retry (default 5 sec)
-          sleep(delay) if delay > 0
+          sleep(delay) if real_delay > 0
 
           # raise error when retry limit is reached
           raise e unless limit.nil? || count < limit

data/lib/datahen/client/job.rb

@@ -57,7 +57,11 @@ module Datahen
 
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:seeder]
         self.retry(limit, 5, "Error while updating the seeder.") do
-          self.class.put("/jobs/#{job_id}/seeding_update", params)
+          response = self.class.put("/jobs/#{job_id}/seeding_update", params)
+          if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/i
+            raise CustomRetryError.new(self.class.random_delay(5), response.body.to_s)
+          end
+          response
         end
       end
 
@@ -71,7 +75,11 @@ module Datahen
 
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:finisher]
         self.retry(limit, 5, "Error while updating the finisher.") do
-          self.class.put("/jobs/#{job_id}/finisher_update", params)
+          response = self.class.put("/jobs/#{job_id}/finisher_update", params)
+          if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/
+            raise CustomRetryError.new(self.class.random_delay(5), response.body.to_s)
+          end
+          response
         end
       end
 

data/lib/datahen/client/job_page.rb

@@ -70,7 +70,11 @@ module Datahen
 
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:parser]
         self.retry(limit, 5, "Error while updating the parser.") do
-          self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
+          response = self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
+          if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/i
+            raise Error::CustomRetryError.new(self.class.random_delay(5), response.body.to_s)
+          end
+          response
         end
       end
 

data/lib/datahen/error/custom_retry_error.rb

@@ -0,0 +1,12 @@
+module Datahen
+  module Error
+    class CustomRetryError < Exception
+      attr_accessor :error, :delay
+
+      def initialize delay, error = nil
+        self.error = error
+        self.delay = delay
+      end
+    end
+  end
+end

data/lib/datahen/error.rb

@@ -1,4 +1,5 @@
 require 'datahen/error/safe_terminate_error'
+require 'datahen/error/custom_retry_error'
 
 module Datahen
   module Error

data/lib/datahen/scraper.rb

@@ -1,5 +1,3 @@
-require "datahen/error"
-require "datahen/plugin"
 require "datahen/scraper/parser"
 require "datahen/scraper/batch_parser"
 require "datahen/scraper/seeder"
@@ -8,7 +6,6 @@ require "datahen/scraper/executor"
 require "datahen/scraper/ruby_parser_executor"
 require "datahen/scraper/ruby_seeder_executor"
 require "datahen/scraper/ruby_finisher_executor"
-require "datahen/client"
 
 module Datahen
   module Scraper

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "1.1.2"
+  VERSION = "1.2.2"
 end

data/lib/datahen.rb

@@ -1,5 +1,8 @@
 require "datahen/version"
+require "datahen/error"
+require "datahen/plugin"
 require "datahen/scraper"
+require "datahen/client"
 
 module Datahen
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.2
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-08-18 00:00:00.000000000 Z
+date: 2022-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -257,6 +257,7 @@ files:
 - lib/datahen/client/scraper_job_var.rb
 - lib/datahen/client/scraper_var.rb
 - lib/datahen/error.rb
+- lib/datahen/error/custom_retry_error.rb
 - lib/datahen/error/safe_terminate_error.rb
 - lib/datahen/plugin.rb
 - lib/datahen/plugin/context_exposer.rb