datahen 1.5.1 → 1.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96d2bc30d1c96ce684d83efa54b6dff5966db2a1bba7ab4856b11caba2803086
-  data.tar.gz: 985712d5d7e6559ac64b76669241f56d704c754deb06a164e1f449aad10ef29e
+  metadata.gz: beadb0e5b371134e96d8e1888250de66411eb287a30cfc5f717e525c19343d6c
+  data.tar.gz: 298daaf6f56f1fcfb4e842b9e82d0e4f8896315d264b9481aa2ed4ec55bc8a9b
 SHA512:
-  metadata.gz: d9c6bd3e60034339a8354fe4bda365b91f21b6ec68da8f384d7380abcafa5ccce2c2aacd6cc7a8da37378b8681afe58765bcc461211812c623a8958eac7a5f72
-  data.tar.gz: ac5eb5c8de4e4b0a6d28d96179bab4bf347662247b94e775ed0a25e0f0ef00a542f01f8a1a06525b565e7bd1055d5cd30b480a28d28c7ebf5de893b89b9f5e3a
+  metadata.gz: 8cba2e599b72315d65cd20440c6a8804d7298bd577d5d24f9408bc71e35f47ee5abb8e7d580b7b42cc95c10608f852339a1d44ecfedaedd82acb565f5ffb05d2
+  data.tar.gz: 6a8c1c8f7d8deca7663699f7f20ea0e9cf7e7608b2d93150fb2fd4da438780fecff010332e43675800f94012739bfcda6a85dd27df1055958e6de0bbd4c158ab

data/lib/datahen/client/base.rb

@@ -13,6 +13,9 @@ module Datahen
         finisher: nil
       }
 
+      CHECK_NIL = lambda{|v|v.nil?}
+      CHECK_EMPTY_BODY = lambda{|v|v.body.nil? || v.body.empty?}
+
       def self.env_auth_token
         ENV['DATAHEN_TOKEN']
       end
@@ -56,14 +59,14 @@ module Datahen
         target.merge(source.select{|k,v|target.has_key?(k)})
       end
 
-      def retry times, delay = nil, err_msg = nil, stream = false
+      def retry(times, delay = nil, err_msg = nil, stream = false, check_nil = CHECK_NIL)
         limit = times.nil? ? nil : times.to_i
         delay = delay.nil? ? 5 : delay.to_i
         count = 0
         begin
           val = yield
           if stream
-            return if val.nil?
+            return if check_nil.call(val)
             if val['error'] != ""
               raise StandardError.new(val['error'])
             end

data/lib/datahen/client/job.rb

@@ -60,7 +60,7 @@ module Datahen
         params = @options.merge({body: body.to_json})
 
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:seeder]
-        self.retry(limit, 5, "Error while updating the seeder.") do
+        self.retry(limit, 5, "Error while updating the seeder.", false, CHECK_EMPTY_BODY) do
           response = self.class.put("/jobs/#{job_id}/seeding_update", params)
           if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/i
             raise CustomRetryError.new(self.class.random_delay(5), response.body.to_s)
@@ -78,7 +78,7 @@ module Datahen
         params = @options.merge({body: body.to_json})
 
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:finisher]
-        self.retry(limit, 5, "Error while updating the finisher.") do
+        self.retry(limit, 5, "Error while updating the finisher.", false, CHECK_EMPTY_BODY) do
           response = self.class.put("/jobs/#{job_id}/finisher_update", params)
           if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/
             raise CustomRetryError.new(self.class.random_delay(5), response.body.to_s)

data/lib/datahen/client/job_output.rb

@@ -7,7 +7,7 @@ module Datahen
 
       def all(job_id, collection = 'default', opts = {})
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : 0
-        self.retry(limit, 10, "Error while updating the seeder.", true) do
+        self.retry(limit, 10, "Error while updating the seeder.", true, CHECK_EMPTY_BODY) do
           self.class.get("/jobs/#{job_id}/output/collections/#{collection}/records", @options)
         end
       end

data/lib/datahen/client/job_page.rb

@@ -63,7 +63,7 @@ module Datahen
         params = @options.merge({body: body.to_json})
 
         limit = opts.has_key?(:retry_limit) ? opts.fetch(:retry_limit) : self.default_retry_limit[:parser]
-        self.retry(limit, 5, "Error while updating the parser.") do
+        self.retry(limit, 5, "Error while updating the parser.", false, CHECK_EMPTY_BODY) do
           response = self.class.put("/jobs/#{job_id}/pages/#{gid}/parsing_update", params)
           if response.code == 422 && response.body.to_s =~ /pq:\s*deadlock/i
             raise Error::CustomRetryError.new(self.class.random_delay(5), response.body.to_s)

data/lib/datahen/scraper/batch_parser.rb

@@ -219,8 +219,9 @@ module Datahen
         self.dequeuer_is_alive!
 
         # ensure a valid response or try again
-        if response.body.nil? || response.body.empty? || response.response.code.to_i != 200
-          self.repeat_puts(response.nil? ? 'null' : response.body)
+        has_empty_response = (response.body.nil? || response.body.empty?)
+        if has_empty_response || response.response.code.to_i != 200
+          self.repeat_puts(has_empty_response ? 'null' : response.body)
           self.recollect_garbage
           return 0
         end

data/lib/datahen/version.rb

@@ -1,3 +1,3 @@
 module Datahen
-  VERSION = "1.5.1"
+  VERSION = "1.5.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datahen
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.5.3
 platform: ruby
 authors:
 - Parama Danoesubroto
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-01-08 00:00:00.000000000 Z
+date: 2024-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor