internetdata 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 001e35eb3c0fef97cb13d9cdb22465c663d215253e299bbff9e61cf016b2aa0a
-  data.tar.gz: aeca5173f71e178c054e3829e89a2715c1d409bc67e352f80d0160eba1c83c86
+  metadata.gz: 6341d4fee4674f684594cba1597efad23a7b83d36117a126bc46db283e562320
+  data.tar.gz: c5e0c83a281b4ff2d51a45021cd9f5b31263c654c2197d8422ce728a3f062966
 SHA512:
-  metadata.gz: 842beb45ab5b427b07dd985412d11cf6cfebd5325c1128e0846fa8d720ab475a0ca66aaf42a557b96675b678b854aa4ce38d50f3edbac9b4c70f3e0b7e3f1554
-  data.tar.gz: 5d39b5cac31bbba68c748aea0d05e5e4d24fb8153d14a46cfa9f3ec76b6aa0068a5638e58944138d65440da9f494c56d90d30dc516658b841b87e6764cf33e3e
+  metadata.gz: 778d2a234d36fb5c9e783fbfdb63df0b1f9bff7f069db19fb677957bd3d36d9ff46dd582a3bb28ab3fb43d31a150f76f538bc80a5cf4decb032e990393bf63ca
+  data.tar.gz: e927fa802dfb5b852aa9a3612fe7de4c47ee0b3e7cbe7704c043bd4864c79f5e6aebb00c9e4bb01ef427b13384b7c7eeaa6b0f8581603e87070557f00c11682f

data/lib/internetdata/database_api.rb

@@ -77,11 +77,7 @@ module InternetData
       partial = "#{path}.part"
       begin
         url = download_url(id, format)
-        written = Retries.with_retries(@retries) do
-          # Reopened per attempt, so a retry restarts the file rather than
-          # appending a second copy of the body to a half-written one.
-          File.open(partial, 'wb') { |file| stream(url) { |chunk| file.write(chunk) } }
-        end
+        written = File.open(partial, 'wb') { |file| transfer(url) { |chunk| file.write(chunk) } }
         File.rename(partial, path)
       rescue StandardError
         File.delete(partial) if File.exist?(partial)
@@ -99,15 +95,27 @@ module InternetData
     # not measured with {#metadata}.
     def download_bytes(id, format)
       url = download_url(id, format)
-      Retries.with_retries(@retries) do
-        bytes = String.new(encoding: Encoding::BINARY)
-        stream(url) { |chunk| bytes << chunk }
-        bytes
-      end
+      bytes = String.new(encoding: Encoding::BINARY)
+      transfer(url) { |chunk| bytes << chunk }
+      bytes
     end
 
     private
 
+    # The transfer of a presigned link, retried only while nothing has reached the
+    # block: object storage failing before the body is as transient as any
+    # outage, while a body that dies part way is not fetched again, because the
+    # bytes already handed over cannot be taken back.
+    def transfer(url, &sink)
+      delivered = false
+      Retries.with_retries(@retries, retry_if: -> { !delivered }) do
+        stream(url) do |chunk|
+          delivered = true
+          sink.call(chunk)
+        end
+      end
+    end
+
     # Runs one transfer of a presigned link, handing each chunk to the block, and
     # returns the bytes that reached it.
     #

data/lib/internetdata/retries.rb

@@ -11,12 +11,14 @@ module InternetData
 
     module_function
 
-    def with_retries(retries)
+    # `retry_if`, when given, is asked after each retryable failure, and a false
+    # answer ends the attempts there.
+    def with_retries(retries, retry_if: nil)
       attempt = 0
       begin
         yield
       rescue Error => e
-        raise unless e.retryable? && attempt < retries
+        raise unless e.retryable? && attempt < retries && (retry_if.nil? || retry_if.call)
 
         attempt += 1
         sleep(delay_for(e, attempt))

data/lib/internetdata/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module InternetData
-  VERSION = '2.0.0'
+  VERSION = '2.0.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: internetdata
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Mslm Dev