internetdata 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 001e35eb3c0fef97cb13d9cdb22465c663d215253e299bbff9e61cf016b2aa0a
-  data.tar.gz: aeca5173f71e178c054e3829e89a2715c1d409bc67e352f80d0160eba1c83c86
+  metadata.gz: 4c4799411a10284db486537f26bec70503a77692f56751121b34ec4bf89b2b22
+  data.tar.gz: db8745ef831d08fbfec78824c184acc194f1eb4bc62ba2e741f530ffff76fd40
 SHA512:
-  metadata.gz: 842beb45ab5b427b07dd985412d11cf6cfebd5325c1128e0846fa8d720ab475a0ca66aaf42a557b96675b678b854aa4ce38d50f3edbac9b4c70f3e0b7e3f1554
-  data.tar.gz: 5d39b5cac31bbba68c748aea0d05e5e4d24fb8153d14a46cfa9f3ec76b6aa0068a5638e58944138d65440da9f494c56d90d30dc516658b841b87e6764cf33e3e
+  metadata.gz: 69292ca30d27411851f776f88a067176a14ef39331f5344dbd30ae345ca369357456b7bf353323f5c7e9f323f695600da02b9723e6a1e663806d36972f8e67a8
+  data.tar.gz: 9d5aa3458e1199c8ed121ecca2a1418dcab23c0725dac8a584f51ffa8ff13b0c1d4f7ac7b4d7a13408e5b928b25c3966a00d1f235ad768d113c3ef5513438762

data/README.md

@@ -23,7 +23,7 @@ Requires Ruby 3.1 or newer.
 
 ## Usage
 
-Every database published today needs an API key carrying the `db.download` scope. Access is granted by contract, one family at a time, so there is no self-serve tier: write to [dev@internetdata.io](mailto:dev@internetdata.io) to be licensed and issued a key. `api_key:` is nevertheless optional - a client built without one sends no `Authorization` header at all, ready for a database served without a licence.
+Every database published today needs an API key carrying the `db.download` scope. Access is granted by contract, one family at a time, so there is no self-serve tier: write to [dev@internetdata.io](mailto:dev@internetdata.io) to be licensed and issued a key. `api_key:` is nevertheless optional - a client built without one sends no `Authorization` header at all, ready for a database served without a license.
 
 ```ruby
 require 'internetdata'
@@ -92,6 +92,18 @@ client.database.downloads(limit: 20).each do |attempt|
 end
 ```
 
+### Timeouts
+
+```ruby
+client = InternetData::Client.new(api_key: ENV['INTERNETDATA_API_KEY'], timeout: 10)
+
+catalog = client.database.list(timeout: 5)
+```
+
+`timeout` is in seconds and bounds each attempt, body included, so a call that is retried can take longer in total. It defaults to 30 seconds. The client's value is the default: from 2.1.0, `list`, `metadata`, `checksums`, `downloads` and `download_url` each take `timeout:` for that call alone.
+
+`download` and `download_bytes` take no `timeout:` and raise `ArgumentError` if handed one, rather than accepting it and quietly doing nothing: a transfer runs to gigabytes and minutes, so any bound that suits a JSON call would abandon a healthy download. They bound only their connection with the client's value. `download_url` does take one, because minting the link is an ordinary API request - it bounds that request, not whatever you do with the link afterwards.
+
 ### Errors
 
 Failures raise an `InternetData::Error` carrying a `kind` and a `retryable?` flag:

data/lib/internetdata/client.rb

@@ -12,7 +12,7 @@ module InternetData
   # Every database published today needs an API key carrying the `db.download`
   # scope, granted by contract one family at a time. The key is still OPTIONAL:
   # a client built without one sends no `Authorization` header at all, which is
-  # what a database served without a licence would need.
+  # what a database served without a license would need.
   class Client
     # The licensed database downloads, and everything about them.
     attr_reader :database

data/lib/internetdata/database_api.rb

@@ -5,6 +5,12 @@ module InternetData
   #
   # Not to be confused with {Database}, which is one entry in a {#list}, nor with
   # the generated {DatabaseV2Api} underneath, which speaks the wire.
+  #
+  # Every JSON call here takes `timeout:`, in seconds, bounding each ATTEMPT of
+  # that call alone and overriding the bound the client was built with. The two
+  # transfers take none, and are refused it rather than ignoring it: a database
+  # runs to gigabytes and minutes, so a bound that suits a JSON call would
+  # abandon a healthy download.
   class DatabaseApi
     def initialize(transport, retries:)
       @transport = transport
@@ -25,8 +31,10 @@ module InternetData
     # server decides that per key. So the catalog is not the same for every key,
     # a listing fetched with one key says nothing about another, and there is no
     # other source to reconstruct it from.
-    def list
-      call { @api.list_databases.databases }
+    #
+    # @param timeout [Numeric, nil] seconds this attempt may take, for THIS call only.
+    def list(timeout: nil)
+      call { @api.list_databases(timeout: timeout).databases }
     end
 
     # What is inside one database: schema, sample rows, row count and sizes.
@@ -34,8 +42,10 @@ module InternetData
     # `updated` and `entries` answer whether today's build is worth fetching, and
     # `size` is bytes per format, which is what a transfer should be budgeted
     # against before it starts.
-    def metadata(id)
-      call { @api.database_metadata_v2(id) }
+    #
+    # @param timeout [Numeric, nil] seconds this attempt may take, for THIS call only.
+    def metadata(id, timeout: nil)
+      call { @api.database_metadata_v2(id, timeout: timeout) }
     end
 
     # The digests for one published file.
@@ -43,15 +53,19 @@ module InternetData
     # Returns the whole set rather than one algorithm: which digests a database
     # publishes is the API's choice, not ours, and the response nests them one
     # level down under `checksums`.
-    def checksums(id, format)
+    #
+    # @param timeout [Numeric, nil] seconds this attempt may take, for THIS call only.
+    def checksums(id, format, timeout: nil)
       check_format!(format)
-      call { @api.database_checksum_v2(id, format).checksums }
+      call { @api.database_checksum_v2(id, format, timeout: timeout).checksums }
     end
 
     # Your organization's recent download attempts, newest first, refusals
     # included.
-    def downloads(limit: nil)
-      call { @api.list_downloads(limit.nil? ? {} : { limit: limit }).downloads }
+    #
+    # @param timeout [Numeric, nil] seconds this attempt may take, for THIS call only.
+    def downloads(limit: nil, timeout: nil)
+      call { @api.list_downloads(limit: limit, timeout: timeout).downloads }
     end
 
     # The time-limited URL for one database file.
@@ -61,9 +75,13 @@ module InternetData
     # caller decides how to transfer a file that routinely runs to gigabytes; the
     # link authorizes the START of a transfer, so one already running is not
     # interrupted when it lapses.
-    def download_url(id, format)
+    #
+    # @param timeout [Numeric, nil] seconds this attempt may take, for THIS call
+    #   only. It bounds the request that MINTS the link, which is an ordinary
+    #   JSON call, and says nothing about the transfer you then run with it.
+    def download_url(id, format, timeout: nil)
       check_format!(format)
-      call { redirect_location(id, format) }
+      call { redirect_location(id, format, timeout) }
     end
 
     # Download one database file to `path`, and return the bytes written.
@@ -77,11 +95,7 @@ module InternetData
       partial = "#{path}.part"
       begin
         url = download_url(id, format)
-        written = Retries.with_retries(@retries) do
-          # Reopened per attempt, so a retry restarts the file rather than
-          # appending a second copy of the body to a half-written one.
-          File.open(partial, 'wb') { |file| stream(url) { |chunk| file.write(chunk) } }
-        end
+        written = File.open(partial, 'wb') { |file| transfer(url) { |chunk| file.write(chunk) } }
         File.rename(partial, path)
       rescue StandardError
         File.delete(partial) if File.exist?(partial)
@@ -99,15 +113,27 @@ module InternetData
     # not measured with {#metadata}.
     def download_bytes(id, format)
       url = download_url(id, format)
-      Retries.with_retries(@retries) do
-        bytes = String.new(encoding: Encoding::BINARY)
-        stream(url) { |chunk| bytes << chunk }
-        bytes
-      end
+      bytes = String.new(encoding: Encoding::BINARY)
+      transfer(url) { |chunk| bytes << chunk }
+      bytes
     end
 
     private
 
+    # The transfer of a presigned link, retried only while nothing has reached the
+    # block: object storage failing before the body is as transient as any
+    # outage, while a body that dies part way is not fetched again, because the
+    # bytes already handed over cannot be taken back.
+    def transfer(url, &sink)
+      delivered = false
+      Retries.with_retries(@retries, retry_if: -> { !delivered }) do
+        stream(url) do |chunk|
+          delivered = true
+          sink.call(chunk)
+        end
+      end
+    end
+
     # Runs one transfer of a presigned link, handing each chunk to the block, and
     # returns the bytes that reached it.
     #
@@ -169,8 +195,8 @@ module InternetData
             "invalid value for \"format\", must be one of #{DatabaseFormat.all_vars}"
     end
 
-    def redirect_location(id, format)
-      @api.download_database_v2(id, format)
+    def redirect_location(id, format, timeout)
+      @api.download_database_v2(id, format, timeout: timeout)
       raise Error.new(:server_error, 'expected a redirect to object storage')
     rescue ApiError => e
       raise unless e.code == 302

data/lib/internetdata/retries.rb

@@ -11,12 +11,14 @@ module InternetData
 
     module_function
 
-    def with_retries(retries)
+    # `retry_if`, when given, is asked after each retryable failure, and a false
+    # answer ends the attempts there.
+    def with_retries(retries, retry_if: nil)
       attempt = 0
       begin
         yield
       rescue Error => e
-        raise unless e.retryable? && attempt < retries
+        raise unless e.retryable? && attempt < retries && (retry_if.nil? || retry_if.call)
 
         attempt += 1
         sleep(delay_for(e, attempt))

data/lib/internetdata/transport.rb

@@ -50,9 +50,15 @@ module InternetData
     # is true for every value it can be given, so the download's 302 would be
     # chased and a multi-gigabyte database read into memory. Nothing this API
     # serves is meant to be followed.
+    #
+    # `opts[:timeout]` is a per-call override of the configured bound, which the
+    # generated client would otherwise apply to every request it builds. It
+    # reaches here as an ordinary generated `opts` entry, so one seam bounds
+    # every call whether the request was built by hand or by the generator.
     def build_request(http_method, path, opts = {})
       request = super
       request.options[:followlocation] = false
+      request.options[:timeout] = opts[:timeout] unless opts[:timeout].nil?
       request
     end
 

data/lib/internetdata/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module InternetData
-  VERSION = '2.0.0'
+  VERSION = '2.1.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: internetdata
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Mslm Dev