kataba 1.1.2 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e2048c2adaac9976b8feb4fcea1da32d937a7d39955152a579f18d81e9fe3f2
-  data.tar.gz: 806c80fff00c2a8dd637b42e1df485787e041a514ecc6372df5cb4902db8889e
+  metadata.gz: 8ae88ae5dafc92e24f90d773300f44d096a5efb5d42e82c047e48c413ffcf852
+  data.tar.gz: 6954878cb4e2fb52646f294efda7a53dd0c11ac97e9a08554691d298ac5e961e
 SHA512:
-  metadata.gz: 15b1a25d1ed46d7510f829f4a6b51ca6516ed29edfa011b4a8395cee0bc36092e37d4ecb925e24791da6d4790435896059cabd8fb375b17278c724dff61540c9
-  data.tar.gz: 64fa54356f4d0c0387a9c52898795462524363b6fccf6632259b635adb7d2c19544760c2e0b91256cf1e22b50a3e3836868ce5581c73499bda3184f32838b086
+  metadata.gz: 4be9c7f61034d3d0dbabcb65ccd56169f1f1f7e5aafeaa583a6cd5ab5cf348a5dac1c378d44a715f8b2aeecac8059a1af39a47a1cce53120bf2a5f3153d20630
+  data.tar.gz: bf9b63fcf72d6ae6c4ee1e515ff81854ae4f88d79b0a4496bdc2bdccd63fce49bcded29d8eb0fddc7c22f83860956716c7c114d101fe4e18a5b02e5f2e735c7e

data/lib/kataba/fetcher.rb

@@ -32,10 +32,25 @@ module Kataba
       '/mods/xml.xsd' => '/standards/mods/xml.xsd',
     }.freeze
 
+    # Seconds to wait for a schema host to accept the connection, and then to
+    # deliver the body. Net::HTTP's own defaults are 60s each, which is a
+    # fallback rather than a considered number: a schema is a small file from a
+    # public host, and both are paid per redirect hop and again on the
+    # alternate-scheme attempt, so an unbounded fetch can hold a caller's
+    # thread for many minutes.
+    DEFAULT_OPEN_TIMEOUT = 3
+    DEFAULT_READ_TIMEOUT = 10
+
     class FetchError < StandardError; end
 
-    def initialize(uri)
+    # A subclass, so a consumer's `rescue FetchError` covers a timeout while
+    # one that wants to retry a timeout without retrying a 404 still can.
+    class FetchTimeout < FetchError; end
+
+    def initialize(uri, open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT)
       @original_uri = uri
+      @open_timeout = open_timeout
+      @read_timeout = read_timeout
     end
 
     def fetch
@@ -72,10 +87,26 @@ module Kataba
     def request_with_redirects(uri, redirect_depth:)
       raise FetchError, "too many redirects fetching #{@original_uri}" if redirect_depth > MAX_REDIRECTS
 
-      parsed = URI.parse(uri)
+      parsed = parse_http_uri(uri)
       http = Net::HTTP.new(parsed.host, parsed.port)
       http.use_ssl = (parsed.scheme == 'https')
-      response = http.get(parsed.request_uri)
+      http.open_timeout = @open_timeout
+      http.read_timeout = @read_timeout
+      # Net::HTTP retries an idempotent request once on a read timeout, so the
+      # ceiling per hop would otherwise be twice read_timeout. Nothing here
+      # reuses a socket, so that retry can only re-wait a timeout that has
+      # already expired.
+      http.max_retries = 0
+
+      response =
+        begin
+          http.get(parsed.request_uri)
+        rescue Timeout::Error => e
+          # Both Net::OpenTimeout and Net::ReadTimeout are Timeout::Errors, and
+          # Timeout::Error is a RuntimeError — so an unwrapped timeout lands in
+          # a consumer's generic rescue rather than its fetch-failure branch.
+          raise FetchTimeout, "timed out fetching #{uri} (#{e.class}: #{e.message})"
+        end
 
       if response.is_a?(Net::HTTPRedirection)
         target = resolve_redirect(parsed, response['location'])
@@ -85,6 +116,17 @@ module Kataba
       end
     end
 
+    # A bare filename parses as a URI with no host, and Net::HTTP then fails
+    # with NoMethodError, which no consumer's fetch-failure branch rescues.
+    def parse_http_uri(uri)
+      parsed = URI.parse(uri)
+      return parsed if parsed.is_a?(URI::HTTP) && parsed.host
+
+      raise FetchError, "not an absolute http(s) URI: #{uri}"
+    rescue URI::InvalidURIError => e
+      raise FetchError, "not a valid URI: #{uri} (#{e.message})"
+    end
+
     def resolve_redirect(from, location)
       raise FetchError, "redirect with no Location header from #{from}" if location.nil? || location.empty?
 

data/lib/kataba.rb

@@ -3,6 +3,8 @@ require 'tmpdir'
 require 'digest/md5'
 require 'fileutils'
 require 'yaml'
+require 'uri'
+require 'set'
 require 'kataba/fetcher'
 
 module Kataba
@@ -46,9 +48,23 @@ module Kataba
     #   Kataba.configuration.mirror_list = File.join(Rails.root, 'config', 'mirror.yml')
     attr_accessor :mirror_list
 
+    # Seconds Kataba waits for a schema host to accept a connection, and then
+    # to deliver the body. Raise them for a slow mirror; lower them when the
+    # caller is a request thread that must not be held. Both are paid per
+    # redirect hop and again on the alternate-scheme attempt, so the ceiling
+    # for one schema is a multiple of these, not either on its own.
+    #
+    # Example:
+    #   Kataba.configuration.open_timeout = 5
+    #   Kataba.configuration.read_timeout = 20
+    attr_accessor :open_timeout
+    attr_accessor :read_timeout
+
     # Default configuration values
     def initialize
       @offline_storage = "#{Dir.tmpdir}/kataba"
+      @open_timeout = Kataba::Fetcher::DEFAULT_OPEN_TIMEOUT
+      @read_timeout = Kataba::Fetcher::DEFAULT_READ_TIMEOUT
     end
   end
 
@@ -102,12 +118,17 @@ module Kataba
 
   private
 
-    def self.download_xsd(xsd_uri_array)
+    def self.download_xsd(xsd_uri_array, visited = Set.new)
       new_xsd_uris = []
       file_paths = []
 
       # Download files
       xsd_uri_array.each do |xsd_uri|
+        # Schemas may import each other (tdwg_dwc_simple.xsd and
+        # tdwg_dwc_extensions.xsd do), so fetch each URI once per call or the
+        # recursion never ends.
+        next unless visited.add?(xsd_uri)
+
         uri_md5 = Digest::MD5.hexdigest(xsd_uri)
 
         dir_name = "#{self.configuration.offline_storage}"
@@ -123,16 +144,22 @@ module Kataba
         # Resolve mirror first, if configured, then fetch. Fetching BEFORE
         # opening tmp_path means a failed network fetch can't leave an
         # orphaned 0-byte .part on disk — the file simply isn't created.
-        fetch_uri = xsd_uri
+        mirror = nil
         if !self.configuration.mirror_list.to_s.empty?
           # YAML.load_file returns nil for a comments-only or empty file.
           # Treat that as "no mirror configured" rather than NoMethodError.
           mirror_list = YAML.load_file(self.configuration.mirror_list) || {}
-          mirror = mirror_list[xsd_uri]
-          fetch_uri = mirror unless mirror.to_s.empty?
+          mirror = mirror_list[xsd_uri].to_s
+          mirror = nil if mirror.empty?
         end
 
-        body = Kataba::Fetcher.new(fetch_uri).fetch
+        body = if mirror && !mirror.match?(%r{\Ahttps?://}i)
+                 read_local_mirror(mirror)
+               else
+                 Kataba::Fetcher.new(mirror || xsd_uri,
+                                     open_timeout: self.configuration.open_timeout,
+                                     read_timeout: self.configuration.read_timeout).fetch
+               end
 
         # Write to a .part file first; only rename to the final cache path
         # after we've confirmed the bytes parse as XML. Without this, a
@@ -149,20 +176,20 @@ module Kataba
         end
 
         File.rename(tmp_path, file_path)
-        file_paths << file_path
+        file_paths << [file_path, xsd_uri]
       end
 
       # Search inside for other schemaLocations
-      file_paths.each do |file_path|
-        new_xsd_uris = find_schemas(file_path)
+      file_paths.each do |file_path, xsd_uri|
+        new_xsd_uris.concat(find_schemas(file_path, xsd_uri))
       end
 
       if !new_xsd_uris.reject(&:empty?).empty?
-        download_xsd(new_xsd_uris)
+        download_xsd(new_xsd_uris, visited)
       end
     end
 
-    def self.find_schemas(xml_file_path)
+    def self.find_schemas(xml_file_path, base_uri)
       xsd_uri_array = []
 
       # Open XML file
@@ -170,10 +197,11 @@ module Kataba
       # search for schemaLocations
       doc.xpath("//@schemaLocation").each do |node|
         if !node.value.to_s.empty?
+          schema_uri = resolve_schema_location(node.value, base_uri)
           # Add to array
-          xsd_uri_array << node.value
+          xsd_uri_array << schema_uri
           # Get MD5
-          uri_md5 = Digest::MD5.hexdigest(node.value)
+          uri_md5 = Digest::MD5.hexdigest(schema_uri)
           # Reassign attribute value
           node.value = "#{uri_md5}.xsd"
         end
@@ -184,4 +212,29 @@ module Kataba
 
       return xsd_uri_array
     end
+
+    # A mirror value that is not an http(s) URL is a file, so a consumer can
+    # ship a corrected schema with its own code. A relative path resolves
+    # against the mirror list's directory, never the working directory, which
+    # fetch_schema changes while it parses.
+    def self.read_local_mirror(path)
+      full_path = File.expand_path(path, File.dirname(self.configuration.mirror_list))
+      File.read(full_path)
+    rescue SystemCallError => e
+      raise Kataba::Fetcher::FetchError, "could not read mirror file #{full_path} (#{e.class})"
+    end
+
+    # An absolute location is returned byte for byte. Its MD5 is the cache
+    # filename and the mirror_list key, so normalising it would orphan every
+    # cached schema and miss every mirror. A relative one resolves against the
+    # canonical URI of the schema importing it, never a mirror or a redirect
+    # target, so its cache name does not depend on where the bytes came from.
+    def self.resolve_schema_location(location, base_uri)
+      return location if URI.parse(location).absolute?
+
+      URI.join(base_uri, location).to_s
+    rescue URI::Error
+      # Left as written, so the Fetcher reports it as a FetchError.
+      location
+    end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kataba
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.4
 platform: ruby
 authors:
 - David Cliff