active_sanction 1.0.0

data/lib/active_sanction/sources/australia_dfat.rb

@@ -0,0 +1,291 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/parsers"
+require "active_sanction/sources"
+require "active_sanction/sources/base"
+
+module ActiveSanction
+  module Sources
+    # Australia's Consolidated List: everybody the Foreign Minister has
+    # designated or declared under the Autonomous Sanctions Regulations 2011,
+    # together with every UN Security Council listing Australia gives effect to.
+    #
+    #   snapshot = ActiveSanction::Sources[:australia_dfat].new.sync
+    #
+    # ### It is published as a spreadsheet, and as nothing else
+    #
+    #   https://www.dfat.gov.au/sites/default/files/Australian_Sanctions_Consolidated_List.xlsx
+    #
+    #     "Consolidated List"   11,163 rows, 19 columns, 1.3 MB
+    #
+    # One `.xlsx` file, one sheet. There is no CSV, no XML and no JSON: DFAT's
+    # Consolidated List page offers exactly one download, and this is it. So
+    # reading a spreadsheet is not a convenience here, it is the price of
+    # screening against Australian sanctions at all.
+    #
+    # That was the open question on this list, and the answer was not to take a
+    # dependency. Parsers::Spreadsheet reads the workbook with `zlib` and the
+    # XML toolkit the other five adapters already use -- an `.xlsx` being a ZIP
+    # of XML parts -- so this adapter costs a host nothing to install. See that
+    # class for what it does and does not read.
+    #
+    # ### The URL moved, and the old one still answers
+    #
+    # The path this gem was scoped against, `regulation8_consolidated.xlsx`,
+    # is gone. What is left in its place is a redirect to
+    # `regulation8_consolidated_2.xls` -- a real file, served 200, in the old
+    # binary Excel format, last modified in March 2022. An adapter pointed at it
+    # would download a list four years stale on every sync and never once look
+    # unhealthy. The URL above is the one DFAT's own page links today.
+    #
+    # ### DFAT's edge rejects this gem's User-Agent
+    #
+    # Verified against the live endpoint: `active_sanction/x.y.z (+https://...)`
+    # gets no response at all -- not a 403, a dropped connection -- while
+    # `curl/8.7.1`, `Wget/1.21` and `python-requests/2.31.0` are all served. The
+    # filter is on the leading product token, and an unrecognised one is
+    # dropped, so identifying ourselves honestly is what gets us blocked.
+    #
+    # What is sent instead is the configured agent inside the form written for
+    # exactly this: `Mozilla/5.0 (compatible; <agent>)`, which is how a
+    # well-behaved crawler has identified itself since Googlebot. It is not a
+    # disguise -- the agent, its version and the contact URL an operator
+    # configured are all still in the string, and DFAT can still see who we are
+    # and block us on purpose. It is the same string in a shape the edge parses.
+    # See #fetch_file, which is the only place this source departs from Base.
+    #
+    # ### A record is several rows
+    #
+    # DFAT publishes one row per name. Reference `1000` is a primary name and
+    # `1000a`, `1000b` are its aliases, with every other column repeated on each
+    # -- so 11,163 rows are 3,906 records: 2,543 people, 1,041 organizations and
+    # 322 vessels. Record says how they are joined, and what the Control Date
+    # is not.
+    #
+    # ### What a clean Australian result is worth
+    #
+    # About what a UN one is worth, which is unsurprising: 1,172 of the 3,906
+    # records are listings by a UN committee that Australia has given effect to,
+    # and they arrive with the UN's own text intact. Alias grading is published
+    # as a field on all 6,802 aliases. Birth dates are published on 6,823 rows
+    # in nine spellings, which PublishedDate reads to within four fragments of
+    # the whole list.
+    #
+    # Against that, DFAT publishes no document numbers of any kind -- no
+    # passport, no national identity number, no company registration -- for any
+    # of the 3,906 records. The only identifier on the list is an IMO number, on
+    # 344 vessel rows. So an Australian name match has nothing behind it to make
+    # it decisive, in the way an OFAC passport number usually settles one, and a
+    # screening policy should know that before it sets a threshold.
+    #
+    # ### What this adapter does not do
+    #
+    # It does not read the Listing Information prose for anything but the
+    # listing date. The column holds relisting histories, UN committee
+    # references and the reasons for a designation, in a dozen shapes, and
+    # reading it properly is the job OFAC's RemarksParser does and wants the
+    # same treatment -- a measured coverage figure -- rather than a regex added
+    # here in passing. It is kept verbatim in remarks.
+    #
+    # It does not decompose an address. DFAT publishes one free-text column and
+    # no parts, so the whole published string is the street; see Record.
+    class AustraliaDfat < Base
+      extend T::Sig
+
+      key :australia_dfat
+      jurisdiction :au
+      authority "Australian Sanctions Office, Department of Foreign Affairs and Trade"
+      format :xlsx
+
+      # Creative Commons Attribution 4.0, with the Commonwealth Coat of Arms
+      # carved out -- the same shape as the UK's: the data is reusable, the
+      # crest is not.
+      licence_notice "Creative Commons Attribution 4.0 International, " \
+                     "attributed to the Department of Foreign Affairs and " \
+                     "Trade. The Commonwealth Coat of Arms is excluded. " \
+                     "Verified 2026-09-11."
+      licence_url "https://www.dfat.gov.au/about-us/about-this-website/copyright"
+
+      url :main, "https://www.dfat.gov.au/sites/default/files/Australian_Sanctions_Consolidated_List.xlsx"
+
+      # @api private
+      SHEET = T.let("Consolidated List", String)
+
+      # @api private
+      LIST = T.let(Parsers::Spreadsheet.new(sheet: SHEET), Parsers::Spreadsheet)
+
+      # The columns this adapter reads by name. The sheet names its own, so
+      # these are not a declaration of its shape -- they are what is checked
+      # before a single row is mapped, so that a column DFAT renames says so
+      # once and loudly rather than reading nil on all 11,163 rows.
+      #
+      # @api private
+      REQUIRED_COLUMNS = T.let(
+        %i[reference name_of_individual_or_entity type name_type alias_strength date_of_birth citizenship
+           address additional_information listing_information imo_number committees control_date
+           instrument_of_designation].freeze,
+        T::Array[Symbol]
+      )
+
+      # The letters DFAT suffixes an alias reference with: `1000a`, and six
+      # times `1000aa`. Stripping them is what joins a group.
+      #
+      # @api private
+      ALIAS_SUFFIX = T.let(/[a-z]+\z/, Regexp)
+
+      # `a) ... b) ...`: a UN enumeration inside one cell, which DFAT carries
+      # through into the addresses and the birth dates.
+      #
+      # @api private
+      ENUMERATOR = T.let(/(?:\A|[[:space:]])[a-z]\)[[:space:]]*/, Regexp)
+
+      # Every kind of space: these cells carry non-breaking ones, which
+      # `String#strip` leaves in place.
+      #
+      # @api private
+      SPACE = T.let(/[[:space:]]+/, Regexp)
+
+      # The form a filtered edge recognises -- see the class comment.
+      #
+      # @api private
+      COMPATIBLE_AGENT = T.let("Mozilla/5.0 (compatible; %s)", String)
+
+      # Records that could not be used, and fields that could not be read.
+      # Read after #parse; sync orchestration (#34) reports them.
+      sig { returns(T::Array[Parsers::Warning]) }
+      attr_reader :warnings
+
+      sig { params(args: T.untyped, options: T.untyped).void }
+      def initialize(*args, **options)
+        super
+        @warnings = T.let([], T::Array[Parsers::Warning])
+        @unmapped = T.let([], T::Array[Parsers::Warning])
+        @modified = T.let(nil, T.nilable(String))
+      end
+
+      sig { override.params(raw: T.untyped).returns(T::Array[Entity]) }
+      def parse(raw)
+        reader = LIST.read(raw)
+        entities = build(reader)
+        @modified = reader.modified
+        @warnings = reader.warnings + @unmapped
+        entities
+      end
+
+      # `2026-09-04T05:37:12Z` -- the moment DFAT saved the workbook, which it
+      # stamps inside the file and reports on its own download page as the date
+      # the list was last updated. Falls back to Last-Modified for a payload
+      # handed straight to #snapshot.
+      sig { override.returns(T.nilable(String)) }
+      def source_version = @modified || super
+
+      # A reference with its alias suffix removed: `1000a` and `1000b` are both
+      # part of `1000`. Public because it is what Record identifies a group by
+      # and what a caller reconciling a hit against DFAT's own site needs.
+      sig { params(reference: T.untyped).returns(T.nilable(String)) }
+      def self.group(reference)
+        base = reference.to_s.strip.sub(ALIAS_SUFFIX, "")
+        base.empty? ? nil : base
+      end
+
+      # One cell split on the `a) ... b) ...` enumeration DFAT writes several
+      # values with, or the whole cell where it wrote only one.
+      sig { params(cell: T.untyped).returns(T::Array[String]) }
+      def self.enumerated(cell)
+        cell.to_s.split(ENUMERATOR).map { |part| part.gsub(SPACE, " ").strip }.reject(&:empty?)
+      end
+
+      private
+
+      # The one place this source departs from Base: the request carries the
+      # configured User-Agent wrapped in the `Mozilla/5.0 (compatible; ...)`
+      # form, because DFAT's edge drops a request whose leading product token
+      # it does not recognise. The class comment says why that is identification
+      # rather than concealment.
+      sig { override.params(name: Symbol, address: String, force: T::Boolean).returns(Fetcher::Result) }
+      def fetch_file(name, address, force)
+        fetcher.fetch(address, key: file_key(name), force: force, headers: { "User-Agent" => compatible_agent })
+               .success!
+      end
+
+      # `Kernel.format` explicitly: `format` on a source is the Definition
+      # reader that answers `:xlsx`, and calling it here would be asking the
+      # declaration for a User-Agent.
+      sig { returns(String) }
+      def compatible_agent = Kernel.format(COMPATIBLE_AGENT, ActiveSanction.config.user_agent)
+
+      sig { params(reader: Parsers::Spreadsheet::Reader).returns(T::Array[Entity]) }
+      def build(reader)
+        @unmapped = []
+        rows = reader.to_a
+        verify_columns!(rows.first)
+        group(rows).filter_map do |reference, group|
+          record = Record.new(group)
+          record.entity || note_nameless(reference, group)
+        end
+      end
+
+      # Rows in published order, gathered under the reference they share. A row
+      # whose reference is blank cannot be joined to anything and is dropped
+      # with a warning; none of the 11,163 published today is.
+      sig do
+        params(rows: T::Array[Parsers::Spreadsheet::Row])
+          .returns(T::Hash[String, T::Array[Parsers::Spreadsheet::Row]])
+      end
+      def group(rows)
+        rows.each_with_object({}) do |row, groups|
+          reference = self.class.group(row[:reference]) || note_unreferenced(row)
+          (groups[reference] ||= []) << row unless reference.nil?
+        end
+      end
+
+      # A sheet whose header no longer names a column this reads is not a list
+      # with an empty column; it is a publisher who has changed the file, and
+      # every record built from it afterwards would be missing whatever that
+      # column carried. There is nothing to salvage, so nothing is.
+      sig { params(row: T.nilable(Parsers::Spreadsheet::Row)).void }
+      def verify_columns!(row)
+        raise Parsers::ParseError, "the #{SHEET.inspect} sheet has a header and no rows" if row.nil?
+
+        missing = REQUIRED_COLUMNS - row.columns
+        return if missing.empty?
+
+        raise Parsers::ParseError,
+              "the #{SHEET.inspect} sheet does not name the column(s) #{missing.join(", ")}. It names: " \
+              "#{row.columns.join(", ")}"
+      end
+
+      sig { params(row: Parsers::Spreadsheet::Row).returns(NilClass) }
+      def note_unreferenced(row)
+        @unmapped << Parsers::Warning.new(
+          line: row.number,
+          message: "row #{row.number} carries no reference, so it cannot be joined to a record",
+          snippet: row[:name_of_individual_or_entity]
+        )
+        nil
+      end
+
+      # A group whose rows are all nameless cannot be screened against. None of
+      # the 3,906 records published today is nameless; the warning exists so
+      # that the day one is, it is visible rather than absent.
+      sig do
+        params(reference: String, group: T::Array[Parsers::Spreadsheet::Row]).returns(NilClass)
+      end
+      def note_nameless(reference, group)
+        @unmapped << Parsers::Warning.new(
+          line: group.first&.number,
+          message: "reference #{reference.inspect} has no name on any of its #{group.size} row(s) and was skipped"
+        )
+        nil
+      end
+    end
+  end
+end
+
+require "active_sanction/sources/australia_dfat/published_date"
+require "active_sanction/sources/australia_dfat/record"
+
+ActiveSanction::Sources.register(ActiveSanction::Sources::AustraliaDfat)

data/lib/active_sanction/sources/base.rb

@@ -0,0 +1,295 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "time"
+require "active_sanction/error"
+require "active_sanction/entity"
+require "active_sanction/snapshot"
+require "active_sanction/fetcher"
+require "active_sanction/parsers"
+require "active_sanction/payload_cache"
+require "active_sanction/sources"
+require "active_sanction/sources/definition"
+require "active_sanction/sources/remarks"
+
+module ActiveSanction
+  module Sources
+    # The contract every sanctions list adapter implements: declare what the
+    # list is and where it lives, then turn its bytes into Entities.
+    #
+    #   class UnConsolidated < ActiveSanction::Sources::Base
+    #     key          :un_consolidated
+    #     jurisdiction :un
+    #     authority    "United Nations Security Council"
+    #     format       :xml
+    #     url          :main, "https://scsanctions.un.org/resources/xml/en/consolidated.xml"
+    #
+    #     def parse(raw)
+    #       ...   # => [Entity, ...]
+    #     end
+    #   end
+    #
+    #   ActiveSanction::Sources.register(UnConsolidated)
+    #
+    # #parse is the whole of what an adapter must write. Everything above it is
+    # declaration (Definition), and everything below it -- conditional GET,
+    # payload caching, checksumming the result into a Snapshot -- is here, the
+    # same for every list, so that adding a jurisdiction is a parsing problem
+    # and not a plumbing one.
+    #
+    #   snapshot = ActiveSanction::Sources[:un_consolidated].new.sync
+    #   snapshot                      # => Snapshot, or nil if nothing changed
+    #
+    # ### What #parse is handed
+    #
+    # A source declaring one URL gets the bytes. One declaring several gets a
+    # Hash keyed by the names it declared, because OFAC's three files only mean
+    # anything joined:
+    #
+    #   def parse(raw)
+    #     join(raw[:sdn], raw[:alt], raw[:add])
+    #   end
+    #
+    # Which of the two it is follows from the declaration, not from what a
+    # caller happened to pass, so an adapter's signature does not change under
+    # it when a fixture is handed to #snapshot directly.
+    #
+    # The bytes arrive as a String. A list too large to hold in memory wants
+    # #15's streaming parse rather than this path; the cached Entry, which
+    # knows how to hand out a verified file handle, is where that will start.
+    #
+    # ### What sync does not do
+    #
+    # It does not store the snapshot, and it does not rescue anything. One
+    # source's failure being isolated from the others, and the previous good
+    # snapshot being kept when a list fails, are decisions about a *run* rather
+    # than about a list -- they belong to sync orchestration (#34), which needs
+    # an exception here to notice.
+    class Base
+      extend T::Sig
+      extend Definition
+
+      sig { returns(Fetcher) }
+      attr_reader :fetcher
+
+      # nil turns payload caching off -- see #initialize.
+      sig { returns(T.nilable(PayloadCache)) }
+      attr_reader :cache
+
+      # Anything Logger-shaped, or nil, as Configuration#logger has it.
+      sig { returns(T.untyped) }
+      attr_reader :logger
+
+      # `cache: nil` turns off payload caching, which costs one thing worth
+      # knowing: a multi-file source can no longer answer a sync where some of
+      # its files changed and others came back 304, so the unchanged ones are
+      # downloaded again in full.
+      sig do
+        params(fetcher: Fetcher, cache: T.nilable(PayloadCache), logger: T.untyped).void
+      end
+      def initialize(fetcher: Fetcher.new, cache: PayloadCache.new, logger: ActiveSanction.config.logger)
+        @fetcher = T.let(fetcher, Fetcher)
+        @cache = T.let(cache, T.nilable(PayloadCache))
+        @logger = T.let(logger, T.untyped)
+        @results = T.let({}, T::Hash[Symbol, Fetcher::Result])
+      end
+
+      sig { returns(Symbol) }
+      def key = self.class.key
+
+      sig { returns(Symbol) }
+      def jurisdiction = self.class.jurisdiction
+
+      sig { returns(String) }
+      def authority = self.class.authority
+
+      sig { returns(T.nilable(Symbol)) }
+      def format = self.class.format
+
+      sig { returns(T::Hash[Symbol, String]) }
+      def urls = self.class.urls
+
+      sig { params(name: T.untyped).returns(String) }
+      def url(name = nil) = name.nil? ? self.class.url : self.class.url(name)
+
+      sig { params(name: T.untyped).returns(Symbol) }
+      def file_key(name) = self.class.file_key(name)
+
+      # The lower bounds this list is held to when there is nothing to compare
+      # it against. See Definition#floor, and Doctor, which is the only thing
+      # that reads them.
+      sig { returns(T::Hash[Symbol, Numeric]) }
+      def floors = self.class.floors
+
+      # A remark with everything this adapter appended stripped back off --
+      # the publisher's own words and nothing else. Inherited, so it reads the
+      # same for every source and a caller does not have to know which list a
+      # remark came from before it can strip one. See Sources::Remarks.
+      sig { params(remarks: T.untyped).returns(T.nilable(String)) }
+      def self.published_remarks(remarks) = Remarks.published(remarks)
+
+      # The one method an adapter must write: bytes in, canonical records out.
+      sig { params(_raw: T.untyped).returns(T::Array[Entity]) }
+      def parse(_raw)
+        raise UnsupportedError,
+              "#{self.class} must implement #parse(raw) and return an Array of ActiveSanction::Entity"
+      end
+
+      # Fetches, parses, and checksums -- or returns nil when the publisher
+      # says nothing has changed, which is the outcome to expect on most runs
+      # and the reason conditional GET exists.
+      sig { params(force: T::Boolean).returns(T.nilable(Snapshot)) }
+      def sync(force: false)
+        payloads = retrieve(force: force)
+        return nil if payloads.nil?
+
+        snapshot(payloads)
+      end
+
+      # Parses payloads already in hand into a Snapshot. What #sync calls, and
+      # what an adapter's own spec calls with a fixture and no network:
+      #
+      #   source.snapshot(main: File.read("spec/fixtures/un_consolidated.xml"))
+      #
+      # The files may be named as keywords, as above, or passed as one Hash --
+      # or, for a source that declares a single file, as the bytes themselves.
+      sig { params(payloads: T.untyped, files: T.untyped).returns(Snapshot) }
+      def snapshot(payloads = nil, **files)
+        Snapshot.new(source: key, entities: parse(parse_argument(payloads || files)),
+                     fetched_at: Time.now.utc, source_version: source_version)
+      rescue ActiveSanction::Error => e
+        raise e.in_source(declared_key)
+      end
+
+      # Every declared file, conditionally: a Hash of name => bytes, or nil
+      # when the publisher answered 304 for all of them.
+      #
+      # A file that came back unchanged is served from the payload cache, so a
+      # sync in which one of OFAC's three files moved downloads one file and
+      # not three. If the cache has nothing to serve -- a first run against a
+      # store that already has validators, a cache directory a user deleted --
+      # that file alone is re-fetched in full.
+      sig { params(force: T::Boolean).returns(T.nilable(T::Hash[Symbol, T.untyped])) }
+      def retrieve(force: false)
+        raise DeclarationError, "#{self.class} declares no URL to retrieve" if urls.empty?
+
+        @results = urls.to_h { |name, address| [name, fetch_file(name, address, force)] }
+        return nil if @results.each_value.all?(&:unchanged?)
+
+        @results.keys.to_h { |name| [name, payload(name)] }
+      rescue ActiveSanction::Error => e
+        raise e.in_source(declared_key)
+      end
+
+      # The positional-column assertions this source's raw files satisfy, or
+      # do not. Takes what #retrieve returned, or what #snapshot would be given,
+      # and answers with one Parsers::ColumnShape::Tally per declared column.
+      #
+      # Empty here, because most publishers ship a file that names its own
+      # fields and a named field cannot be quietly swapped with the one beside
+      # it. An adapter over a headerless file overrides #column_shapes -- see
+      # Sources::Ofac, and Parsers::ColumnShape for why a declared width is not
+      # enough on its own.
+      sig { params(payloads: T.untyped, files: T.untyped).returns(T::Array[Parsers::ColumnShape::Tally]) }
+      def column_tallies(payloads = nil, **files)
+        column_shapes(parse_argument(payloads || files))
+      end
+
+      # The hook #column_tallies dispatches to, handed exactly what #parse is
+      # handed. Overridden by an adapter over a positional file.
+      sig { params(_raw: T.untyped).returns(T::Array[Parsers::ColumnShape::Tally]) }
+      def column_shapes(_raw) = []
+
+      # Whether any of this source's files is due a fetch, answered locally and
+      # without a request. See Fetcher#stale? for what that does and does not
+      # claim.
+      sig { returns(T::Boolean) }
+      def stale? = urls.any? { |name, address| fetcher.stale?(file_key(name), url: address) }
+
+      sig { returns(T::Boolean) }
+      def fresh? = !stale?
+
+      # The publisher's own marker for the version just fetched. Last-Modified
+      # is the only one every launch source serves; an adapter whose document
+      # carries a generation date inside it should override this and say so,
+      # because that is the string an examiner will recognise.
+      sig { returns(T.nilable(String)) }
+      def source_version = @results.values.first&.last_modified
+
+      sig { returns(String) }
+      def inspect
+        name = self.class.declared?(:key) ? key : "(no key)"
+        "#<#{self.class} #{name} #{urls.size} url(s)>"
+      end
+
+      # What #parse is handed, worked out from what #retrieve returned: the
+      # bytes for a source declaring one file, the Hash keyed by declaration
+      # name for one declaring several. Public because a caller that has
+      # already fetched -- Doctor, which parses and then reads the same
+      # payloads a second time -- has to be able to produce the same argument
+      # without knowing how many files this source declares.
+      sig { params(payloads: T.untyped).returns(T.untyped) }
+      def parse_argument(payloads)
+        return payloads if payloads.is_a?(String)
+
+        self.class.multi_url? ? payloads.to_h : payloads.to_h.values.first
+      end
+
+      private
+
+      # This adapter's key, or nil for one that never declared it. What
+      # #source_id is stamped from -- see Error#in_source. The layers under
+      # here are deliberately ignorant of which list they are working on: the
+      # HTTP client sees a URL, the XML reader sees a payload, and neither can
+      # name the list in the error it raises. This is the one place that can,
+      # and it is the boundary a caller rescues at.
+      sig { returns(T.nilable(Symbol)) }
+      def declared_key = self.class.declared?(:key) ? key : nil
+
+      sig { params(name: Symbol, address: String, force: T::Boolean).returns(Fetcher::Result) }
+      def fetch_file(name, address, force)
+        fetcher.fetch(address, key: file_key(name), force: force).success!
+      end
+
+      sig { params(name: Symbol).returns(T.untyped) }
+      def payload(name)
+        result = @results.fetch(name)
+        return store(name, result) if result.changed?
+
+        cached(name) || store(name, refetch(name))
+      end
+
+      sig { params(name: Symbol, result: Fetcher::Result).returns(T.nilable(String)) }
+      def store(name, result)
+        cache&.write(file_key(name), result.body, url: url(name), final_url: result.uri.to_s,
+                                                  etag: result.etag, last_modified: result.last_modified)
+        result.body
+      end
+
+      # A cached payload that no longer hashes to its sidecar is not used and
+      # not repaired -- but it is also not fatal here, because the bytes it
+      # failed to prove are a download away. The corrupt entry stays on disk
+      # for whoever investigates it.
+      sig { params(name: Symbol).returns(T.nilable(String)) }
+      def cached(name)
+        cache&.latest(file_key(name))&.read
+      rescue PayloadCache::CorruptEntry => e
+        logger&.info("[active_sanction] #{key} #{name} cached payload unusable (#{e.class}); fetching in full")
+        nil
+      end
+
+      sig { params(name: Symbol).returns(Fetcher::Result) }
+      def refetch(name)
+        logger&.info("[active_sanction] #{key} #{name} unchanged but not cached; fetching in full")
+        result = fetcher.fetch(url(name), key: file_key(name), force: true).success!
+        return result if result.changed?
+
+        raise MissingPayload,
+              "#{key} #{name} answered #{result.status} to an unconditional request, so the bytes " \
+              "for #{url(name)} could not be obtained"
+      end
+    end
+  end
+end