active_sanction 1.0.0

data/lib/active_sanction/fetcher/result.rb

@@ -0,0 +1,116 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  class Fetcher
+    # What one conditional fetch came back with, in the terms a caller cares
+    # about: did the list change, and if not, may we keep what we have.
+    #
+    #   result = fetcher.fetch(url, key: :ofac_sdn)
+    #   return storage.latest(:ofac_sdn) if result.unchanged?
+    #
+    #   parse(result.body)
+    #
+    # Three outcomes, not two. `changed?` is a fresh payload to parse,
+    # `unchanged?` is the publisher confirming the copy already held, and
+    # `failed?` is any status that is neither -- a 403 for a missing header, a
+    # 404 for a retired URL, a 500 the retries could not get past. Sync
+    # orchestration (#34) needs to tell the second from the third: one source
+    # answering 304 while another answers 500 is a successful run with one
+    # failure in it, not a run that transferred nothing.
+    #
+    # `body` is nil for both an unchanged result -- a 304 has none by
+    # definition -- and a streamed download, where the bytes went to disk.
+    #
+    # Instances are frozen on construction.
+    class Result
+      extend T::Sig
+
+      # `key` is what the validators were filed under, `url` the URL asked for.
+      # `validators` is what is now filed there: after a 304 the previous
+      # record with its clock moved forward, after a 200 the publisher's new
+      # ETag and Last-Modified, after a failure whatever was already stored --
+      # a bad afternoon at a file server teaches us nothing. It is nil only
+      # when nothing is stored, which for a 200 means a publisher that served
+      # no validators at all, and so a full download again next time.
+      sig { returns(T.untyped) }
+      attr_reader :key
+
+      sig { returns(T.untyped) }
+      attr_reader :url
+
+      sig { returns(HttpClient::Response) }
+      attr_reader :response
+
+      sig { returns(T.nilable(Validators)) }
+      attr_reader :validators
+
+      sig do
+        params(key: T.untyped, url: T.untyped, response: HttpClient::Response,
+               validators: T.nilable(Validators)).void
+      end
+      def initialize(key:, url:, response:, validators: nil)
+        @key = T.let(key, T.untyped)
+        @url = T.let(url, T.untyped)
+        @response = T.let(response, HttpClient::Response)
+        @validators = T.let(validators, T.nilable(Validators))
+        freeze
+      end
+
+      sig { returns(T::Boolean) }
+      def unchanged? = response.not_modified?
+
+      sig { returns(T::Boolean) }
+      def changed? = response.success?
+
+      sig { returns(T::Boolean) }
+      def failed? = !changed? && !unchanged?
+
+      sig { returns(Integer) }
+      def status = response.status
+
+      sig { returns(T.nilable(String)) }
+      def body = response.body
+
+      sig { returns(URI::Generic) }
+      def uri = response.uri
+
+      sig { returns(T.nilable(String)) }
+      def etag = response.etag
+
+      sig { returns(T.nilable(String)) }
+      def last_modified = response.last_modified
+
+      sig { params(name: T.untyped).returns(T.nilable(String)) }
+      def [](name) = response[name]
+
+      # True when the publisher answered at all -- 200 or 304. What a caller
+      # checks before deciding a sync succeeded, as against what it checks
+      # before deciding there is anything new to parse.
+      sig { returns(T::Boolean) }
+      def ok? = changed? || unchanged?
+
+      # For callers that want a failed fetch to be fatal. A 304 passes here,
+      # unlike Response#success!, because an unchanged list is the outcome this
+      # whole mechanism exists to produce.
+      sig { returns(T.self_type) }
+      def success!
+        return self if ok?
+
+        response.success!
+        self
+      end
+
+      sig { returns(String) }
+      def to_s = "#{status} #{url}#{" (unchanged)" if unchanged?}"
+
+      sig { returns(String) }
+      def inspect
+        bytes = body
+        "#<#{self.class} #{key} #{status}#{" unchanged" if unchanged?}#{" body=#{bytes.bytesize}B" if bytes}>"
+      end
+    end
+  end
+end

data/lib/active_sanction/fetcher.rb

@@ -0,0 +1,242 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/error"
+require "active_sanction/http_client"
+require "active_sanction/validators"
+require "active_sanction/validator_store"
+require "active_sanction/fetcher/result"
+
+module ActiveSanction
+  # An HttpClient that remembers what it fetched last time.
+  #
+  #   fetcher = ActiveSanction::Fetcher.new
+  #   result  = fetcher.fetch(url, key: :ofac_sdn)
+  #
+  #   result.unchanged?   # publisher answered 304; keep the snapshot we have
+  #   result.changed?     # a new payload to parse
+  #
+  # Every launch source serves both `ETag` and `Last-Modified`, verified live:
+  #
+  #   OFAC SDN.CSV        "0953154d0fb5aff918c5ec1daf6e9c0e"
+  #   UN consolidated.xml "0x8DF0558663FC719"
+  #   Canada sema-lmes    "0bf613fb33dd1:0"
+  #
+  # and they publish changes daily at most. So a sync that runs hourly should
+  # move tens of megabytes once a day and three empty 304 responses the other
+  # twenty-three times. That is the whole purpose of this class: it holds the
+  # validators, sends them, and reports "unchanged" in a form a caller can act
+  # on without inspecting a status code.
+  #
+  # The saving is not only bandwidth. Skipping the download also skips the
+  # parse, which for OFAC's SDN join (#18) is the expensive half.
+  #
+  # What it deliberately does not do is decide what "unchanged" means for the
+  # application. It reuses no snapshot and returns no cached body -- storage
+  # (#23, #24) owns those, and a fetch layer that quietly handed back a
+  # previous payload would make it impossible to tell a list that did not
+  # change from a sync that did not run.
+  #
+  # @api private
+  class Fetcher
+    extend T::Sig
+
+    # Sent by us unless the caller sent its own. A caller doing its own
+    # conditional request -- a range fetch, a probe against a mirror -- has a
+    # reason we do not know, and layering a stored ETag on top of it would
+    # produce a request neither side meant.
+    CONDITIONAL_HEADERS = T.let(%w[if-none-match if-modified-since].freeze, T::Array[String])
+
+    sig { returns(HttpClient) }
+    attr_reader :client
+
+    # Any store answering the ValidatorStore contract -- see #initialize.
+    sig { returns(T.untyped) }
+    attr_reader :store
+
+    # Seconds, or nil to disable the staleness clock -- see #stale?.
+    sig { returns(T.nilable(Numeric)) }
+    attr_reader :stale_after
+
+    # Anything Logger-shaped, or nil, as Configuration#logger has it.
+    sig { returns(T.untyped) }
+    attr_reader :logger
+
+    # The store defaults to disk, so the second run of a cron job benefits and
+    # not merely the second call in one process. A caller that would rather
+    # keep nothing between runs passes ValidatorStore::Memory.new.
+    sig do
+      params(client: HttpClient, store: T.untyped, stale_after: T.nilable(Numeric), logger: T.untyped).void
+    end
+    def initialize(client: HttpClient.new,
+                   store: ValidatorStore::FileSystem.new,
+                   stale_after: ActiveSanction.config.stale_after,
+                   logger: ActiveSanction.config.logger)
+      @client = T.let(client, HttpClient)
+      @store = T.let(store, T.untyped)
+      @stale_after = T.let(stale_after, T.nilable(Numeric))
+      @logger = T.let(logger, T.untyped)
+    end
+
+    # Fetches conditionally and buffers the body, like HttpClient#get.
+    #
+    # `key` is what the validators are filed under, defaulting to the URL. A
+    # source adapter (#12) passes its own name instead, so that a publisher
+    # moving a file changes which URL is fetched without orphaning the record
+    # of what was fetched -- the stored URL is compared before its validators
+    # are used, and a moved file downloads in full exactly once.
+    #
+    # `force: true` sends no validators, so the publisher has no way to answer
+    # 304. For the operator who suspects the cached copy is wrong and wants the
+    # bytes regardless of what the ETag says.
+    sig do
+      params(url: T.untyped, key: T.untyped, force: T::Boolean, headers: T::Hash[T.untyped, T.untyped])
+        .returns(Result)
+    end
+    def fetch(url, key: url, force: false, headers: {})
+      conditional(url, key, force, headers) { |request| client.get(url, headers: request) }
+    end
+
+    # Streams conditionally to disk, like HttpClient#download. A 304 writes
+    # nothing: HttpClient only streams a 2xx body, so the file already at `to:`
+    # is left exactly as the last download left it.
+    sig do
+      params(url: T.untyped, to: T.untyped, key: T.untyped, force: T::Boolean,
+             headers: T::Hash[T.untyped, T.untyped]).returns(Result)
+    end
+    def download(url, to:, key: url, force: false, headers: {})
+      conditional(url, key, force, headers) { |request| client.download(url, to: to, headers: request) }
+    end
+
+    # Whether a sync is due, answered locally and without a request.
+    #
+    # True when the key has never been fetched, when its validators were stored
+    # against a different URL, or when nothing has confirmed the copy within
+    # `stale_after` (default 24 hours; nil disables the clock). It is a
+    # scheduling predicate -- what #36's `sources` command prints, and what a
+    # caller checks before deciding to spend a round-trip -- not a claim about
+    # the publisher's current file. Only a fetch can make that claim, and a
+    # cheap #fetch that comes back `unchanged?` is how to ask for it.
+    sig { params(key: T.untyped, url: T.untyped).returns(T::Boolean) }
+    def stale?(key, url: nil)
+      stored = store[key]
+      return true if stored.nil? || stored.empty?
+      return true if url && !stored.for?(url)
+
+      after = stale_after
+      return false if after.nil?
+
+      Time.now - stored.checked_at >= after.to_f
+    end
+
+    sig { params(key: T.untyped, url: T.untyped).returns(T::Boolean) }
+    def fresh?(key, url: nil) = !stale?(key, url: url)
+
+    # What is stored for a key, or nil. Mostly for a CLI that wants to print
+    # when a source was last confirmed.
+    sig { params(key: T.untyped).returns(T.nilable(Validators)) }
+    def validators(key) = store[key]
+
+    # Drops a key's validators, so the next fetch downloads in full. The
+    # supported way to do what deleting the store file does for every source at
+    # once.
+    sig { params(key: T.untyped).returns(T.untyped) }
+    def forget(key) = store.delete(key)
+
+    private
+
+    sig do
+      params(url: T.untyped, key: T.untyped, force: T::Boolean, headers: T::Hash[T.untyped, T.untyped],
+             block: T.proc.params(request: T::Hash[T.untyped, T.untyped]).returns(HttpClient::Response))
+        .returns(Result)
+    end
+    def conditional(url, key, force, headers, &block)
+      stored = force ? nil : usable(key, url)
+      log_request(key, url, stored, force)
+      response = block.call(merge(headers, stored))
+      record(key, url, stored, response)
+    end
+
+    # Validators stored against a different URL are not merely useless, they
+    # are dangerous: a 304 from the new address would say "the file you have is
+    # current" about a file that came from somewhere else.
+    sig { params(key: T.untyped, url: T.untyped).returns(T.nilable(Validators)) }
+    def usable(key, url)
+      stored = store[key]
+      return nil if stored.nil? || stored.empty?
+      return stored if stored.for?(url)
+
+      logger&.info("[active_sanction] #{key} moved from #{stored.url} to #{url}; fetching in full")
+      nil
+    end
+
+    sig do
+      params(headers: T::Hash[T.untyped, T.untyped], stored: T.nilable(Validators))
+        .returns(T::Hash[T.untyped, T.untyped])
+    end
+    def merge(headers, stored)
+      return headers.to_h if stored.nil?
+
+      supplied = headers.to_h.keys.map { |name| name.to_s.downcase }
+      return headers.to_h if CONDITIONAL_HEADERS.any? { |name| supplied.include?(name) }
+
+      stored.request_headers.merge(headers.to_h)
+    end
+
+    # Only an answer the publisher stands behind updates the store. A 500 that
+    # outlived its retries, or a 403, says nothing about whether the list
+    # changed, and letting one clear the validators would turn a bad afternoon
+    # at a government file server into a full re-download of every list.
+    sig do
+      params(key: T.untyped, url: T.untyped, stored: T.nilable(Validators),
+             response: HttpClient::Response).returns(Result)
+    end
+    def record(key, url, stored, response)
+      learned = validators_for(url, stored, response)
+      store[key] = learned if learned
+      log_response(key, response, learned)
+      Result.new(key: key, url: url, response: response, validators: store[key])
+    end
+
+    sig do
+      params(url: T.untyped, stored: T.nilable(Validators), response: HttpClient::Response)
+        .returns(T.nilable(Validators))
+    end
+    def validators_for(url, stored, response)
+      if response.not_modified?
+        # A caller may have sent its own conditional headers, in which case a
+        # 304 arrives with nothing stored behind it; the response still says
+        # what the current validators are.
+        stored ? stored.confirmed_by(response) : Validators.from_response(response, url: url)
+      elsif response.success?
+        Validators.from_response(response, url: url)
+      end
+    end
+
+    sig { params(key: T.untyped, url: T.untyped, stored: T.nilable(Validators), force: T::Boolean).void }
+    def log_request(key, url, stored, force)
+      return unless logger
+
+      logger.info("[active_sanction] fetching #{key} #{url}#{" (forced)" if force}" \
+                  "#{" if-none-match=#{stored.etag}" if stored&.etag}")
+    end
+
+    # The line an operator greps for to answer "did last night's sync actually
+    # transfer anything?".
+    sig { params(key: T.untyped, response: HttpClient::Response, learned: T.nilable(Validators)).void }
+    def log_response(key, response, learned)
+      return unless logger
+
+      if response.not_modified?
+        logger.info("[active_sanction] #{key} 304 Not Modified; unchanged since " \
+                    "#{learned&.updated_at&.iso8601 || "the last download"}")
+      elsif response.success?
+        logger.info("[active_sanction] #{key} #{response.status} changed; etag=#{response.etag.inspect}")
+      else
+        logger.info("[active_sanction] #{key} #{response.status}; keeping stored validators")
+      end
+    end
+  end
+end

data/lib/active_sanction/http_client/errors.rb

@@ -0,0 +1,94 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "active_sanction/error"
+
+module ActiveSanction
+  class HttpClient
+    # Base for every failure this client raises, all of them cases where the
+    # server never produced a status to hand back. Statuses it *did* produce --
+    # 403, 500, 304 -- come back as a Response instead; see Response.
+    #
+    # A FetchError, so a caller backing off on "the bytes could not be
+    # obtained" does not have to know that this library speaks `net/http`. The
+    # subclasses below split it further only where the fix differs.
+    #
+    # @api public
+    class Error < FetchError; end
+
+    # The connection or a read exceeded its timeout, and retries did not save
+    # it. Kept distinct from ConnectionError because it is the failure that
+    # usually means "the publisher is slow today", not "the URL is wrong".
+    #
+    # @api public
+    class TimeoutError < Error
+      extend T::Sig
+
+      # No status to reason from, and the answer is still yes: a government
+      # file server that timed out this afternoon serves the same file
+      # tomorrow morning.
+      sig { returns(T::Boolean) }
+      def retryable? = retryable_or(true)
+    end
+
+    # The request never completed: DNS failure, refused or reset connection,
+    # TLS failure.
+    #
+    # @api public
+    class ConnectionError < Error
+      extend T::Sig
+
+      sig { returns(T::Boolean) }
+      def retryable? = retryable_or(true)
+    end
+
+    # The redirect chain exceeded `max_redirects` without reaching a body. A
+    # cap rather than an unbounded follow, because a publisher misconfiguring
+    # a redirect should cost one request too many and not a crawl.
+    #
+    # Not retryable, and neither are the two below: a misrouted URL is routed
+    # the same way on the next attempt, and this is the shape of failure that
+    # needs somebody to look at where the publisher moved the file to.
+    #
+    # @api public
+    class TooManyRedirects < Error; end
+
+    # A redirect chain that returns to a URL already visited. It would trip the
+    # hop cap on its own, but a loop and a genuinely long chain call for
+    # different fixes, so they get different errors.
+    #
+    # @api public
+    class RedirectLoop < Error; end
+
+    # A `Location` that cannot be resolved, or that leaves HTTP entirely. A
+    # sanctions file served over `ftp://` is a sign something is wrong upstream,
+    # not an opportunity to be accommodating.
+    #
+    # @api public
+    class InvalidRedirect < Error; end
+
+    # Raised by Response#success! for a status the caller declared fatal. It
+    # carries the response, so a rescuer can still log what came back, and the
+    # status, so `retryable?` answers from it without anybody unwrapping the
+    # response to look: a 503 is retryable, a 403 for a missing User-Agent is
+    # not. See FetchError::RETRYABLE_STATUSES.
+    #
+    # @api public
+    class ResponseError < Error
+      extend T::Sig
+
+      sig { returns(Response) }
+      attr_reader :response
+
+      # Which list this was is stamped on afterwards, by the adapter -- see
+      # Error#in_source. This layer has a URL and no idea what is behind it.
+      sig { params(response: Response).void }
+      def initialize(response)
+        @response = T.let(response, Response)
+        super("#{response.uri} returned #{response.status}", status: response.status)
+      end
+    end
+  end
+end

data/lib/active_sanction/http_client/response.rb

@@ -0,0 +1,129 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module ActiveSanction
+  class HttpClient
+    # What a server said, after redirects were followed and retries exhausted.
+    #
+    # Any status the server actually produced comes back as one of these
+    # rather than as an exception, including 403 and 500. The fetch layer has
+    # to reason about statuses -- conditional GET (#10) treats 304 as
+    # success-unchanged, sync orchestration (#34) isolates a single source's
+    # failure rather than aborting the run -- and a client that raised on
+    # everything but 200 would force each of them to rescue what it wanted
+    # back. Failures the server never got to answer (timeouts, refused
+    # connections, redirect loops) do raise: there is no status to hand over.
+    #
+    # `body` is nil for a streamed download, where the bytes went to disk
+    # instead, and for a 304, which has none by definition.
+    #
+    # Instances are frozen on construction.
+    #
+    # @api private
+    class Response
+      extend T::Sig
+
+      # `uri` is where the response finally came from, which is not the URL
+      # asked for when redirects were followed, and `redirects` is the hops it
+      # took to get there. #11 stores both beside the cached payload: a bug
+      # report about an OFAC download is much easier to read when it names the
+      # blob-storage host that actually served the bytes.
+      sig { returns(Integer) }
+      attr_reader :status
+
+      # Downcased, and each value joined the way the wire format does -- see
+      # #normalize.
+      sig { returns(T::Hash[String, String]) }
+      attr_reader :headers
+
+      # nil for a streamed download, where the bytes went to disk, and for a
+      # 304, which has none by definition.
+      sig { returns(T.nilable(String)) }
+      attr_reader :body
+
+      sig { returns(URI::Generic) }
+      attr_reader :uri
+
+      sig { returns(T::Array[URI::Generic]) }
+      attr_reader :redirects
+
+      sig do
+        params(status: T.untyped, headers: T.untyped, uri: URI::Generic, body: T.nilable(String),
+               redirects: T::Array[URI::Generic]).void
+      end
+      def initialize(status:, headers:, uri:, body: nil, redirects: [])
+        @status = T.let(Integer(status), Integer)
+        @headers = T.let(normalize(headers), T::Hash[String, String])
+        @body = T.let(body, T.nilable(String))
+        @uri = T.let(uri.freeze, URI::Generic)
+        @redirects = T.let(redirects.dup.freeze, T::Array[URI::Generic])
+        freeze
+      end
+
+      # Header lookup is case-insensitive because HTTP header names are, and
+      # because the same header reaches us capitalized differently depending on
+      # which CDN a publisher put in front of its file this quarter.
+      sig { params(name: T.untyped).returns(T.nilable(String)) }
+      def [](name)
+        headers[name.to_s.downcase]
+      end
+
+      sig { returns(T::Boolean) }
+      def success? = status.between?(200, 299)
+
+      sig { returns(T::Boolean) }
+      def not_modified? = status == 304
+
+      sig { returns(T::Boolean) }
+      def redirect? = REDIRECT_STATUSES.include?(status)
+
+      sig { returns(T::Boolean) }
+      def client_error? = status.between?(400, 499)
+
+      sig { returns(T::Boolean) }
+      def server_error? = status.between?(500, 599)
+
+      sig { returns(T.nilable(String)) }
+      def content_type = self["content-type"]
+
+      sig { returns(T.nilable(String)) }
+      def etag = self["etag"]
+
+      sig { returns(T.nilable(String)) }
+      def last_modified = self["last-modified"]
+
+      # For callers that want a non-2xx to be fatal without writing the check
+      # themselves. Deliberately not the default path -- see the class comment.
+      sig { returns(T.self_type) }
+      def success!
+        return self if success?
+
+        raise ResponseError, self
+      end
+
+      sig { returns(String) }
+      def to_s = "#{status} #{uri}"
+
+      sig { returns(String) }
+      def inspect
+        bytes = body
+        "#<#{self.class} #{status} #{uri}#{" body=#{bytes.bytesize}B" if bytes}>"
+      end
+
+      private
+
+      # Net::HTTPResponse hands back each header as an array of values, since
+      # a header may legally repeat. Repeats are vanishingly rare on the ones
+      # this library reads, so they are joined the way the wire format does it
+      # and the caller sees a plain string.
+      sig { params(headers: T.untyped).returns(T::Hash[String, String]) }
+      def normalize(headers)
+        headers.to_h do |name, value|
+          [-name.to_s.downcase, value.is_a?(Array) ? value.join(", ") : value.to_s]
+        end.freeze
+      end
+    end
+  end
+end