monkrb 0.22.2 → 0.23.0

data/lib/monk/storage/input.rb

@@ -0,0 +1,83 @@
+module Monk
+  module Storage
+    # Checks on the arguments both backends take, so they refuse the same
+    # input the same way. Keys have their own module (Key).
+    module Input
+      DEFAULT_CONTENT_TYPE = "application/octet-stream".freeze
+      CHUNK_BYTES = 64 * 1024
+      # SigV4's longest signed link: 7 days. A url is valid until the end
+      # of its cache window plus expires_in, so the two together stay
+      # under it.
+      MAX_LINK_SECONDS = 7 * 24 * 3600
+      # type/subtype with optional parameters, and nothing that could end a
+      # header line.
+      CONTENT_TYPE = %r{\A[a-z0-9][a-z0-9!\#$&^_.+-]*/[a-z0-9][a-z0-9!\#$&^_.+-]*(;[ \t]*[ -~]*)?\z}i
+
+      module_function
+
+      def content_type!(value)
+        return value if value.is_a?(String) && value.match?(CONTENT_TYPE)
+
+        raise ArgumentError, "content_type: must be a MIME type such as \"image/png\", not #{value.inspect}"
+      end
+
+      # A String, or an IO the backend can measure, read, and read again
+      # (S3 hashes the body before sending it).
+      def body!(body)
+        return body if body.is_a?(String)
+        return body if %i[size read rewind].all? { |name| body.respond_to?(name) }
+
+        raise ArgumentError,
+          "a stored body is a String or an IO that responds to size, read and rewind (File, Tempfile, " \
+          "StringIO), not #{body.class}; write a pipe to a Tempfile first"
+      end
+
+      def link_times!(expires_in:, cache_window:)
+        { expires_in: expires_in, cache_window: cache_window }.each do |name, value|
+          next if value.is_a?(Integer) && value.positive?
+
+          raise ArgumentError, "#{name}: is a number of seconds above 0, not #{value.inspect}"
+        end
+        return if expires_in + cache_window <= MAX_LINK_SECONDS
+
+        raise ArgumentError,
+          "expires_in: #{expires_in} + cache_window: #{cache_window} is more than 7 days (#{MAX_LINK_SECONDS}), " \
+          "the longest a signed S3 link can be valid"
+      end
+
+      def byte_size!(value)
+        return value if value.is_a?(Integer) && !value.negative?
+
+        raise ArgumentError, "byte_size: is the upload's length in bytes, not #{value.inspect}"
+      end
+
+      # The body's SHA-256 in hex, read in chunks and rewound, so an IO can
+      # then be sent from the start.
+      def sha256(body)
+        return Digest::SHA256.hexdigest(body) if body.is_a?(String)
+
+        digest = Digest::SHA256.new
+        body.rewind
+        while (chunk = body.read(CHUNK_BYTES))
+          digest << chunk
+        end
+        body.rewind
+        digest.hexdigest
+      end
+
+      # A Range of byte offsets -> [first, last], last nil for "to the end".
+      def byte_span(range)
+        first = range.begin if range.is_a?(Range)
+        last = range.end if range.is_a?(Range)
+        unless first.is_a?(Integer) && first >= 0 && (last.nil? || last.is_a?(Integer))
+          raise ArgumentError, "range: is a Range of byte offsets from 0, e.g. 0..15, not #{range.inspect}"
+        end
+
+        last -= 1 if last && range.exclude_end?
+        raise ArgumentError, "range: #{range.inspect} is empty" if last && last < first
+
+        [first, last]
+      end
+    end
+  end
+end

data/lib/monk/storage/key.rb

@@ -0,0 +1,40 @@
+require "securerandom"
+
+module Monk
+  module Storage
+    # Keys name objects in the bucket (or under the file backend's root).
+    # Monk makes them random and never from a user's filename, which rules
+    # out path traversal and keeps file names out of URLs
+    # (docs/adr/0018-storage-s3-compatible-direct-upload.md).
+    module Key
+      # Lowercase only: macOS disks are case-insensitive, and the file
+      # backend must not merge two keys a bucket keeps apart. A segment
+      # can't start with a dot, so no "." or ".." segment, and no empty one.
+      FORMAT = %r{\A[a-z0-9][a-z0-9._-]*(/[a-z0-9][a-z0-9._-]*)*\z}
+      MAX_BYTES = 512
+
+      module_function
+
+      # "tmp/1f0c...": 128 random bits under prefix.
+      def generate(prefix:)
+        validate!(prefix)
+
+        "#{prefix}/#{SecureRandom.hex(16)}"
+      end
+
+      # Runs on every call to either backend, so a key that would mean
+      # something else on disk than in a bucket never reaches one.
+      def validate!(key)
+        raise InvalidKeyError, "a storage key is a String, not #{key.class}" unless key.is_a?(String)
+        raise InvalidKeyError, "storage key is #{key.bytesize} bytes; at most #{MAX_BYTES}" if key.bytesize > MAX_BYTES
+        unless key.match?(FORMAT)
+          raise InvalidKeyError,
+            "storage key #{key.inspect}: segments of a-z, 0-9, . _ and -, each starting with a letter or digit, " \
+            "joined by / (e.g. from Monk::Storage.generate_key(prefix: \"tmp\"))"
+        end
+
+        key
+      end
+    end
+  end
+end

data/lib/monk/storage/object.rb

@@ -0,0 +1,7 @@
+module Monk
+  module Storage
+    # What head returns for a stored object. etag is the bytes' MD5 in hex,
+    # without the quotes S3 puts around it.
+    Object = Data.define(:key, :byte_size, :content_type, :etag)
+  end
+end

data/lib/monk/storage/routes.rb

@@ -0,0 +1,110 @@
+module Monk
+  module Storage
+    ROUTE_PREFIX = "/_monk/storage/".freeze
+
+    # PUT and GET /_monk/storage/<token>: the file backend standing in for
+    # a bucket's signed URLs, refusing what a bucket would refuse, the way
+    # it would (docs/history/plan-storage.md decisions 7 and 8). Served whenever
+    # the backend is file://, in any environment, since its links only
+    # work if the routes exist.
+    module Routes
+      # Shareable, not only frozen: worker Ractors serve these routes.
+      NOSNIFF = Ractor.make_shareable({ "x-content-type-options" => "nosniff" })
+      RANGE = /\Abytes=(\d*)-(\d*)\z/
+
+      module_function
+
+      # A Rack response, or nil when the request isn't for these routes.
+      def response(env, backend, now: Time.now.to_i)
+        path = env["PATH_INFO"].to_s
+        return nil unless backend.is_a?(Backends::File) && path.start_with?(ROUTE_PREFIX)
+
+        payload = Token.verify(path.delete_prefix(ROUTE_PREFIX), backend.secret)
+        return refuse(403, "signature doesn't match") unless payload
+        return refuse(403, "link expired") unless payload[:e].is_a?(Integer) && payload[:e] > now
+        return refuse(403, "this link is for #{payload[:m]}") unless env["REQUEST_METHOD"] == payload[:m]
+
+        case payload[:m]
+        when "PUT" then upload(env, backend, payload)
+        when "GET" then download(env, backend, payload, now)
+        else refuse(403, "signature doesn't match")
+        end
+      rescue InvalidKeyError
+        refuse(403, "signature doesn't match")
+      end
+
+      # The browser's PUT. Content-Length and Content-Type are inside the
+      # token, as in a presigned S3 PUT: another value is a 403. A body
+      # shorter or longer than its Content-Length is a 400.
+      def upload(env, backend, payload)
+        return refuse(403, "Content-Length isn't the signed one") unless env["CONTENT_LENGTH"] == payload[:l].to_s
+        return refuse(403, "Content-Type isn't the signed one") unless env["CONTENT_TYPE"] == payload[:t]
+
+        md5 = backend.receive(payload[:k], env["rack.input"], content_type: payload[:t], byte_size: payload[:l])
+        return refuse(400, "the body isn't Content-Length bytes") unless md5
+
+        [200, NOSNIFF.merge("etag" => %("#{md5}")), []]
+      end
+
+      # The signed GET: the stored bytes, as their stored type, cached
+      # privately until the link expires. A Range is answered with 206, as
+      # a bucket does: Safari won't play <audio> or <video> without it.
+      def download(env, backend, payload, now)
+        object = backend.head(payload[:k])
+        return refuse(404, "no such object") unless object
+
+        headers = NOSNIFF.merge(
+          "content-type" => object.content_type,
+          "accept-ranges" => "bytes",
+          "etag" => %("#{object.etag}"),
+          "cache-control" => "private, max-age=#{payload[:e] - now}",
+        )
+        span = span(env["HTTP_RANGE"], object.byte_size)
+        return whole(backend, payload[:k], headers) if span.nil?
+        return refuse(416, "range not satisfiable", "content-range" => "bytes */#{object.byte_size}") if span == :none
+
+        first, last = span
+        bytes = backend.get(payload[:k], range: first..last)
+        [206, headers.merge("content-range" => "bytes #{first}-#{last}/#{object.byte_size}",
+          "content-length" => bytes.bytesize.to_s,), [bytes],]
+      rescue NotFoundError
+        refuse(404, "no such object")
+      end
+
+      def whole(backend, key, headers)
+        bytes = backend.get(key)
+        [200, headers.merge("content-length" => bytes.bytesize.to_s), [bytes]]
+      end
+
+      # One range of a Range header -> [first, last] within size; :none
+      # when it can't be satisfied; nil to ignore the header and send the
+      # whole object (no header, a syntax this doesn't read, several
+      # ranges), as RFC 9110 allows.
+      def span(header, size)
+        match = header && RANGE.match(header)
+        return nil unless match
+
+        first, last = match.captures
+        return nil if first.empty? && last.empty?
+
+        if first.empty?
+          suffix = last.to_i
+          return :none if suffix.zero? || size.zero?
+
+          return [[size - suffix, 0].max, size - 1]
+        end
+
+        first = first.to_i
+        last = last.empty? ? size - 1 : [last.to_i, size - 1].min
+        return nil if last < first && first < size
+        return :none if first >= size
+
+        [first, last]
+      end
+
+      def refuse(status, reason, extra = {})
+        [status, NOSNIFF.merge("content-type" => "text/plain; charset=utf-8").merge(extra), ["#{reason}\n"]]
+      end
+    end
+  end
+end

data/lib/monk/storage/sig_v4.rb

@@ -0,0 +1,124 @@
+require "digest"
+# Loaded here, at boot: left to Digest.const_missing it loads on the first
+# signed request, and a worker Ractor signing another in parallel can
+# catch it half-defined (test/digest_preload_test.rb).
+require "digest/sha2"
+require "openssl"
+
+module Monk
+  module Storage
+    # AWS Signature Version 4, for the S3 backend: the Authorization header
+    # of a request Monk sends, and the query string of a link a browser
+    # uses (a presigned PUT, a signed GET). Pure functions of their
+    # arguments, the time included, so they're tested against AWS's
+    # published vectors without stubbing Time (docs/history/plan-storage.md
+    # decision 10).
+    #
+    # Paths and query strings are encoded the S3 way: each byte outside
+    # A-Z a-z 0-9 - _ . ~ as %XX, "/" kept in paths, and paths encoded once
+    # and never normalized (S3 differs from other AWS services there).
+    module SigV4
+      ALGORITHM = "AWS4-HMAC-SHA256".freeze
+      # The payload hash of a presigned link: the browser sends the body,
+      # so it can't be signed in advance.
+      UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD".freeze
+      EMPTY_PAYLOAD = Digest::SHA256.hexdigest("").freeze
+      MAX_EXPIRES = 7 * 24 * 3600
+      UNRESERVED = /[^A-Za-z0-9\-_.~]/
+
+      module_function
+
+      # The headers to add to a request: x-amz-date and authorization.
+      # headers: are the ones to sign, host included (and
+      # x-amz-content-sha256 for S3); their names may be in any case.
+      # payload_hash: is the body's SHA-256 in hex. now: is a Unix time.
+      def sign_headers(method:, path:, headers:, payload_hash:, credentials:, now:, query: {})
+        date = timestamp(now)
+        signed = canonical_headers(headers.merge("x-amz-date" => date))
+        signature = signature(method, path, canonical_query(query), signed, payload_hash, credentials, date)
+        {
+          "x-amz-date" => date,
+          "authorization" => "#{ALGORITHM} Credential=#{credentials[:access_key]}/#{scope(credentials, date)}, " \
+                             "SignedHeaders=#{signed.keys.join(";")}, Signature=#{signature}",
+        }
+      end
+
+      # The query string of a presigned link, signature last: the request's
+      # own query: parameters, then the X-Amz-* ones. headers: are the ones
+      # the client must send exactly (host, and for a presigned PUT
+      # content-length and content-type).
+      def presign(method:, path:, headers:, expires_in:, credentials:, now:, query: {},
+                  payload_hash: UNSIGNED_PAYLOAD)
+        unless expires_in.is_a?(Integer) && expires_in.between?(1, MAX_EXPIRES)
+          raise ArgumentError, "a presigned link is valid 1 to #{MAX_EXPIRES} seconds, not #{expires_in.inspect}"
+        end
+
+        date = timestamp(now)
+        signed = canonical_headers(headers)
+        params = query.merge(
+          "X-Amz-Algorithm" => ALGORITHM,
+          "X-Amz-Credential" => "#{credentials[:access_key]}/#{scope(credentials, date)}",
+          "X-Amz-Date" => date,
+          "X-Amz-Expires" => expires_in.to_s,
+          "X-Amz-SignedHeaders" => signed.keys.join(";"),
+        )
+        canonical = canonical_query(params)
+        "#{canonical}&X-Amz-Signature=#{signature(method, path, canonical, signed, payload_hash, credentials, date)}"
+      end
+
+      def signature(method, path, query, headers, payload_hash, credentials, date)
+        request = canonical_request(method, path, query, headers, payload_hash)
+        OpenSSL::HMAC.hexdigest("SHA256", signing_key(credentials, date), string_to_sign(request, credentials, date))
+      end
+
+      def canonical_request(method, path, query, headers, payload_hash)
+        [
+          method,
+          encode_path(path),
+          query,
+          headers.map { |name, value| "#{name}:#{value}\n" }.join,
+          headers.keys.join(";"),
+          payload_hash,
+        ].join("\n")
+      end
+
+      def string_to_sign(canonical_request, credentials, date)
+        [ALGORITHM, date, scope(credentials, date), Digest::SHA256.hexdigest(canonical_request)].join("\n")
+      end
+
+      def signing_key(credentials, date)
+        key = "AWS4#{credentials[:secret_key]}"
+        [date[0, 8], credentials[:region], credentials.fetch(:service, "s3"), "aws4_request"].each do |part|
+          key = OpenSSL::HMAC.digest("SHA256", key, part)
+        end
+        key
+      end
+
+      def scope(credentials, date)
+        "#{date[0, 8]}/#{credentials[:region]}/#{credentials.fetch(:service, "s3")}/aws4_request"
+      end
+
+      # Lowercase names, sorted; values trimmed, inner runs of spaces made one.
+      def canonical_headers(headers)
+        headers.to_h { |name, value| [name.to_s.downcase, value.to_s.strip.gsub(/\s+/, " ")] }.sort.to_h
+      end
+
+      def canonical_query(params)
+        params.map { |name, value| [encode(name.to_s), encode(value.to_s)] }.sort
+              .map { |name, value| "#{name}=#{value}" }.join("&")
+      end
+
+      def encode_path(path)
+        path.split("/", -1).map { |segment| encode(segment) }.join("/")
+      end
+
+      def encode(text)
+        text.b.gsub(UNRESERVED) { |byte| format("%%%02X", byte.ord) }
+      end
+
+      def timestamp(now)
+        Time.at(now).utc.strftime("%Y%m%dT%H%M%SZ")
+      end
+    end
+  end
+end

data/lib/monk/storage/sniff.rb

@@ -0,0 +1,81 @@
+module Monk
+  module Storage
+    # A file's type from its first bytes, since the Content-Type a client
+    # sends can't be trusted. Defence in depth: the type a signed link
+    # serves is the one the app signed into presigned_put from its allowed
+    # list, and this catches a file whose bytes don't match it
+    # (docs/history/plan-storage.md decision 12). At confirm, read only these
+    # bytes:
+    #
+    #   head = Monk::Storage.get(key, range: Monk::Storage::Sniff::RANGE)
+    #   Monk::Storage::Sniff.refuse_svg!(head)
+    #   Monk::Storage::Sniff.content_type(head) # => "image/png" or nil
+    module Sniff
+      # 64 bytes, not 16: an Ogg page's header is 27 bytes plus one byte
+      # per segment, so OpusHead starts at byte 28 on a usual first page.
+      RANGE = 0..63
+
+      PNG = "\x89PNG\r\n\x1A\n".b.freeze
+      JPEG = "\xFF\xD8\xFF".b.freeze
+      GIFS = Ractor.make_shareable(["GIF87a".b, "GIF89a".b])
+      PDF = "%PDF-".b.freeze
+      # Where an Ogg page's segment count sits; the segment table follows.
+      OGG_SEGMENTS_AT = 26
+      # Lowercased before matching, so <SVG and <?XML count too.
+      SVG_MARKERS = Ractor.make_shareable(["<svg".b, "<?xml".b])
+
+      module_function
+
+      # "image/png", "image/jpeg", "image/gif", "image/webp", "audio/ogg"
+      # (an Ogg stream whose first packet is OpusHead), "application/pdf",
+      # or nil for anything else, an empty String included.
+      def content_type(bytes)
+        head = prefix(bytes)
+
+        if head.start_with?(PNG) then "image/png"
+        elsif head.start_with?(JPEG) then "image/jpeg"
+        elsif head.start_with?(*GIFS) then "image/gif"
+        elsif webp?(head) then "image/webp"
+        elsif opus?(head) then "audio/ogg"
+        elsif head.start_with?(PDF) then "application/pdf"
+        end
+      end
+
+      # Raises UnsafeContentError when <svg or <?xml appears in the first
+      # bytes: SVG can carry scripts, and an SVG may start with an XML
+      # declaration, a comment or a doctype before its <svg. Only RANGE is
+      # looked at, so the XMP metadata deep inside a JPEG or PDF doesn't
+      # count. Returns bytes.
+      def refuse_svg!(bytes)
+        head = prefix(bytes).downcase
+        if SVG_MARKERS.any? { |marker| head.include?(marker) }
+          raise UnsafeContentError, "the file starts like SVG or XML, which can carry scripts; refused"
+        end
+
+        bytes
+      end
+
+      def prefix(bytes)
+        unless bytes.is_a?(String)
+          raise ArgumentError,
+            "Sniff takes the file's first bytes as a String, not #{bytes.class}"
+        end
+
+        bytes.byteslice(RANGE).b
+      end
+
+      # RIFF, a 4-byte size, then WEBP.
+      def webp?(head)
+        head.start_with?("RIFF") && head.byteslice(8, 4) == "WEBP"
+      end
+
+      def opus?(head)
+        return false unless head.start_with?("OggS") && head.bytesize > OGG_SEGMENTS_AT
+
+        packet_at = OGG_SEGMENTS_AT + 1 + head.getbyte(OGG_SEGMENTS_AT)
+        head.byteslice(packet_at, 8) == "OpusHead"
+      end
+      private_class_method :prefix, :webp?, :opus?
+    end
+  end
+end

data/lib/monk/storage/token.rb

@@ -0,0 +1,45 @@
+require "json"
+require "openssl"
+
+module Monk
+  module Storage
+    # The file backend's signed links: /_monk/storage/<token> stands in
+    # for a bucket's signed URL. A token is base64url(JSON payload) + "." +
+    # base64url(HMAC-SHA256 of that first part), the payload
+    # {m: "GET"|"PUT", k: key, e: expires_at, l: length, t: type}, l and t
+    # for uploads only (docs/history/plan-storage.md decision 8).
+    module Token
+      module_function
+
+      def sign(payload, secret)
+        body = encode(JSON.generate(payload))
+        "#{body}.#{encode(OpenSSL::HMAC.digest("SHA256", secret, body))}"
+      end
+
+      # The payload, with symbol keys, or nil when the token is malformed
+      # or its signature doesn't match. Expiry is the caller's check.
+      def verify(token, secret)
+        body, signature = token.split(".", -1)
+        return nil unless body && signature && token.count(".") == 1
+
+        expected = encode(OpenSSL::HMAC.digest("SHA256", secret, body))
+        return nil unless OpenSSL.secure_compare(expected, signature)
+
+        payload = JSON.parse(decode(body), symbolize_names: true)
+        payload.is_a?(Hash) ? payload : nil
+      rescue JSON::ParserError, ArgumentError
+        nil
+      end
+
+      def encode(bytes)
+        [bytes].pack("m0").tr("+/", "-_").delete("=")
+      end
+
+      def decode(text)
+        raise ArgumentError, "not base64url" unless text.match?(/\A[A-Za-z0-9_-]*\z/)
+
+        (text.tr("-_", "+/") + ("=" * (-text.length % 4))).unpack1("m0")
+      end
+    end
+  end
+end

data/lib/monk/storage/upload.rb

@@ -0,0 +1,10 @@
+module Monk
+  module Storage
+    # What presigned_put returns: the URL the browser PUTs the file to, and
+    # exactly the headers it must send with it. Content-Type and
+    # Content-Length are inside the signature, so the bucket refuses any
+    # other (browsers set Content-Length themselves; it's listed for
+    # clarity).
+    Upload = Data.define(:url, :headers)
+  end
+end

data/lib/monk/storage/url.rb

@@ -0,0 +1,126 @@
+require "uri"
+
+module Monk
+  module Storage
+    # STORAGE_URL -> a backend value. Runs once, from Monk::Storage.configure
+    # in the main Ractor at boot, so a malformed URL fails the boot rather
+    # than the first upload (same posture as Monk::Mail::URL). Error
+    # messages redact the secret key: the URL is a secret.
+    module URL
+      SUPPORTED = "file://, s3://".freeze
+      S3_EXAMPLE = "s3://KEY:SECRET@bucket?endpoint=fsn1.your-objectstorage.com&region=fsn1".freeze
+      S3_OPTIONS = %w[endpoint region path_style].freeze
+      # S3's rule for bucket names that also work in a host name
+      # (virtual-hosted addressing, the default).
+      BUCKET = /\A[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]\z/
+      HOST = /\A[a-z0-9]([a-z0-9.-]*[a-z0-9])?(:\d{1,5})?\z/
+
+      module_function
+
+      def parse(url)
+        case url[%r{\A([a-z][a-z0-9+.-]*)://}i, 1]&.downcase
+        when "file" then file(url)
+        when "s3" then s3(url)
+        else invalid!(url, "unsupported scheme; supported: #{SUPPORTED}")
+        end
+      end
+
+      # file://storage/development (relative to the current directory) or
+      # file:///var/lib/app/storage (absolute).
+      def file(url)
+        dir = url.sub(%r{\A[a-z]+://}i, "")
+        invalid!(url, "missing directory, e.g. file://storage/production") if dir.empty?
+        invalid!(url, "a directory takes no options") if dir.include?("?")
+
+        Backends::File.new(root: File.expand_path(dir), secret: nil)
+      end
+
+      def s3(url)
+        uri = URI.parse(url)
+        options = s3_options(uri, url)
+        endpoint, tls = endpoint(options["endpoint"], url)
+        Backends::S3.new(
+          bucket: bucket(uri, url),
+          endpoint: endpoint,
+          region: required(options, "region", url),
+          access_key: credential(uri.user, "access key", url),
+          secret_key: credential(uri.password, "secret key", url),
+          path_style: path_style(options["path_style"], url),
+          tls: tls,
+        )
+      rescue URI::InvalidURIError
+        invalid!(url, "not a valid URL (percent-encode / @ : in the keys); expected e.g. #{S3_EXAMPLE}")
+      end
+
+      def s3_options(uri, url)
+        options = URI.decode_www_form(uri.query || "").to_h
+        unknown = options.keys - S3_OPTIONS
+        invalid!(url, "unknown option(s) #{unknown.join(", ")}; supported: #{S3_OPTIONS.join(", ")}") if unknown.any?
+
+        options
+      end
+
+      def bucket(uri, url)
+        invalid!(url, "a port goes in endpoint=, not after the bucket") if uri.port
+        invalid!(url, "nothing goes after the bucket but options") unless ["", "/"].include?(uri.path)
+
+        name = uri.host.to_s
+        invalid!(url, "missing bucket, e.g. #{S3_EXAMPLE}") if name.empty?
+        invalid!(url, "bucket #{name.inspect}: 3-63 of a-z, 0-9, . and -") unless name.match?(BUCKET)
+
+        name
+      end
+
+      # A bare host means HTTPS. http:// sends the credentials and every
+      # body in clear: accepted for a local server (floci in Monk's own
+      # suite), refused in staging and production.
+      def endpoint(value, url)
+        invalid!(url, "missing endpoint=, e.g. #{S3_EXAMPLE}") if value.nil? || value.empty?
+
+        scheme = value[%r{\A([a-z]+)://}i, 1]&.downcase
+        invalid!(url, "endpoint= must be https:// or http://") unless [nil, "https", "http"].include?(scheme)
+
+        host = value.sub(%r{\A[a-z]+://}i, "").chomp("/")
+        invalid!(url, "endpoint= must be a host, optionally with a port") unless host.match?(HOST)
+
+        tls = scheme != "http"
+        if !tls && (Monk.env.staging? || Monk.env.production?)
+          invalid!(url, "endpoint=http:// sends the keys in clear; it's for a local server, " \
+                        "not MONK_ENV=#{Monk.env}",)
+        end
+
+        [host.downcase, tls]
+      end
+
+      def path_style(value, url)
+        case value
+        when nil, "0" then false
+        when "1" then true
+        else invalid!(url, "path_style= must be 1 or 0")
+        end
+      end
+
+      def required(options, name, url)
+        value = options[name]
+        invalid!(url, "missing #{name}=, e.g. #{S3_EXAMPLE}") if value.nil? || value.empty?
+
+        value
+      end
+
+      # A secret with @, : or / in it has to be percent-encoded in the URL.
+      def credential(component, name, url)
+        invalid!(url, "missing #{name}, e.g. #{S3_EXAMPLE}") if component.nil? || component.empty?
+
+        URI.decode_uri_component(component)
+      end
+
+      def invalid!(url, reason)
+        raise InvalidStorageUrlError, "STORAGE_URL #{redact(url).inspect}: #{reason}"
+      end
+
+      def redact(url)
+        url.to_s.sub(%r{(://[^:/@]*:)[^@/]*@}, '\1***@')
+      end
+    end
+  end
+end