butler-http 0.1.0

data/lib/butler/security/host_policy.rb

@@ -0,0 +1,29 @@
+class Butler
+  module Security
+    # An allow/block-list check against the request's host. This is
+    # deliberately simple (glob matching, no DNS-rebinding protection, no
+    # IP-literal/private-range awareness) — it's SSRF-*lite*, a cheap first
+    # line of defense, not a substitute for application-level SSRF
+    # protection, exactly as the PRD calls out.
+    module HostPolicy
+      module_function
+
+      def check!(uri, config)
+        security = config.security
+        host = uri.host.to_s
+
+        if security.allowed_hosts && security.allowed_hosts.none? { |pattern| match?(pattern, host) }
+          raise Butler::Errors::HostNotAllowed, "#{host} is not in security.allowed_hosts"
+        end
+
+        if security.blocked_hosts.any? { |pattern| match?(pattern, host) }
+          raise Butler::Errors::HostNotAllowed, "#{host} is in security.blocked_hosts"
+        end
+      end
+
+      def match?(pattern, host)
+        ::File.fnmatch?(pattern.to_s, host, ::File::FNM_CASEFOLD)
+      end
+    end
+  end
+end

data/lib/butler/security/limits.rb

@@ -0,0 +1,39 @@
+class Butler
+  module Security
+    # Enforces response header/body size caps. Header size is checked in one
+    # shot once the response headers are in hand; body size is checked
+    # incrementally as Transport reads chunks off the wire, so an
+    # oversized/slow-drip response is aborted rather than fully buffered
+    # into memory first.
+    module Limits
+      module_function
+
+      def check_headers!(headers, config)
+        max = config.security.max_header_size
+        return unless max
+
+        total = headers.to_a.sum { |k, v| k.bytesize + v.to_s.bytesize }
+        return unless total > max
+
+        raise Butler::Errors::LimitExceeded, "response headers exceeded security.max_header_size (#{max} bytes)"
+      end
+
+      # Reads +raw_body+ (anything responding to #each, yielding byte-string
+      # chunks) into a single buffered String, raising once it would exceed
+      # security.max_response_size. Returns "" if raw_body is nil.
+      def read_body(raw_body, config)
+        buffer = +""
+        return buffer unless raw_body
+
+        max = config.security.max_response_size
+        raw_body.each do |chunk|
+          buffer << chunk
+          next unless max && buffer.bytesize > max
+
+          raise Butler::Errors::LimitExceeded, "response body exceeded security.max_response_size (#{max} bytes)"
+        end
+        buffer
+      end
+    end
+  end
+end

data/lib/butler/security/redirect_policy.rb

@@ -0,0 +1,28 @@
+class Butler
+  module Security
+    # Pure decision logic for following a redirect; the actual hop-by-hop
+    # loop lives in Client#request (see butler/client.rb) since each hop
+    # needs a fresh HostPolicy check and independent
+    # security/telemetry/circuit-breaker accounting against the *new* host
+    # while still sharing one Deadline across the whole call.
+    module RedirectPolicy
+      module_function
+
+      # 303 always downgrades to GET. 301/302 downgrade to GET too, unless
+      # the original request was HEAD — this matches how browsers (and most
+      # HTTP client libraries) behave in practice, rather than the stricter
+      # original RFC 7231 reading that would replay POST on 301/302.
+      def method_for(status, current_method)
+        case status
+        when 303 then "GET"
+        when 301, 302 then (current_method == "HEAD" ? "HEAD" : "GET")
+        else current_method
+        end
+      end
+
+      def strip_credentials?(config, from_uri, to_uri)
+        config.security.strip_credentials_on_redirect && from_uri.cross_origin?(to_uri)
+      end
+    end
+  end
+end

data/lib/butler/security/tls.rb

@@ -0,0 +1,51 @@
+class Butler
+  module Security
+    # TLS + hostname verification are on by default and only turned off by
+    # an explicit security.verify_tls = false — and doing so logs one loud
+    # warning per process so it's never silently insecure.
+    module TLS
+      module_function
+
+      def context_for(config)
+        context = ::OpenSSL::SSL::SSLContext.new
+        if config.security.verify_tls
+          # #set_params (not a bare verify_mode= assignment) is what loads
+          # the system's trusted CA store — a plain SSLContext.new has none
+          # at all, so every certificate would fail verification regardless
+          # of how legitimate it is.
+          context.set_params(verify_mode: ::OpenSSL::SSL::VERIFY_PEER)
+        else
+          warn_once!
+          context.set_params(verify_mode: ::OpenSSL::SSL::VERIFY_NONE)
+        end
+
+        # Async::HTTP::Endpoint's own `alpn_protocols:` option only takes
+        # effect on a context *it* builds internally — handing it a custom
+        # ssl_context (as Butler always does, for verify_mode/CA control)
+        # bypasses that, silently falling back to HTTP/1.1 negotiation.
+        # Setting alpn_protocols directly on this context is what actually
+        # makes ALPN negotiate h2 when both ends support it.
+        context.alpn_protocols = alpn_protocols_for(config)
+        context
+      end
+
+      def alpn_protocols_for(config)
+        case config.http_version
+        when :http1 then ["http/1.1"]
+        when :http2 then ["h2"]
+        else defined?(::Async::HTTP::Protocol::HTTPS) ? ::Async::HTTP::Protocol::HTTPS.names : ["h2", "http/1.1"]
+        end
+      end
+
+      def warn_once!
+        return if @warned
+
+        @warned = true
+        Butler::Telemetry::Logger.warn(
+          "Butler: TLS certificate verification is DISABLED (security.verify_tls = false). " \
+          "Do not use this outside of local development/testing.",
+        )
+      end
+    end
+  end
+end

data/lib/butler/stream.rb

@@ -0,0 +1,44 @@
+# Wraps a streaming response body for callers who pass stream: true, without
+# ever exposing the underlying Protocol::HTTP::Body::Readable that produced
+# it — Transport::Async is the only place that constructs one.
+class Butler::Stream
+  def initialize(&reader)
+    @reader = reader
+    @on_close = nil
+    @closed = false
+  end
+
+  def on_close(&block)
+    @on_close = block
+    self
+  end
+
+  # Yields each chunk in turn, closing the underlying connection once the
+  # caller has consumed (or abandoned) the stream.
+  def each_chunk
+    return enum_for(:each_chunk) unless block_given?
+
+    @reader.call { |chunk| yield chunk }
+  ensure
+    close
+  end
+
+  def read_all
+    buffer = +""
+    each_chunk { |chunk| buffer << chunk }
+    buffer
+  end
+  alias read read_all
+  alias to_s read_all
+
+  def close
+    return if @closed
+
+    @closed = true
+    @on_close&.call
+  end
+
+  def closed?
+    @closed
+  end
+end

data/lib/butler/telemetry/instrumentation.rb

@@ -0,0 +1,59 @@
+class Butler
+  module Telemetry
+    # A standalone publish/subscribe mechanism — no ActiveSupport required.
+    # #instrument measures wall-clock duration around the block, notifies
+    # every plain subscriber, and (only if ActiveSupport happens to already
+    # be loaded) also fires the equivalent ActiveSupport::Notifications
+    # event, so Rails-side tooling (e.g. the Rails log subscriber, or a
+    # custom one an app registers) sees "butler.request" for free.
+    #
+    # +payload+ is a plain Hash built from an explicit allow-list of fields
+    # by TelemetryMiddleware (method/host/port/status/duration/attempt/
+    # error) — this file never reads request/response bodies or
+    # Authorization/Cookie/Set-Cookie headers, so there's no redaction step
+    # that could be forgotten.
+    module Instrumentation
+      @subscribers = Hash.new { |hash, key| hash[key] = [] }
+      @mutex = Mutex.new
+
+      class << self
+        def subscribe(event, &block)
+          @mutex.synchronize { @subscribers[event.to_sym] << block }
+          block
+        end
+
+        def instrument(event, payload = {})
+          start = monotonic_now
+          result = with_active_support(event, payload) { yield payload }
+          payload[:duration] = monotonic_now - start
+          notify(event, payload)
+          result
+        rescue StandardError => e
+          payload[:duration] ||= monotonic_now - start
+          payload[:error] ||= e.class.name
+          notify(event, payload)
+          raise
+        end
+
+        private
+
+        def notify(event, payload)
+          subscribers = @mutex.synchronize { @subscribers[event.to_sym].dup }
+          subscribers.each { |subscriber| subscriber.call(payload.dup) }
+        end
+
+        def with_active_support(event, payload, &block)
+          if defined?(::ActiveSupport::Notifications)
+            ::ActiveSupport::Notifications.instrument("butler.#{event}", payload, &block)
+          else
+            block.call
+          end
+        end
+
+        def monotonic_now
+          ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        end
+      end
+    end
+  end
+end

data/lib/butler/telemetry/logger.rb

@@ -0,0 +1,37 @@
+class Butler
+  module Telemetry
+    # One structured log line per completed request by default (subscribed
+    # below), plus a couple of standalone warnings (e.g. TLS verification
+    # disabled) other parts of Butler call directly. Uses
+    # Butler.configuration.telemetry.logger when set (Butler::Rails::Railtie
+    # points this at Rails.logger) and otherwise a plain
+    # Logger.new($stdout).
+    module Logger
+      module_function
+
+      def info(payload)
+        logger.info(format_payload(payload))
+      end
+
+      def warn(message)
+        logger.warn(message)
+      end
+
+      def logger
+        Butler.configuration.telemetry.logger || default_logger
+      end
+
+      def default_logger
+        @default_logger ||= ::Logger.new($stdout)
+      end
+
+      def format_payload(payload)
+        payload.map { |key, value| "#{key}=#{value.inspect}" }.join(" ")
+      end
+    end
+  end
+end
+
+Butler::Telemetry::Instrumentation.subscribe(:request) do |payload|
+  Butler::Telemetry::Logger.info(payload) if Butler.configuration.telemetry.enabled
+end

data/lib/butler/telemetry/open_telemetry_bridge.rb

@@ -0,0 +1,46 @@
+class Butler
+  module Telemetry
+    # Only ever touched when the host application has already loaded
+    # opentelemetry-api itself — Butler never requires it. Wraps the *real*
+    # request execution in a proper span (rather than recording one after
+    # the fact), so span timing and any downstream context propagation
+    # (e.g. into the outgoing request headers, if the app wires that up via
+    # its own OTel HTTP instrumentation) behaves like any other traced call.
+    #
+    # NOTE: the attribute names below (http.request.method, server.address,
+    # server.port, http.response.status_code, network.protocol.name/
+    # version, error.type) reflect OpenTelemetry's HTTP semantic conventions
+    # as of this writing. Semantic conventions have changed before and will
+    # again — re-verify against the installed opentelemetry-semantic-
+    # conventions gem rather than trusting this comment indefinitely.
+    module OpenTelemetryBridge
+      module_function
+
+      def available?(config)
+        return false unless config.telemetry.opentelemetry
+        return false if config.telemetry.opentelemetry == false
+
+        defined?(::OpenTelemetry) && ::OpenTelemetry.respond_to?(:tracer_provider)
+      end
+
+      def in_span(payload)
+        tracer.in_span("HTTP #{payload[:method]}", kind: :client) do |span|
+          result = yield
+          span.set_attribute("http.request.method", payload[:method].to_s)
+          span.set_attribute("server.address", payload[:host].to_s) if payload[:host]
+          span.set_attribute("server.port", payload[:port]) if payload[:port]
+          span.set_attribute("http.response.status_code", payload[:status]) if payload[:status]
+          span.set_attribute("network.protocol.name", "http")
+          result
+        rescue StandardError => e
+          span.set_attribute("error.type", e.class.name)
+          raise
+        end
+      end
+
+      def tracer
+        ::OpenTelemetry.tracer_provider.tracer("butler-http", Butler::VERSION)
+      end
+    end
+  end
+end

data/lib/butler/testing/fake_transport.rb

@@ -0,0 +1,29 @@
+class Butler
+  module Testing
+    # Implements the same contract as Butler::Transport::Async (see
+    # butler/transport.rb) but never opens a real socket — it just matches
+    # the request against Butler::Testing::StubRegistry. Because it's
+    # selected by Butler::Transport.current at call time and plugged into
+    # the *same* pipeline, a stubbed response/error still flows through
+    # RetryMiddleware/CircuitBreakerMiddleware/TelemetryMiddleware exactly
+    # like a real one would.
+    module FakeTransport
+      module_function
+
+      def build_connection(uri, _config)
+        Butler::Connection.new(origin_key: uri.origin_key, async_client: nil, http_version: :fake)
+      end
+
+      def call(_connection, request, _config, timeout: nil)
+        stub = Butler::Testing::StubRegistry.match(request)
+        unless stub
+          raise Butler::Errors::RequestError,
+                "no stub registered for #{request.method} #{request.uri} " \
+                "(call Butler::Testing.stub_request(#{request.method.downcase.inspect}, \"#{request.uri.uri.path}\") first)"
+        end
+
+        stub.response_for(request)
+      end
+    end
+  end
+end

data/lib/butler/testing/stub.rb

@@ -0,0 +1,68 @@
+class Butler
+  module Testing
+    # One registered stub: a method+URL/path matcher plus the canned
+    # response (or error) to hand back. Deliberately smaller than WebMock's
+    # API — one matcher shape (method + URL-or-path, optional header match),
+    # one response builder.
+    class Stub
+      def initialize(method, matcher)
+        @method = method.to_s.upcase
+        @matcher = matcher.to_s
+        @with_headers = nil
+        @response_status = 200
+        @response_headers = Butler::Headers.new
+        @response_body = ""
+        @raise_error = nil
+      end
+
+      def with(headers: nil)
+        @with_headers = headers
+        self
+      end
+
+      def to_return(status: 200, json: nil, body: nil, headers: {})
+        @response_status = status
+        @response_headers = Butler::Headers.coerce(headers)
+        @response_body =
+          if json
+            @response_headers["Content-Type"] = "application/json" unless @response_headers.key?("content-type")
+            json.to_json
+          else
+            body.to_s
+          end
+        self
+      end
+
+      def to_raise(error)
+        @raise_error = error
+        self
+      end
+
+      def match?(request)
+        request.method == @method && target_matches?(request) && headers_match?(request)
+      end
+
+      def response_for(request)
+        raise(@raise_error.is_a?(Class) ? @raise_error.new : @raise_error) if @raise_error
+
+        Butler::Response.new(uri: request.uri, status: @response_status, headers: @response_headers, body: @response_body)
+      end
+
+      private
+
+      def target_matches?(request)
+        if @matcher =~ %r{\Ahttps?://}i
+          request.uri.to_s.split("?").first == @matcher.split("?").first
+        else
+          request.uri.uri.path == @matcher
+        end
+      end
+
+      def headers_match?(request)
+        return true unless @with_headers
+
+        @with_headers.all? { |k, v| request.headers[k].to_s == v.to_s }
+      end
+    end
+  end
+end

data/lib/butler/testing/stub_registry.rb

@@ -0,0 +1,32 @@
+class Butler
+  module Testing
+    # Process-wide, thread-safe registry of active stubs. Its #active? is
+    # checked at the *start of every request* (see Butler::Transport.current)
+    # rather than once at Client construction time, so stubbing/unstubbing
+    # around a Butler::Client instance that already exists (the normal case
+    # in a test suite's setup/teardown) works correctly.
+    module StubRegistry
+      @stubs = []
+      @mutex = Mutex.new
+
+      class << self
+        def register(stub)
+          @mutex.synchronize { @stubs << stub }
+          stub
+        end
+
+        def match(request)
+          @mutex.synchronize { @stubs.reverse_each.find { |stub| stub.match?(request) } }
+        end
+
+        def active?
+          @mutex.synchronize { !@stubs.empty? }
+        end
+
+        def reset!
+          @mutex.synchronize { @stubs.clear }
+        end
+      end
+    end
+  end
+end

data/lib/butler/testing.rb

@@ -0,0 +1,33 @@
+class Butler
+  # Native request stubbing — no WebMock/VCR dependency needed. Stubs are
+  # matched at the transport boundary, so everything above it (retry,
+  # circuit breaker, telemetry) behaves exactly as it would against a real
+  # server.
+  #
+  #   Butler::Testing.stub("GET", "https://api.example.com/users/1", status: 200, json: { id: 1 })
+  #
+  #   stub = Butler::Testing.stub_request(:get, "/users/1")
+  #   stub.to_return(status: 200, json: { id: 1 })
+  #
+  # Remember to call Butler::Testing.reset! between tests (e.g. in an
+  # after(:each) hook) — stubs are process-wide, not scoped to a Client.
+  module Testing
+    module_function
+
+    def stub(method, url, status: 200, json: nil, body: nil, headers: {})
+      stub_request(method, url).to_return(status: status, json: json, body: body, headers: headers)
+    end
+
+    def stub_request(method, url_or_path)
+      StubRegistry.register(Stub.new(method, url_or_path))
+    end
+
+    def reset!
+      StubRegistry.reset!
+    end
+
+    def active?
+      StubRegistry.active?
+    end
+  end
+end

data/lib/butler/transport.rb

@@ -0,0 +1,118 @@
+require "async"
+require "async/http/client"
+require "async/http/endpoint"
+require "protocol/http/body/buffered"
+
+class Butler
+  module Transport
+    # The contract any transport implements — the real Async-backed one
+    # below, or Butler::Testing::FakeTransport. Not enforced through
+    # inheritance (Ruby doesn't need that to get the benefit); this exists
+    # purely as the documented interface a future real HTTP/3 transport
+    # would also implement.
+    module Base
+      # @return [Butler::Connection]
+      def self.build_connection(uri, config)
+        raise NotImplementedError
+      end
+
+      # @return [Butler::Response]
+      def self.call(connection, request, config, timeout:)
+        raise NotImplementedError
+      end
+    end
+
+    # Which transport module is live right now. A single switch point used
+    # by both ConnectionPool (to build the right kind of Connection) and
+    # Client's pipeline terminal (to dispatch the actual call), so stubbing
+    # activated via Butler::Testing after a Client already exists still
+    # takes effect on its very next request.
+    def self.current
+      Butler::Testing::StubRegistry.active? ? Butler::Testing::FakeTransport : Async
+    end
+
+    # Wraps Async::HTTP::Client to give Butler real HTTP/1.1 *and* HTTP/2
+    # (transparent ALPN negotiation over TLS), Fiber-native I/O, and
+    # per-origin connection reuse/multiplexing — all implemented by the
+    # `async`/`async-http` gems rather than hand-written here. This is the
+    # only file in the gem that touches Async::HTTP/Protocol::HTTP types;
+    # everything above it deals exclusively in Butler::Request/Response.
+    #
+    # Async::HTTP::Client's own built-in retry is explicitly disabled
+    # (retries: 0 below) so Butler::Resilience::RetryPolicy is the single
+    # source of truth for retry decisions — otherwise a request could be
+    # silently retried twice (once inside async-http, once inside Butler's
+    # pipeline) and the Deadline accounting would no longer be accurate.
+    module Async
+      module_function
+
+      def build_connection(uri, config)
+        endpoint = build_endpoint(uri, config)
+        async_client = ::Async::HTTP::Client.new(endpoint, retries: 0)
+        Butler::Connection.new(origin_key: uri.origin_key, async_client: async_client, http_version: config.http_version)
+      end
+
+      def call(connection, request, config, timeout:)
+        headers = request.headers.to_a
+        body = build_request_body(request.body)
+
+        raw_response = Butler::Resilience::Timeout.enforce(timeout) do
+          connection.async_client.public_send(http_method(request.method), request.uri.request_target, headers, body)
+        end
+
+        build_response(request, config, raw_response)
+      rescue ::Errno::ECONNREFUSED, ::Errno::ECONNRESET, ::Errno::EPIPE, ::EOFError, ::SocketError => e
+        raise Butler::Errors::ConnectionError, "#{e.class}: #{e.message} requesting #{request.uri}"
+      rescue ::OpenSSL::SSL::SSLError => e
+        raise tls_error_class(e), "#{e.class}: #{e.message} requesting #{request.uri}"
+      end
+
+      # OpenSSL's own verification-failure text is a stable, unified
+      # "certificate verify failed (<reason>)" across self-signed/expired/
+      # hostname-mismatch/untrusted-root — confirmed live against
+      # badssl.com's test certificates for all four. Anything else raised
+      # as OpenSSL::SSL::SSLError (a handshake timeout, a reset
+      # mid-handshake) stays a plain TLSError, since those genuinely can
+      # be worth retrying — unlike a bad certificate, which won't become
+      # valid on the next attempt (see RetryPolicy::NON_RETRYABLE_EXCEPTIONS).
+      def tls_error_class(ssl_error)
+        ssl_error.message.include?("certificate verify failed") ? Butler::Errors::CertificateVerificationError : Butler::Errors::TLSError
+      end
+
+      def build_endpoint(uri, config)
+        options = { timeout: config.connect_timeout }
+        # ALPN protocol selection lives entirely in Security::TLS.context_for
+        # (it has to be set on the ssl_context object itself, not passed as
+        # a separate Endpoint.parse option — see that method for why).
+        options[:ssl_context] = Butler::Security::TLS.context_for(config) if uri.scheme == "https"
+        ::Async::HTTP::Endpoint.parse(uri.origin_key, **options)
+      end
+
+      def http_method(method)
+        method.to_s.downcase.to_sym
+      end
+
+      def build_request_body(body)
+        return nil if body.nil? || body.empty?
+
+        ::Protocol::HTTP::Body::Buffered.wrap(body.to_s)
+      end
+
+      def build_response(request, config, raw_response)
+        headers = Butler::Headers.from_a(raw_response.headers.to_a)
+        Butler::Security::Limits.check_headers!(headers, config)
+
+        body =
+          if request.stream?
+            Butler::Stream.new { |&block| raw_response.body&.each(&block) }.on_close { raw_response.close }
+          else
+            buffered = Butler::Security::Limits.read_body(raw_response.body, config)
+            raw_response.close
+            buffered
+          end
+
+        Butler::Response.new(uri: request.uri, status: raw_response.status, headers: headers, body: body)
+      end
+    end
+  end
+end