contractful 0.1.0

data/lib/contractful/abstract_response.rb

@@ -0,0 +1,301 @@
+# frozen_string_literal: true
+
+require 'cgi'
+require 'http-cookie'
+
+# Mixin giving responses (and raw responses) their shared accessors and
+# behavior: status/header/body inspection, cookie handling and return
+# semantics.
+module Contractful
+  # Response behavior shared by Response and RawResponse: status and header
+  # inspection, cookie handling, redirection history and return semantics.
+  module AbstractResponse
+    # Headers that must never be forwarded to a different origin when
+    # following a redirect (RFC 9110 §15.4: a client should not forward
+    # credentials to a third party).
+    SENSITIVE_REDIRECT_HEADERS = %w[authorization proxy-authorization cookie].freeze
+
+    attr_reader :net_http_res, :request, :start_time, :end_time, :duration
+
+    def inspect
+      raise NotImplementedError.new('must override in subclass')
+    end
+
+    # Logger from the request, potentially nil.
+    def log
+      request.log
+    end
+
+    def log_response
+      return unless log
+
+      code = net_http_res.code
+      res_name = net_http_res.class.to_s.gsub(/\ANet::HTTP/, '')
+      content_type = (net_http_res['Content-type'] || '').gsub(/;.*\z/, '')
+
+      log << "# => #{code} #{res_name} | #{content_type} #{size} bytes, #{format('%.2f', duration)}s\n"
+    end
+
+    # HTTP status code
+    def code
+      @code ||= @net_http_res.code.to_i
+    end
+
+    def history
+      @history ||= request.redirection_history || []
+    end
+
+    # A hash of the headers, beautified with symbols and underscores.
+    # e.g. "Content-type" will become :content_type.
+    def headers
+      @headers ||= AbstractResponse.beautify_headers(@net_http_res.to_hash)
+    end
+
+    # Parsed rate-limit information ({Contractful::RateLimit}), or nil when
+    # the server did not send any rate-limit headers. Memoized.
+    def rate_limit
+      @rate_limit ||= RateLimit.from_headers(headers)
+    end
+
+    # True when the response indicates rate limiting: a 429 status, or a
+    # server-reported remaining quota of zero.
+    def rate_limited?
+      return true if code == 429
+
+      rate_limit ? rate_limit.exhausted? : false
+    end
+
+    # The raw headers.
+    def raw_headers
+      @raw_headers ||= @net_http_res.to_hash
+    end
+
+    # @param [Net::HTTPResponse] net_http_res
+    # @param [Contractful::Request] request
+    # @param [Time] start_time
+    def response_set_vars(net_http_res, request, start_time)
+      @net_http_res = net_http_res
+      @request = request
+      @start_time = start_time
+      @end_time = Time.now
+
+      @duration = @start_time ? @end_time - @start_time : nil
+
+      # prime redirection history
+      history
+    end
+
+    # Hash of cookies extracted from response headers.
+    #
+    # NB: This will return only cookies whose domain matches this request, and
+    # may not even return all of those cookies if there are duplicate names.
+    # Use the full cookie_jar for more nuanced access.
+    #
+    # @see #cookie_jar
+    #
+    # @return [Hash]
+    #
+    def cookies
+      hash = {}
+
+      cookie_jar.cookies(@request.uri).each do |cookie|
+        hash[cookie.name] = cookie.value
+      end
+
+      hash
+    end
+
+    # Cookie jar extracted from response headers.
+    #
+    # @return [HTTP::CookieJar]
+    #
+    def cookie_jar
+      return @cookie_jar if defined?(@cookie_jar) && @cookie_jar
+
+      jar = @request.cookie_jar.dup
+      headers.fetch(:set_cookie, []).each do |cookie|
+        jar.parse(cookie, @request.uri)
+      end
+
+      @cookie_jar = jar
+    end
+
+    # Return the default behavior corresponding to the response code:
+    #
+    # For 2xx status codes: return the response itself
+    #
+    # For 30x status codes:
+    #   301, 302, 307: redirect GET / HEAD if there is a Location header
+    #   303: redirect, changing method to GET, if there is a Location header
+    #
+    # For all other responses, raise a response exception
+    #
+    def return!(&block)
+      case code
+      when 200..299
+        self
+      when 301, 302, 307
+        case request.method
+        when 'get', 'head'
+          check_max_redirects
+          follow_redirection(&block)
+        else
+          raise exception_with_response
+        end
+      when 303
+        check_max_redirects
+        follow_get_redirection(&block)
+      else
+        raise exception_with_response
+      end
+    end
+
+    def to_i
+      warn('warning: calling Response#to_i is not recommended')
+      super
+    end
+
+    def description
+      "#{code} #{STATUSES[code]} | #{(headers[:content_type] || '').gsub(/;.*$/, '')} #{size} bytes\n"
+    end
+
+    # Follow a redirection response by making a new HTTP request to the
+    # redirection target.
+    def follow_redirection(&block)
+      _follow_redirection(request.args.dup, &block)
+    end
+
+    # Follow a redirection response, but change the HTTP method to GET and drop
+    # the payload from the original request.
+    def follow_get_redirection(&block)
+      new_args = request.args.dup
+      new_args[:method] = :get
+      new_args.delete(:payload)
+
+      _follow_redirection(new_args, &block)
+    end
+
+    # Convert headers hash into canonical form.
+    #
+    # Header names will be converted to lowercase symbols with underscores
+    # instead of hyphens.
+    #
+    # Headers specified multiple times will be joined by comma and space,
+    # except for Set-Cookie, which will always be an array.
+    #
+    # Per RFC 2616, if a server sends multiple headers with the same key, they
+    # MUST be able to be joined into a single header by a comma. However,
+    # Set-Cookie (RFC 6265) cannot because commas are valid within cookie
+    # definitions. The newer RFC 7230 notes (3.2.2) that Set-Cookie should be
+    # handled as a special case.
+    #
+    # http://tools.ietf.org/html/rfc2616#section-4.2
+    # http://tools.ietf.org/html/rfc7230#section-3.2.2
+    # http://tools.ietf.org/html/rfc6265
+    #
+    # @param headers [Hash]
+    # @return [Hash]
+    #
+    def self.beautify_headers(headers)
+      headers.each_with_object({}) do |(key, value), out|
+        key_sym = key.tr('-', '_').downcase.to_sym
+
+        # Handle Set-Cookie specially since it cannot be joined by comma.
+        out[key_sym] =
+          if key.downcase == 'set-cookie'
+            value
+          else
+            value.join(', ')
+          end
+      end
+    end
+
+    private
+
+    # Follow a redirection
+    #
+    # @param new_args [Hash] Start with this hash of arguments for the
+    #   redirection request. The hash will be mutated, so be sure to dup any
+    #   existing hash that should not be modified.
+    #
+    def _follow_redirection(new_args, &block)
+      # parse location header and merge into existing URL
+      location = headers[:location]
+
+      # cannot follow redirection if there is no location header
+      raise exception_with_response unless location
+
+      target = URI.parse(location)
+      target = request.uri.merge(target) unless target.absolute?
+
+      # Never downgrade an encrypted request to plaintext.
+      raise exception_with_response if request.uri.scheme == 'https' && target.scheme == 'http'
+
+      # Reject redirect targets that are not http/https. Without this
+      # check, contractful follows file://, data:, gopher:// and other
+      # exotic schemes; Net::HTTP then fails with a confusing error
+      # after the request itself has been issued. More importantly, the
+      # SSRF surface is wider than intended: a 302 to http://localhost:6379
+      # or http://169.254.169.254 (cloud metadata) is followed as-is.
+      # Sensitive headers (Authorization, Cookie) are stripped cross-origin
+      # below, but the request body still goes out to an attacker-chosen
+      # target. Restrict to http/https so at least the protocol surface
+      # stays bounded.
+      raise exception_with_response unless %w[http https].include?(target.scheme.to_s.downcase)
+
+      new_args[:url] = target.to_s
+
+      if same_origin?(request.uri, target)
+        new_args[:password] = request.password
+        new_args[:user] = request.user
+        new_args[:headers] = request.headers
+      else
+        # A redirect to another origin must not receive the caller's
+        # credentials (basic auth or an explicit Authorization/Cookie header).
+        new_args.delete(:user)
+        new_args.delete(:password)
+        new_args[:headers] = without_sensitive_headers(request.headers)
+      end
+      new_args[:max_redirects] = request.max_redirects - 1
+
+      # pass through our new cookie jar
+      new_args[:cookies] = cookie_jar
+
+      # prepare new request
+      new_req = Request.new(new_args)
+
+      # append self to redirection history
+      new_req.redirection_history = history + [self]
+
+      # execute redirected request
+      new_req.execute(&block)
+    end
+
+    def same_origin?(from, to)
+      from.scheme == to.scheme && from.host == to.host && from.port == to.port
+    end
+
+    def without_sensitive_headers(headers)
+      headers.reject do |key, _value|
+        SENSITIVE_REDIRECT_HEADERS.include?(key.to_s.downcase)
+      end
+    end
+
+    def check_max_redirects
+      return unless request.max_redirects <= 0
+
+      raise exception_with_response
+      
+    end
+
+    def exception_with_response
+      begin
+        klass = Exceptions::EXCEPTIONS_MAP.fetch(code)
+      rescue KeyError
+        raise RequestFailed.new(self, code)
+      end
+
+      raise klass.new(self, code)
+    end
+  end
+end

data/lib/contractful/circuit_breaker.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+module Contractful
+  # Raised instead of sending a request when the circuit for the target
+  # host is open. Not retryable within the same call — the breaker exists
+  # precisely to stop hammering a host that is failing; back off at a
+  # higher level and try again after the cooldown.
+  # (Contractful::Exception is itself a RuntimeError; the cop's literal
+  # lint on '< Exception' does not see through the constant.)
+  class CircuitOpenError < Exception # rubocop:disable Lint/InheritException
+    # The base Contractful::Exception#initialize takes a response object,
+    # not a message — take the message explicitly.
+    def initialize(message = nil)
+      super()
+      @message = message
+    end
+
+    def retryable?
+      false
+    end
+  end
+
+  # A host-level circuit breaker (closed → open → half-open → closed).
+  #
+  # Counted as failures: network errors from {Request::RETRIABLE_EXCEPTIONS}
+  # and responses carrying a retryable status (408/425/429/5xx — see
+  # {Exceptions::RETRIABLE_STATUSES}). Everything else, including 4xx
+  # client errors, is a success: a circuit must not open because a caller
+  # keeps asking for a missing resource.
+  #
+  # Breakers are process-wide, keyed by scheme/host/port and guarded by a
+  # mutex; while half-open, requests are allowed through and each outcome
+  # decides the next state. State transitions post a `circuit` event to
+  # {Notifications} with the key and both states.
+  #
+  # Opt in per request with `breaker: true` (defaults) or
+  # `breaker: { threshold: 5, cooldown: 30 }`:
+  #
+  #   Contractful.get('https://api.example.com/x', breaker: true)
+  #
+  # After `threshold` consecutive failures the circuit opens and every
+  # request to that host fails fast with {CircuitOpenError} for `cooldown`
+  # seconds; the next request after the cooldown probes the host
+  # (half-open) and its outcome closes or re-opens the circuit. Failures
+  # are counted per logical request — the retry layer inside a request is
+  # already absorbing transient errors, so only a request that ultimately
+  # failed counts.
+  class CircuitBreaker
+    DEFAULT_THRESHOLD = 5
+    DEFAULT_COOLDOWN = 30.0
+
+    CLOCK = Process::CLOCK_MONOTONIC
+
+    attr_reader :key, :threshold, :cooldown
+
+    def initialize(key, threshold: DEFAULT_THRESHOLD, cooldown: DEFAULT_COOLDOWN)
+      raise ArgumentError, 'threshold must be a positive Integer' unless threshold.is_a?(Integer) && threshold.positive?
+      raise ArgumentError, 'cooldown must be a positive Numeric' unless cooldown.is_a?(Numeric) && cooldown.positive?
+
+      @key = key
+      @threshold = threshold
+      @cooldown = cooldown
+      @state = :closed
+      @failures = 0
+      @opened_at = nil
+      @mutex = Mutex.new
+    end
+
+    # Current state, lazily transitioning open → half-open once the
+    # cooldown has elapsed. Notifications are posted after releasing the
+    # lock so that subscribers may safely inspect the breaker.
+    def state
+      from, current = @mutex.synchronize { advance_open_state }
+      post_transition(from, :half_open) if from
+      current
+    end
+
+    # Raise {CircuitOpenError} if the circuit is open; otherwise allow the
+    # request (a half-open circuit lets the probe through).
+    def check!
+      from = nil
+      error = @mutex.synchronize do
+        from, = advance_open_state
+        @state == :open ? open_error : nil
+      end
+      post_transition(from, :half_open) if from
+      raise error if error
+    end
+
+    def record_success
+      from = nil
+      @mutex.synchronize do
+        from = @state unless @state == :closed
+        @state = :closed
+        @failures = 0
+        @opened_at = nil
+      end
+      post_transition(from, :closed) if from
+      nil
+    end
+
+    def record_failure
+      from = @mutex.synchronize { count_failure! }
+      post_transition(from, :open) if from
+      nil
+    end
+
+    # Judge a transmitted response: retryable statuses are failures,
+    # everything else (4xx included) is a success. An unknown result (for
+    # example the raw `block_response` path, which has no Response object)
+    # is not evidence of success and leaves the breaker unchanged.
+    def record_response(response)
+      return unless response.respond_to?(:code)
+
+      code = response.code.to_i
+      return if code.zero?
+
+      if Exceptions.retryable?(code)
+        record_failure
+      else
+        record_success
+      end
+    end
+
+    # Process-wide registry; breaker_for lazily creates one breaker per
+    # scheme/host/port key.
+    #
+    # The previous `||=` form was not atomic: two threads racing on
+    # first use both observed @registry as nil and each built a fresh
+    # Registry — one was discarded, losing track of the breakers it
+    # had created (so the breaker state diverged per-thread for the
+    # lifetime of the process until the next reset). Use a Mutex
+    # constant to serialize the lazy init. A constant is used because
+    # methods defined inside `class << self` execute with self = the
+    # class, but instance variables assigned inside the `class << self`
+    # block live on the singleton class — the two scopes do not match.
+    REGISTRY_MUTEX = Mutex.new
+
+    class << self
+      def registry
+        REGISTRY_MUTEX.synchronize { @registry ||= Registry.new }
+      end
+
+      def reset_registry
+        REGISTRY_MUTEX.synchronize { @registry = nil }
+      end
+    end
+
+    # Lazily-materializing breaker store. The first configuration for an
+    # origin wins: later calls for the same scheme/host/port reuse the
+    # existing breaker and ignore their threshold/cooldown options.
+    class Registry
+      def initialize
+        @breakers = {}
+        @mutex = Mutex.new
+      end
+
+      def breaker_for(uri, threshold: DEFAULT_THRESHOLD, cooldown: DEFAULT_COOLDOWN)
+        key = "#{uri.scheme}://#{uri.host}:#{uri.port}"
+        @mutex.synchronize do
+          @breakers[key] ||= CircuitBreaker.new(key, threshold: threshold, cooldown: cooldown)
+        end
+      end
+
+      def clear!
+        @mutex.synchronize { @breakers.clear }
+      end
+
+      def size
+        @mutex.synchronize { @breakers.size }
+      end
+    end
+
+    private
+
+    # Move an expired open circuit to half-open. Must be called with the
+    # mutex held; returns the previous state (nil when no transition) and
+    # the resulting state.
+    def advance_open_state
+      return [nil, @state] unless @state == :open && cooldown_elapsed?
+
+      from = @state
+      @state = :half_open
+      @opened_at = nil
+      [from, @state]
+    end
+
+    # Count a failure and open the circuit at the threshold. Must be called
+    # with the mutex held; returns the previous state when it opened.
+    def count_failure!
+      @failures += 1
+      return nil unless @state == :half_open || @failures >= @threshold
+
+      from = @state
+      @state = :open
+      @opened_at = Process.clock_gettime(CLOCK)
+      @failures = 0
+      from
+    end
+
+    def open_error
+      CircuitOpenError.new(
+        "circuit open for #{@key} " \
+        "(retry after #{remaining_cooldown.round(1)}s)"
+      )
+    end
+
+    def cooldown_elapsed?
+      @opened_at && (Process.clock_gettime(CLOCK) - @opened_at) >= @cooldown
+    end
+
+    def remaining_cooldown
+      left = @cooldown - (Process.clock_gettime(CLOCK) - @opened_at)
+      left.positive? ? left : 0.0
+    end
+
+    def post_transition(from, to)
+      Notifications.post('circuit', key: @key, from: from, state: to)
+    end
+  end
+end

data/lib/contractful/connection_pool.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+# Root namespace; this file adds the persistent-connection pool.
+module Contractful
+  # A small thread-safe pool for reusing started Net::HTTP connections
+  # (HTTP keep-alive / persistent connections) across requests.
+  #
+  # The pool is deliberately simple: there is no limit on the number of
+  # simultaneously checked-out connections (a concurrent request that cannot
+  # reuse an idle connection simply opens a new one, exactly like a
+  # non-persistent request would), but at most `max_idle` connections are
+  # retained per pool key once they are returned. Connections are checked out
+  # exclusively, so a connection is never shared by two threads at once.
+  #
+  # Requests opt in per request with `persistent: true`; the pool itself is
+  # process-wide and available as `Contractful.connection_pool` (for example
+  # call `Contractful.connection_pool.clear!` before forking, or after a
+  # network topology change).
+  class ConnectionPool
+    attr_reader :max_idle
+
+    def initialize(max_idle: 8)
+      unless max_idle.is_a?(Integer) && max_idle >= 0
+        raise ArgumentError.new(
+          ":max_idle must be a non-negative Integer, got #{max_idle.inspect}"
+        )
+      end
+
+      @max_idle = max_idle
+      @mutex = Mutex.new
+      @idle = Hash.new { |hash, key| hash[key] = [] }
+      @checked_out = Hash.new(0)
+    end
+
+    # Check a connection out for `key`. The `builder` block is invoked (outside
+    # the pool lock) to create a new connection when no idle one is available.
+    #
+    # @return the checked-out connection
+    #
+    def checkout(key, &builder)
+      raise ArgumentError.new('builder block is required') unless block_given?
+
+      conn = nil
+      @mutex.synchronize { conn = @idle[key].shift }
+
+      if conn && !revive(conn)
+        # Idle connection was closed or otherwise unusable; close it
+        # before discarding the reference so any half-opened socket is
+        # released immediately rather than waiting for GC to finalize
+        # the Net::HTTP object.
+        shutdown(conn)
+        conn = nil
+      end
+
+      conn ||= builder.call
+
+      @mutex.synchronize { @checked_out[key] += 1 }
+      conn
+    end
+
+    # Return a healthy connection to the pool for future reuse. If the pool is
+    # already retaining `max_idle` connections for this key, the connection is
+    # closed instead.
+    #
+    # Guard the counter decrement against drift: invalidate() decrements
+    # conditionally (only if positive), but checkin() previously decremented
+    # unconditionally, which would push the count below zero on a
+    # double-checkin or checkin-without-checkout. Both methods now share the
+    # same guard.
+    def checkin(key, conn)
+      keep = checkin_under_lock(key, conn)
+      shutdown(conn) unless keep
+      nil
+    end
+
+    # @api private
+    def checkin_under_lock(key, conn)
+      @mutex.synchronize do
+        return nil unless @checked_out[key].positive?
+
+        @checked_out[key] -= 1
+        if @idle[key].size < @max_idle
+          @idle[key].push(conn)
+          true
+        else
+          false
+        end
+      end
+    end
+
+    # Discard a connection whose state may no longer be trusted (for example
+    # after a network error). The connection is closed and never reused.
+    def invalidate(key, conn)
+      @mutex.synchronize do
+        @checked_out[key] -= 1 if @checked_out[key].positive?
+      end
+      shutdown(conn)
+      nil
+    end
+
+    # Close all idle connections and forget them. Connections that are
+    # currently checked out cannot be closed here; they will be discarded or
+    # reused normally when returned.
+    def clear!
+      conns = []
+      @mutex.synchronize do
+        @idle.each_value do |idle_conns| 
+          conns.concat(idle_conns)
+                                         idle_conns.clear
+end
+      end
+      conns.each { |conn| shutdown(conn) }
+      nil
+    end
+
+    # @return [Hash] counts of idle and checked-out connections
+    def stats
+      @mutex.synchronize do
+        {
+          idle: @idle.values.reduce(0) { |sum, idle_conns| sum + idle_conns.size },
+          checked_out: @checked_out.values.reduce(0, :+)
+        }
+      end
+    end
+
+    private
+
+    # Prepare an idle connection for reuse. Returns true if the connection is
+    # started (possibly after re-establishing it), false if it is unusable and
+    # a new connection should be built instead.
+    def revive(conn)
+      return true if conn.started?
+
+      conn.start
+      true
+    rescue StandardError
+      false
+    end
+
+    def shutdown(conn)
+      conn.finish if conn.started?
+    rescue StandardError
+      nil
+    end
+  end
+
+  # The process-wide connection pool used by requests created with
+  # `persistent: true`.
+  #
+  # Initialized eagerly at module load. The previous `||=` form was not
+  # atomic: two threads racing on first use both observed @connection_pool
+  # as nil and each built a fresh pool — one was discarded, and its idle
+  # connections' sockets leaked until GC finalized the Net::HTTP objects
+  # (Net::HTTP has no finalizer that closes them). The same pattern exists
+  # in Contractful::CircuitBreaker.registry and Contractful::Pacer.registry
+  # — those modules share the same fix shape.
+  @connection_pool = ConnectionPool.new
+
+  def self.connection_pool
+    @connection_pool
+  end
+
+  def self.connection_pool=(pool)
+    @connection_pool = pool
+  end
+end