coolhand 0.9.0 → 0.10.1

data/lib/coolhand/help_center_service.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require_relative "api_service"
+require_relative "pagination"
+require_relative "read_query"
+
+module Coolhand
+  # Rows stay plain Symbol-keyed Hashes so a field the server adds later is not dropped in transit.
+  HelpArticleSearchResult = Struct.new(:articles, :pagination, keyword_init: true)
+
+  # Read-only access to the Coolhand help center: the `public_content` endpoints (`GET /api/v2/public_content`,
+  # `/:slug` and `/categories`), the same pages served at `/help/:slug` and `/updates/:slug`. Create, update and
+  # archive are Coolhand-internal and not wrapped.
+  #
+  # Unlike every other service this one needs no API key. A configured `api_key` is sent as `X-API-Key` and lifts
+  # the anonymous limit of 30 requests per minute per IP (429 beyond); with none, no key header is sent. A key the
+  # server rejects raises a 401 {Coolhand::HttpError}; it is not retried anonymously. See docs/help-center.md.
+  class HelpCenterService < ApiService
+    include ReadQuery
+
+    ERROR_NOUN = "Help center"
+    BLANK_SLUG_MESSAGE = "get_help_article: slug must be a non-empty string"
+
+    def initialize
+      super("v2/public_content")
+    end
+
+    # Relevance search with `query`, a full listing without. `category` filters to one section slug; the
+    # endpoint has no such parameter, so it is sent as `q[help_category_slug_eq]` and `query` moves to
+    # `q[full_text]` (the plain `q` string and the `q[...]` form cannot be mixed). A blank `query` or
+    # `category` counts as omitted. Returns a {HelpArticleSearchResult}; raises {Coolhand::HttpError} on a
+    # non-2xx response.
+    def search_help_articles(query: nil, category: nil, page: nil, per_page: nil)
+      params = { **search_params(query, category), page: page, per_page: per_page }.compact
+
+      fetch_list(params, HelpArticleSearchResult, :articles, page: page, per: per_page)
+    end
+
+    # One published article with its markdown `body`, `noindex` and `toc`. Raises {Coolhand::Error} before any
+    # request for a blank slug, and {Coolhand::HttpError} (404) for an unknown or unpublished one.
+    def get_help_article(slug)
+      get_json(resource_url(slug, BLANK_SLUG_MESSAGE), ERROR_NOUN)
+    end
+
+    # Every section an article can be filed under, with its `url_prefix` and `allowed_tags`.
+    def list_help_categories
+      get_json(URI.parse("#{api_endpoint}/categories"), ERROR_NOUN)
+    end
+
+    protected
+
+    def api_key_required?
+      false
+    end
+
+    private
+
+    def search_params(query, category)
+      query = nil if omitted?(query)
+      return { q: query } if omitted?(category)
+
+      { "q[help_category_slug_eq]": category, "q[full_text]": query }
+    end
+
+    def omitted?(value)
+      value.nil? || value.to_s.strip.empty?
+    end
+  end
+end

data/lib/coolhand/logger_service.rb

@@ -4,6 +4,25 @@ require_relative "api_service"
 
 module Coolhand
   class LoggerService < ApiService
+    # CGI/Rack/Rails/app-server variables that show up in a Rails `request.headers` iteration
+    # (it walks the whole Rack env) but are not HTTP headers. CONTENT_TYPE and
+    # CONTENT_LENGTH are real headers and are deliberately absent. ORIGINAL_FULLPATH and
+    # REQUEST_PATH carry the query string, so they must never be forwarded. HTTP_PROXY is the
+    # process env var (it may embed credentials), never a client header (the "httpoxy" case).
+    RACK_ENV_CGI_KEYS = %w[
+      AUTH_TYPE GATEWAY_INTERFACE HTTPS HTTP_PROXY HTTP_VERSION ORIGINAL_FULLPATH ORIGINAL_SCRIPT_NAME
+      PATH_INFO PATH_TRANSLATED QUERY_STRING REMOTE_ADDR REMOTE_HOST REMOTE_IDENT REMOTE_PORT
+      REMOTE_USER REQUEST_METHOD REQUEST_PATH REQUEST_URI SCRIPT_NAME SERVER_ADDR SERVER_NAME
+      SERVER_PORT SERVER_PROTOCOL SERVER_SOFTWARE
+    ].freeze
+
+    # CGI-style env key: uppercase letters, digits and underscores (REQUEST_METHOD, DATABASE_URL).
+    CGI_ENV_KEY_PATTERN = /\A[A-Z0-9_]+\z/
+    # Lowercase dotted Rack/Rails/server key: rack.session, action_dispatch.request.parameters, puma.config.
+    RACK_DOTTED_KEY_PATTERN = /\A[a-z0-9_]+(?:\.[a-z0-9_]+)+\z/
+    RACK_ENV_HEADER_KEYS = %w[CONTENT_TYPE CONTENT_LENGTH].freeze
+    private_constant :CGI_ENV_KEY_PATTERN, :RACK_DOTTED_KEY_PATTERN, :RACK_ENV_HEADER_KEYS
+
     def initialize
       super("v2/llm_request_logs")
     end
@@ -24,10 +43,10 @@ module Coolhand
         id: options[:id] || SecureRandom.uuid,
         timestamp: options[:timestamp] || Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.%6NZ"),
         method: options[:method] || "POST",
-        url: options[:url] || build_webhook_url(source, event_type),
+        url: BaseInterceptor.sanitize_url(options[:url] || build_webhook_url(source, event_type)),
         headers: sanitize_headers(headers),
-        request_body: clean_webhook_body(webhook_body, source),
-        response_body: options[:response_body],
+        request_body: BodyRedactor.redact(clean_webhook_body(webhook_body, source)),
+        response_body: BodyRedactor.redact(options[:response_body]),
         response_headers: options[:response_headers] && sanitize_headers(options[:response_headers]),
         status_code: options[:status_code] || 200,
         source: "#{source}_webhook"
@@ -58,21 +77,51 @@ module Coolhand
       return {} unless headers.respond_to?(:each)
 
       # Handle Rails request headers or plain hash
+      entries = headers.enum_for(:each).to_a
+      rack_env = rack_env_shaped?(entries)
+
       clean_headers = {}
-      headers.each do |key, value|
-        next unless key && value
+      entries.each do |key, value|
+        next unless forwardable_header?(key, value, rack_env: rack_env)
 
         # Convert Rails HTTP_ prefix headers
         clean_key = key.to_s.gsub(/^HTTP_/, "").tr("_", "-").downcase
 
         # Redact sensitive headers (shared with BaseInterceptor so both logging
         # paths treat the same header names as sensitive)
-        clean_value = clean_key.match?(BaseInterceptor::SENSITIVE_HEADER_PATTERN) ? "[REDACTED]" : value.to_s
+        clean_value = BaseInterceptor.sensitive_header?(clean_key) ? "[REDACTED]" : value.to_s
         clean_headers[clean_key] = clean_value
       end
       clean_headers
     end
 
+    # Rails' request.headers#each yields the whole Rack env, not just headers. Drop the
+    # non-header entries (rack.session, action_dispatch.*, REQUEST_METHOD, QUERY_STRING, ...)
+    # so session/params data is never forwarded. Ordinary header names (including normalized
+    # plain-Hash ones like "Content-Type" or "x-webhook-id") never match the denylist.
+    # CGI/FastCGI handlers also put the process env (DATABASE_URL, REDIS_URL, ...) in the Rack
+    # env, so once the input looks like one, only HTTP_*, CONTENT_TYPE and CONTENT_LENGTH pass
+    # (a process env var that itself starts with HTTP_ still gets through; HTTP_PROXY is denied).
+    def forwardable_header?(key, value, rack_env: false)
+      return false unless key && value
+
+      name = key.to_s
+      return false if name.include?(".") || RACK_ENV_CGI_KEYS.include?(name)
+
+      !rack_env || name.start_with?("HTTP_") || RACK_ENV_HEADER_KEYS.include?(name)
+    end
+
+    # A Rack env is recognised by a positive signal the Rack spec guarantees (REQUEST_METHOD, or
+    # a lowercase dotted key like rack.input), or by every key being env-shaped. Requiring only
+    # the latter would let a single lowercase key (Warden's "warden", a shell's http_proxy)
+    # switch the allowlist off for the whole env. Mixed-case dotted names ("X.Trace") are
+    # valid header names, so they are not a signal.
+    def rack_env_shaped?(entries)
+      keys = entries.filter_map { |key, _value| key&.to_s }
+      keys.any? { |name| name == "REQUEST_METHOD" || name.match?(RACK_DOTTED_KEY_PATTERN) } ||
+        (keys.any? && keys.all? { |name| name.match?(CGI_ENV_KEY_PATTERN) })
+    end
+
     def clean_webhook_body(body, source)
       # Service-specific binary field filters
       binary_fields = case source.to_s.downcase

data/lib/coolhand/net_http_interceptor.rb

@@ -1,22 +1,29 @@
 # frozen_string_literal: true
 
+require "net/http"
+require "openssl"
 require "stringio"
+require "timeout"
+require "zlib"
+
+require_relative "body_stream_tap"
+require_relative "path_normalizer"
 
 module Coolhand
   module NetHttpInterceptor
     # Response streaming interceptor nested under NetHttpInterceptor
     module ResponseInterceptor
       def read_body(dest = nil, &block)
-        # Only buffer while a #request call below is actively capturing —
-        # otherwise every block-form read_body in the process (including
-        # ones on responses this gem never intercepted) accumulates into a
-        # thread-local that's never freed, which is an unbounded memory
-        # leak on any thread that streams large non-LLM responses.
-        return super unless block && Thread.current[:coolhand_capturing_stream]
+        # Only buffer this exact response while a #request call below is
+        # actively capturing it — otherwise every block-form read_body in
+        # the process (including ones on responses this gem never
+        # intercepted, such as a nested request made from inside the host's
+        # streaming block) accumulates into a thread-local that's never
+        # freed, and leaks one request's chunks into another's log.
+        return super unless block && Thread.current[:coolhand_capturing_stream].equal?(self)
 
         super do |chunk|
-          Thread.current[:coolhand_stream_buffer] ||= +""
-          Thread.current[:coolhand_stream_buffer] << chunk
+          Helpers.buffer_stream_chunk(chunk)
           yield(chunk)
         end
       end
@@ -119,19 +126,19 @@ module Coolhand
       previous_stream_buffer = Thread.current[:coolhand_stream_buffer]
       previous_capturing_stream = Thread.current[:coolhand_capturing_stream]
       Thread.current[:coolhand_stream_buffer] = nil
-      Thread.current[:coolhand_capturing_stream] = true
+      Thread.current[:coolhand_capturing_stream] = nil
+      host_block_errors = []
+      tracked_block = Helpers.track_streaming_block(block, host_block_errors)
 
       begin
-        response = super
-        body_content = Thread.current[:coolhand_stream_buffer] || response&.body
-        body_content = body_content.dup.force_encoding("UTF-8") if body_content.is_a?(String)
-        status_code = response.respond_to?(:code) ? response.code.to_i : nil
-        response_body = BaseInterceptor.parse_json(body_content)
+        response = super(req, body, &tracked_block)
+        status_code, response_body = Helpers.capture_response(response)
       rescue StandardError => e
-        status_code = Helpers.extract_status_from_exception(e)
-        response_body = { "error" => { "class" => e.class.name, "message" => e.message } }
+        status_code, response_body = Helpers.error_details(e, host_block_errors)
         raise
       ensure
+        # First, so the host's own stream is back on its request whatever fails below.
+        request_body = Helpers.finalize_request_body(captured_body, req)
         active.delete(self)
         Thread.current[:coolhand_stream_buffer] = previous_stream_buffer
         Thread.current[:coolhand_capturing_stream] = previous_capturing_stream
@@ -145,7 +152,7 @@ module Coolhand
           method: req.method,
           url: url,
           request_headers: BaseInterceptor.sanitize_headers(req),
-          request_body: captured_body,
+          request_body: request_body,
           response_headers: BaseInterceptor.sanitize_headers(response),
           response_body: response_body,
           status_code: status_code,
@@ -166,6 +173,46 @@ module Coolhand
     module Helpers
       module_function
 
+      HOST_BLOCK_ERROR_MESSAGE = "[redacted: raised in host streaming block]"
+      # Socket-level failures can surface inside the host's block (read_body runs there); they
+      # carry no app data, so they keep their message even when they pass through the host block.
+      TRANSPORT_ERRORS = [
+        IOError, SystemCallError, Timeout::Error, Net::ProtocolError, OpenSSL::SSL::SSLError,
+        Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Zlib::Error
+      ].freeze
+      private_constant :HOST_BLOCK_ERROR_MESSAGE, :TRANSPORT_ERRORS
+
+      # Wraps the host's streaming block so (a) read_body only buffers the response being
+      # captured here, and (b) exceptions raised by the host's own code can be told apart from
+      # transport errors (see TRANSPORT_ERRORS). Returns nil when there is no block.
+      def track_streaming_block(block, host_block_errors)
+        return unless block
+
+        proc do |res|
+          Thread.current[:coolhand_capturing_stream] = res
+          block.call(res)
+        rescue StandardError => e
+          host_block_errors << e unless transport_error?(e)
+          raise
+        end
+      end
+
+      # Matched by class, so a host block raising its own IOError/SystemCallError subclass keeps its
+      # message too; accepted trade-off, as telling those apart from socket failures isn't possible.
+      def transport_error?(error)
+        TRANSPORT_ERRORS.any? { |klass| error.is_a?(klass) }
+      end
+
+      # An error raised by the host's streaming block can carry app-level data in its message (and
+      # in a #status or message-embedded status code), so only its class is logged and no status is
+      # derived from it; transport errors keep their message and status. Returns [status, body].
+      def error_details(error, host_block_errors)
+        from_host = host_block_errors.any? { |host_error| host_error.equal?(error) }
+        status = from_host ? nil : extract_status_from_exception(error)
+        message = from_host ? HOST_BLOCK_ERROR_MESSAGE : error.message
+        [status, { "error" => { "class" => error.class.name, "message" => message } }]
+      end
+
       def should_capture?
         return true if Coolhand.configuration.debug_mode
 
@@ -184,24 +231,132 @@ module Coolhand
 
         content = body || req.body
         if content.nil? && req.respond_to?(:body_stream) && req.body_stream
-          content = req.body_stream.read
-          req.body_stream = StringIO.new(content)
+          result = capture_body_stream(req)
+          return result unless result.is_a?(String)
+
+          content = result
         end
         return nil if content.nil?
 
         cap_and_parse(content, req)
       end
 
+      # A body_stream request: the log body (a marker, or the BodyStreamTap resolved after the send),
+      # or the stream's content as a String when it was read here and still needs cap_and_parse.
+      def capture_body_stream(req)
+        # A declared Content-Length over the cap means we never need the bytes: skip
+        # before .read so a huge stream is not buffered just to be discarded.
+        too_large = oversized_declared_length(req)
+        return too_large if too_large
+
+        # No usable Content-Length (chunked): the size is unknown, so don't read ahead. Tap the
+        # stream instead and build the log body once Net::HTTP has sent it (finalize_request_body).
+        unless declared_content_length(req)
+          return skipped_capture_marker(req, "unsupported_body_stream") unless BodyStreamTap.tappable?(req.body_stream)
+
+          return BodyStreamTap.install(req)
+        end
+
+        content = req.body_stream.read
+        req.body_stream = StringIO.new(content)
+        content
+      end
+
+      # The effective cap: nil disables it; a non-Integer or negative value (e.g. an unparsed
+      # ENV string) falls back to the default rather than raising into the host's request or stream.
+      def capture_limit
+        limit = Coolhand.configuration.max_captured_body_bytes
+        return limit if limit.nil? || (limit.is_a?(Integer) && !limit.negative?)
+
+        Coolhand::Configuration::DEFAULT_MAX_CAPTURED_BODY_BYTES
+      end
+
+      def over_capture_limit?(size)
+        limit = capture_limit
+        !limit.nil? && size > limit
+      end
+
+      def body_too_large_marker(carrier, size)
+        skipped_capture_marker(carrier, "body_too_large", size_bytes: size, max_bytes: capture_limit)
+      end
+
+      # Returns a body_too_large marker when the request declares a Content-Length above the
+      # cap, else nil. Chunked/absent/malformed lengths return nil (the caller then taps the
+      # stream, which enforces the cap on the bytes actually sent).
+      def oversized_declared_length(req)
+        length = declared_content_length(req)
+        body_too_large_marker(req, length) if length && over_capture_limit?(length)
+      end
+
+      # Net::HTTP sends a chunked body without regard to any Content-Length header, so it is not a
+      # declaration of what goes on the wire then.
+      def declared_content_length(req)
+        req.content_length if req.respond_to?(:content_length) && !req.chunked?
+      rescue StandardError
+        nil
+      end
+
+      # The request body for the log. A tapped body_stream (see BodyStreamTap.install) only has its
+      # content once the request has been attempted, so it is resolved here and the host's own
+      # stream is put back on the request. Runs in #request's ensure: it must not raise.
+      def finalize_request_body(captured, req)
+        return captured unless captured.is_a?(BodyStreamTap)
+
+        captured.restore_stream(req)
+        captured.body_for_log(req)
+      rescue StandardError => e
+        Coolhand.log "❌ Error capturing request body: #{e.message}"
+        nil
+      end
+
+      # Appends a streamed response chunk to the thread-local capture buffer, up to the cap.
+      # Past the cap the buffer is dropped and replaced by a running byte count (an Integer),
+      # so memory stays bounded; the host always still receives every chunk because the caller
+      # yields it regardless. Only #request (via response_body_for_log) reads this slot.
+      def buffer_stream_chunk(chunk)
+        buffer = Thread.current[:coolhand_stream_buffer]
+        total = (buffer.is_a?(Integer) ? buffer : buffer.to_s.bytesize) + chunk.bytesize
+
+        Thread.current[:coolhand_stream_buffer] =
+          if buffer.is_a?(Integer) || over_capture_limit?(total)
+            total
+          else
+            (buffer || +"") << chunk
+          end
+      rescue StandardError => e
+        # Runs inside the host's read_body loop: a capture failure must never reach the host.
+        Coolhand.log "❌ Error buffering streamed response chunk: #{e.message}"
+      end
+
+      # [status_code, response_body] for the log. Called after the real request has succeeded, so a
+      # failure here is logged and swallowed rather than re-raised into the host app.
+      def capture_response(response)
+        captured = Thread.current[:coolhand_stream_buffer] || response&.body
+        # captured is a String, or an Integer byte count when a streamed body overflowed the cap.
+        [response.respond_to?(:code) ? response.code.to_i : nil, response_body_for_log(captured, response)]
+      rescue StandardError => e
+        Coolhand.log "❌ Error capturing response: #{e.message}"
+        [nil, nil]
+      end
+
+      # Builds the logged response body from either an Integer (streamed body that overflowed
+      # the cap, see buffer_stream_chunk) or the body String/nil. An oversized String is
+      # replaced by a marker before the dup/JSON.parse so we don't copy or parse it.
+      def response_body_for_log(body, response)
+        size = body.is_a?(Integer) ? body : body.to_s.bytesize
+        return body_too_large_marker(response, size) if body.is_a?(Integer) || over_capture_limit?(size)
+
+        body = body.dup.force_encoding("UTF-8") if body.is_a?(String)
+        BaseInterceptor.parse_json(body)
+      end
+
       def binary_upload?(req)
         content_type = req.respond_to?(:content_type) ? req.content_type : nil
         content_type && !content_type.match?(/json/i)
       end
 
       def cap_and_parse(content, req)
-        max_bytes = Coolhand.configuration.max_captured_body_bytes
-        if max_bytes && content.bytesize > max_bytes
-          return skipped_capture_marker(req, "body_too_large", size_bytes: content.bytesize, max_bytes: max_bytes)
-        end
+        return body_too_large_marker(req, content.bytesize) if over_capture_limit?(content.bytesize)
 
         BaseInterceptor.parse_json(content)
       end
@@ -227,10 +382,11 @@ module Coolhand
         uri = safe_parse(url)
         return false unless uri&.host
 
-        return false if excluded_by_pattern?(uri)
+        raw_path = uri.path.to_s
+        path = normalize_path(raw_path)
+        return false if excluded_by_pattern?(uri, raw_path, path)
 
         host = uri.host.downcase.chomp(".")
-        path = uri.path.to_s
         addresses = Coolhand.configuration.intercept_addresses
         authorities = [host, "#{host}:#{uri.port}"]
         return true if addresses.any? { |a| authorities.any? { |authority| address_matches?(authority, path, a) } }
@@ -251,12 +407,13 @@ module Coolhand
         nil
       end
 
-      def excluded_by_pattern?(uri)
+      # The deny-list fails closed: a pattern that matches either the raw or the normalized path
+      # excludes the request, so normalization can only add exclusions, never lift one.
+      def excluded_by_pattern?(uri, raw_path, path)
         patterns = Coolhand.configuration.exclude_api_patterns
         return false if patterns.nil? || patterns.empty?
 
-        path = uri.path.to_s
-        matched = patterns.find { |pattern| path.include?(pattern) }
+        matched = patterns.find { |pattern| path.include?(pattern) || raw_path.include?(pattern) }
         if matched && Coolhand.configuration.debug_mode
           Coolhand.log "🚫 Skipping capture for #{BaseInterceptor.sanitize_url(uri.to_s)} " \
                        "(matched exclude_api_pattern: \"#{matched}\")"
@@ -264,6 +421,11 @@ module Coolhand
         !!matched
       end
 
+      # See PathNormalizer: used for matching only, never for the URL that is sent or logged.
+      def normalize_path(path)
+        PathNormalizer.normalize(path)
+      end
+
       # intercept_path_patterns only ever applies to googleapis.com hosts, and only when the
       # user still wants Google API traffic intercepted at all — otherwise overriding
       # intercept_addresses to exclude Google hosts wouldn't actually stop Google API capture.