logister-ruby 0.5.1 → 0.6.0

data/lib/logister/context_helpers.rb

@@ -21,12 +21,12 @@ module Logister
 
     def deployment_context
       config = Logister.configuration if Logister.respond_to?(:configuration)
-      environment = config&.respond_to?(:environment) ? config.environment.to_s.presence : nil
-      service = config&.respond_to?(:service) ? config.service.to_s.presence : nil
-      release = config&.respond_to?(:release) ? config.release.to_s.presence : nil
-      repository = config&.respond_to?(:repository) ? config.repository.to_s.presence : nil
-      commit_sha = config&.respond_to?(:commit_sha) ? config.commit_sha.to_s.presence : nil
-      branch = config&.respond_to?(:branch) ? config.branch.to_s.presence : nil
+      environment = config.respond_to?(:environment) ? config.environment.to_s.presence : nil
+      service = config.respond_to?(:service) ? config.service.to_s.presence : nil
+      release = config.respond_to?(:release) ? config.release.to_s.presence : nil
+      repository = config.respond_to?(:repository) ? config.repository.to_s.presence : nil
+      commit_sha = config.respond_to?(:commit_sha) ? config.commit_sha.to_s.presence : nil
+      branch = config.respond_to?(:branch) ? config.branch.to_s.presence : nil
 
       {
         deployment: {
@@ -105,7 +105,7 @@ module Logister
 
         "#{segments[0]}.#{segments[1]}.#{segments[2]}.0"
       else
-        "#{parsed.mask(64).to_s}/64"
+        "#{parsed.mask(64)}/64"
       end
     rescue StandardError
       ip.to_s

data/lib/logister/context_store.rb

@@ -43,6 +43,16 @@ module Logister
       request_scope[:trace_context] = value
     end
 
+    # User identity lives in the request scope so resetting the scope at the end
+    # of a request or job also clears it.
+    def user
+      request_scope[:user]
+    end
+
+    def user=(value)
+      request_scope[:user] = value
+    end
+
     def breadcrumbs
       request_scope[:breadcrumbs].dup
     end

data/lib/logister/event_sanitizer.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+require 'date'
+require_relative 'redaction'
+
+module Logister
+  # Converts an event into JSON-safe primitives before it is serialized:
+  # applies the shared redaction spec, scrubs invalid UTF-8, bounds string
+  # length, collection size, nesting depth, and total key count. The limits stay
+  # below the Logister ingest limits so one oversized event is trimmed locally
+  # instead of being rejected by the server.
+  class EventSanitizer
+    TRUNCATED = '[TRUNCATED]'
+    TRUNCATION_MARKER_KEY = 'logister_truncated'
+
+    # SDK-generated envelope fields: never redacted, so correlation survives.
+    ENVELOPE_KEYS = Set.new(%w[
+      uuid event_id event_type trace_id span_id parent_span_id request_id session_id
+      requestId traceId spanId parentSpanId sessionId
+    ]).freeze
+
+    # Large, low-priority context collections are visited last so the key
+    # budget is spent on the exception, request, and user context first.
+    DEFERRED_CONTEXT_KEYS = Set.new(%w[breadcrumbs dependencyCalls]).freeze
+
+    Limits = Struct.new(:max_string_bytes, :max_message_bytes, :max_items, :max_depth, :max_keys, keyword_init: true)
+
+    DEFAULT_LIMITS = Limits.new(
+      max_string_bytes: 16 * 1024,
+      max_message_bytes: 32 * 1024,
+      max_items: 100,
+      max_depth: 10,
+      max_keys: 400
+    ).freeze
+
+    COMPACT_LIMITS = Limits.new(
+      max_string_bytes: 1024,
+      max_message_bytes: 4096,
+      max_items: 20,
+      max_depth: 6,
+      max_keys: 200
+    ).freeze
+
+    def self.call(payload, extra_keys: Redaction.configured_keys, limits: DEFAULT_LIMITS)
+      new(extra_keys: extra_keys, limits: limits).sanitize_payload(payload)
+    end
+
+    def initialize(extra_keys:, limits:)
+      @extra_keys = extra_keys
+      @limits = limits
+      @keys_used = 0
+      @truncated = false
+    end
+
+    def sanitize_payload(payload)
+      result = {}
+      payload.to_h.each do |key, value|
+        name = key.to_s
+        result[key] = case name
+                      when 'context'
+                        sanitize_context(value)
+                      when 'message'
+                        value.nil? ? nil : string(value.to_s, @limits.max_message_bytes)
+                      else
+                        envelope_value(value)
+                      end
+        @keys_used += 1
+      end
+      result
+    end
+
+    private
+
+    def sanitize_context(context)
+      return sanitize(context, 1) unless context.is_a?(Hash)
+
+      deferred, primary = context.to_a.partition { |key, _| DEFERRED_CONTEXT_KEYS.include?(key.to_s) }
+      result = {}
+      (primary + deferred).each do |key, value|
+        unless take_key
+          @truncated = true
+          break
+        end
+
+        name = utf8(key.to_s)
+        result[name] = if ENVELOPE_KEYS.include?(name)
+                         envelope_value(value)
+                       elsif Redaction.sensitive_key?(name, @extra_keys)
+                         Redaction::REDACTED
+                       else
+                         sanitize(value, 2)
+                       end
+      end
+      result[TRUNCATION_MARKER_KEY] = true if @truncated
+      result
+    end
+
+    def envelope_value(value)
+      case value
+      when String then truncate(utf8(value), @limits.max_string_bytes)
+      when Symbol then value.to_s
+      when Hash, Array then sanitize(value, 1)
+      else primitive(value)
+      end
+    end
+
+    def sanitize(value, depth)
+      case value
+      when nil, true, false, Integer then value
+      when Float then value.finite? ? value : value.to_s
+      when String then string(value)
+      when Symbol then string(value.to_s)
+      when Hash then sanitize_hash(value, depth)
+      when Array, Set then sanitize_array(value.to_a, depth)
+      else sanitize_object(value, depth)
+      end
+    end
+
+    def sanitize_hash(hash, depth)
+      return mark_truncated(TRUNCATED) if depth > @limits.max_depth
+
+      result = {}
+      hash.each_with_index do |(key, value), index|
+        if index >= @limits.max_items || !take_key
+          @truncated = true
+          break
+        end
+
+        key = utf8(key.to_s)
+        result[key] = Redaction.sensitive_key?(key, @extra_keys) ? Redaction::REDACTED : sanitize(value, depth + 1)
+      end
+      result
+    end
+
+    def sanitize_array(array, depth)
+      return mark_truncated(TRUNCATED) if depth > @limits.max_depth
+
+      @truncated = true if array.length > @limits.max_items
+      array.first(@limits.max_items).map { |item| sanitize(item, depth + 1) }
+    end
+
+    def sanitize_object(value, depth)
+      case value
+      when Time, DateTime then value.iso8601
+      when Date then value.iso8601
+      when Exception then { 'class' => value.class.name, 'message' => string(value.message.to_s) }
+      when Struct then sanitize_hash(value.to_h, depth)
+      else
+        # Never enumerate lazy relations or IO-like objects; they can run
+        # queries or block. Everything else uses its JSON representation.
+        return string("#<#{value.class.name}>") if lazy_collection?(value)
+        return string(value.to_s) unless value.respond_to?(:as_json)
+
+        converted = value.as_json
+        return string(value.to_s) if converted.equal?(value)
+
+        sanitize(converted, depth)
+      end
+    rescue StandardError
+      string("#<#{value.class.name}>")
+    end
+
+    def lazy_collection?(value)
+      (defined?(ActiveRecord::Relation) && value.is_a?(ActiveRecord::Relation)) ||
+        value.is_a?(Enumerator) || value.is_a?(IO) || value.is_a?(Proc)
+    end
+
+    def primitive(value)
+      case value
+      when nil, true, false, Integer then value
+      when Float then value.finite? ? value : value.to_s
+      else string(value.to_s)
+      end
+    end
+
+    def string(value, max_bytes = @limits.max_string_bytes)
+      # Bound regex work on huge strings, keeping enough slack that a credential
+      # straddling the truncation point is still recognized and redacted.
+      value = value.byteslice(0, max_bytes + 4096) if value.bytesize > max_bytes + 4096
+      truncate(Redaction.scrub_string(utf8(value), @extra_keys), max_bytes)
+    end
+
+    def utf8(value)
+      value = value.dup.force_encoding(Encoding::UTF_8) if value.encoding == Encoding::BINARY
+      value = value.encode(Encoding::UTF_8, invalid: :replace, undef: :replace) unless value.encoding == Encoding::UTF_8
+      value.valid_encoding? ? value : value.scrub
+    rescue StandardError
+      value.to_s.b.force_encoding(Encoding::UTF_8).scrub
+    end
+
+    def truncate(value, max_bytes)
+      return value if value.bytesize <= max_bytes
+
+      @truncated = true
+      "#{value.byteslice(0, max_bytes).scrub('')}…#{TRUNCATED}"
+    end
+
+    def take_key
+      return false if @keys_used >= @limits.max_keys
+
+      @keys_used += 1
+      true
+    end
+
+    def mark_truncated(value)
+      @truncated = true
+      value
+    end
+  end
+end

data/lib/logister/middleware.rb

@@ -3,49 +3,106 @@ require_relative "context_store"
 
 module Logister
   class Middleware
-    FILTERED_HEADER_PLACEHOLDER = "[FILTERED]".freeze
-    SENSITIVE_HEADERS = %w[authorization cookie set-cookie x-api-key x-csrf-token].freeze
+    FILTERED_HEADER_PLACEHOLDER = Logister::Redaction::REDACTED
+
+    # Only these request headers are captured. Add more with
+    # `config.capture_headers`; the shared sensitive-key rule still applies to
+    # every captured header name, and values are scrubbed for credentials.
+    CAPTURED_HEADERS = %w[
+      accept accept-encoding accept-language cache-control content-length content-type
+      dnt host origin referer user-agent via
+      traceparent tracestate x-request-id x-amzn-trace-id
+      x-b3-traceid x-b3-spanid x-b3-parentspanid x-b3-sampled
+      x-datadog-trace-id x-datadog-parent-id x-datadog-sampling-priority
+      x-forwarded-for x-forwarded-host x-forwarded-proto x-forwarded-port x-real-ip
+      sec-fetch-dest sec-fetch-mode sec-fetch-site sec-fetch-user
+      sec-ch-ua sec-ch-ua-mobile sec-ch-ua-platform upgrade-insecure-requests
+    ].freeze
+    IP_HEADERS = %w[x-forwarded-for x-real-ip].freeze
 
     def initialize(app)
       @app = app
     end
 
     def call(env)
+      trace_context = begin_request(env)
+      started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      status, headers, body = @app.call(env)
+      [status, with_request_id_header(headers, trace_context), body]
+    rescue StandardError => e
+      report_exception(e, env, started_at)
+      raise
+    ensure
+      end_request(env, trace_context)
+    end
+
+    private
+
+    def begin_request(env)
       Logister::ContextStore.reset_request_scope!
       trace_context = Logister::TraceContext.from_headers(
         traceparent: env["HTTP_TRACEPARENT"], request_id: env["action_dispatch.request_id"] || env["HTTP_X_REQUEST_ID"]
       )
       Logister::ContextStore.trace_context = trace_context
       env["logister.trace_context"] = trace_context
-      started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      status, headers, body = @app.call(env)
-      headers = headers.merge("x-request-id" => trace_context.request_id)
-      [status, headers, body]
+      trace_context
     rescue StandardError => e
-      request = ActionDispatch::Request.new(env)
-      request_context = build_request_context(request, env, error: e, started_at: started_at)
+      Logister.log_internal_error("middleware setup", e)
+      nil
+    end
+
+    def with_request_id_header(headers, trace_context)
+      return headers unless trace_context
+
+      headers.merge("x-request-id" => trace_context.request_id)
+    rescue StandardError => e
+      Logister.log_internal_error("middleware response header", e)
+      headers
+    end
+
+    # Context building and reporting run in their own rescue so a telemetry
+    # failure can never replace the application's exception.
+    def report_exception(error, env, started_at)
+      request = build_request(env)
+      request_context = build_request_context(request, env, error: error, started_at: started_at)
 
       Logister.report_error(
-        e,
+        error,
         context: request_context
       )
-      raise
-    ensure
+    rescue StandardError => e
+      Logister.log_internal_error("middleware error report", e)
+      nil
+    end
+
+    def end_request(env, trace_context)
       Logister::ContextStore.clear_request_summary(trace_context&.request_id)
       request_id = env["action_dispatch.request_id"]
       Logister::ContextStore.clear_request_summary(request_id)
       Logister::ContextStore.reset_request_scope!
+    rescue StandardError => e
+      Logister.log_internal_error("middleware cleanup", e)
+      nil
     end
 
-    private
+    # Rails apps get ActionDispatch::Request, including filtered parameters.
+    # Plain Rack apps get Rack::Request; parameters are then omitted because no
+    # host filter is configured, and the redacted URL still records the query.
+    def build_request(env)
+      return ActionDispatch::Request.new(env) if defined?(ActionDispatch::Request)
+
+      require "rack/request" unless defined?(Rack::Request)
+      Rack::Request.new(env)
+    end
 
     def build_request_context(request, env, error:, started_at:)
       request_id = env["action_dispatch.request_id"].to_s.presence
       path = request.path.to_s
       method = request.request_method.to_s
-      params = request.filtered_parameters.to_h
+      params = request_params(request)
       headers = extract_headers(env)
-      referer = request.referer.to_s.presence || headers["Referer"]
+      referer = Logister::Redaction.redact_url(request.referer.to_s.presence || headers["Referer"])
+      url = Logister::Redaction.redact_url(request_url(request))
       http_version = env["HTTP_VERSION"].to_s.presence || env["SERVER_PROTOCOL"].to_s.presence
       rails_action = rails_action_name(params)
       response_status = response_status_for(error)
@@ -71,11 +128,11 @@ module Logister
         railsAction: rails_action,
         referer: referer,
         requestId: request_id,
-        url: request.original_url.to_s.presence,
+        url: url,
         response: {
           status: request_summary[:status] || response_status,
           contentType: request.content_type.to_s.presence,
-          format: request_summary[:format] || request.format&.to_s.presence,
+          format: request_summary[:format] || request_format(request),
           durationMs: duration_ms
         }.compact,
         route: {
@@ -100,7 +157,7 @@ module Logister
           railsAction: rails_action,
           referer: referer,
           requestId: request_id,
-          url: request.original_url.to_s.presence
+          url: url
         }.compact
       }.compact
 
@@ -114,6 +171,29 @@ module Logister
       )
     end
 
+    def request_params(request)
+      return {} unless request.respond_to?(:filtered_parameters)
+
+      request.filtered_parameters.to_h
+    rescue StandardError
+      {}
+    end
+
+    def request_url(request)
+      url = request.respond_to?(:original_url) ? request.original_url : request.url
+      url.to_s.presence
+    rescue StandardError
+      nil
+    end
+
+    def request_format(request)
+      return nil unless request.respond_to?(:format)
+
+      request.format&.to_s.presence
+    rescue StandardError
+      nil
+    end
+
     def rails_action_name(params)
       return nil unless params.is_a?(Hash)
 
@@ -125,13 +205,14 @@ module Logister
     end
 
     def extract_headers(env)
+      allowed = captured_header_names
       headers = {}
 
       env.each do |key, value|
         next unless value.is_a?(String)
 
         header_name = rack_env_to_header_name(key)
-        next unless header_name
+        next unless header_name && allowed.include?(header_name.downcase)
 
         headers[header_name] = filter_header_value(header_name, value)
       end
@@ -139,6 +220,13 @@ module Logister
       headers.sort.to_h
     end
 
+    def captured_header_names
+      extra = Array(Logister.configuration.capture_headers).map { |name| name.to_s.downcase.tr("_", "-") }
+      CAPTURED_HEADERS | extra
+    rescue StandardError
+      CAPTURED_HEADERS
+    end
+
     def rack_env_to_header_name(key)
       if key.start_with?("HTTP_")
         key.delete_prefix("HTTP_").split("_").map(&:capitalize).join("-")
@@ -146,17 +234,22 @@ module Logister
         "Content-Type"
       elsif key == "CONTENT_LENGTH"
         "Content-Length"
-      else
-        nil
       end
     end
 
     def filter_header_value(name, value)
-      return FILTERED_HEADER_PLACEHOLDER if SENSITIVE_HEADERS.include?(name.to_s.downcase)
+      normalized = name.to_s.downcase
+      return FILTERED_HEADER_PLACEHOLDER if Logister::Redaction.sensitive_key?(normalized)
+      return anonymize_ip_list(value) if IP_HEADERS.include?(normalized)
+      return Logister::Redaction.redact_url(value) if normalized == "referer"
 
       value
     end
 
+    def anonymize_ip_list(value)
+      value.split(",").map { |ip| Logister::ContextHelpers.anonymize_ip(ip.strip) }.compact.join(", ")
+    end
+
     def current_user(env)
       controller = env["action_controller.instance"]
       return nil unless controller

data/lib/logister/railtie.rb

@@ -38,6 +38,14 @@ module Logister
         copy_setting(app, config, :capture_request_spans)
         copy_setting(app, config, :capture_sql_breadcrumbs)
         copy_setting(app, config, :sql_breadcrumb_min_duration_ms)
+        copy_setting(app, config, :redact_keys)
+        copy_setting(app, config, :capture_headers)
+        copy_setting(app, config, :max_event_bytes)
+        copy_setting(app, config, :sync_max_retries)
+        copy_setting(app, config, :delivery_observer)
+        copy_setting(app, config, :repository)
+        copy_setting(app, config, :commit_sha)
+        copy_setting(app, config, :branch)
       end
     end
 

data/lib/logister/redaction.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module Logister
+  # Shared default redaction spec used by every Logister SDK. Keep the key lists
+  # and value patterns identical across SDKs; add project-specific keys through
+  # `config.redact_keys` instead of changing these defaults.
+  module Redaction
+    REDACTED = '[REDACTED]'
+
+    SENSITIVE_KEYS = Set.new(%w[
+      password passwd pwd secret token apikey authorization proxyauthorization
+      cookie setcookie sessiontoken sessionkey credential credentials privatekey
+      clientsecret accesstoken refreshtoken idtoken authtoken xapikey xauthtoken
+      xaccesstoken xcsrftoken xxsrftoken signature ssn creditcard cardnumber cvv
+    ]).freeze
+
+    SENSITIVE_KEY_FRAGMENTS = %w[
+      token secret password passwd apikey authorization credential cookie
+    ].freeze
+
+    AUTH_SCHEME_PATTERN = %r{\b(Bearer|Basic)\s+[A-Za-z0-9\-._~+/]+=*}i
+    JWT_PATTERN = /\beyJ[A-Za-z0-9_-]*\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/
+    LOGISTER_KEY_PATTERN = /\blgst_[A-Za-z0-9_-]+/
+    # A `name=value` pair at the start of a string or after a query, form, or
+    # whitespace separator. Only values whose name is sensitive are replaced.
+    USERINFO_PASSWORD_PATTERN = %r{\A([a-z][a-z0-9+.-]*://[^/?#@:]*):[^/?#@]*@}i
+    QUERY_PAIR_PATTERN = /(\A|[?&;\s])([^=&;#?\s]+)=([^&;#\s]*)/
+    # Single-quoted strings (with '' escapes and optional E/N/X/B prefixes),
+    # dollar-quoted bodies, hex blobs, and numbers that are not part of an
+    # identifier or a `$1` bind placeholder. Double quotes are left alone because
+    # PostgreSQL and SQLite use them for identifiers.
+    SQL_STRING_PATTERN = /(?<![A-Za-z0-9_])[EeNnXxBb]?'(?:[^'\\]|''|\\.)*'/
+    SQL_DOLLAR_PATTERN = /\$([A-Za-z_][A-Za-z0-9_]*|)\$.*?\$\1\$/m
+    SQL_NUMBER_PATTERN = /(?<![A-Za-z0-9_$.])-?(?:0x[0-9A-Fa-f]+|\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)(?![A-Za-z0-9_])/
+
+    module_function
+
+    def normalize_key(key)
+      key.to_s.downcase.gsub(/[^a-z0-9]/, '')
+    end
+
+    def sensitive_key?(key, extra_keys = configured_keys)
+      normalized = normalize_key(key)
+      return false if normalized.empty?
+
+      SENSITIVE_KEYS.include?(normalized) ||
+        extra_keys.include?(normalized) ||
+        SENSITIVE_KEY_FRAGMENTS.any? { |fragment| normalized.include?(fragment) }
+    end
+
+    # Scrubs credentials embedded in free text: auth schemes, JWTs, Logister
+    # keys, and sensitive `name=value` pairs such as query parameters.
+    def scrub_string(value, extra_keys = configured_keys)
+      return value unless value.is_a?(String) && !value.empty?
+
+      scrubbed = value.gsub(AUTH_SCHEME_PATTERN) { "#{Regexp.last_match(1)} #{REDACTED}" }
+      scrubbed = scrubbed.gsub(JWT_PATTERN, REDACTED)
+      scrubbed = scrubbed.gsub(LOGISTER_KEY_PATTERN, REDACTED)
+      return scrubbed unless scrubbed.include?('=')
+
+      scrubbed.gsub(QUERY_PAIR_PATTERN) do
+        prefix, name, pair_value = Regexp.last_match.captures
+        if !pair_value.empty? && sensitive_key?(decode_component(name), extra_keys)
+          "#{prefix}#{name}=#{REDACTED}"
+        else
+          Regexp.last_match(0)
+        end
+      end
+    end
+
+    # Keeps scheme, host, and path; replaces credentials in userinfo and the
+    # values of sensitive query parameters; drops the fragment.
+    def redact_url(value, extra_keys = configured_keys)
+      return value unless value.is_a?(String) && !value.empty?
+
+      uri = URI.parse(value)
+      uri.fragment = nil
+      uri.query = redact_query(uri.query, extra_keys) if uri.query
+      scrub_string(redact_userinfo(uri.to_s), extra_keys)
+    rescue URI::Error, ArgumentError
+      scrub_string(redact_userinfo(value.split('#', 2).first), extra_keys)
+    end
+
+    def redact_userinfo(url)
+      url.sub(USERINFO_PASSWORD_PATTERN) { "#{Regexp.last_match(1)}:#{REDACTED}@" }
+    end
+
+    def redact_query(query, extra_keys = configured_keys)
+      return query if query.nil? || query.empty?
+
+      query.split('&', -1).map do |pair|
+        name, separator, pair_value = pair.partition('=')
+        if !separator.empty? && !pair_value.empty? && sensitive_key?(decode_component(name), extra_keys)
+          "#{name}=#{REDACTED}"
+        else
+          pair
+        end
+      end.join('&')
+    end
+
+    def strip_sql_literals(sql)
+      sql.to_s
+         .gsub(SQL_DOLLAR_PATTERN, '?')
+         .gsub(SQL_STRING_PATTERN, '?')
+         .gsub(SQL_NUMBER_PATTERN, '?')
+    end
+
+    def configured_keys
+      keys = Logister.configuration.redact_keys if Logister.respond_to?(:configuration)
+      normalize_keys(keys)
+    rescue StandardError
+      Set.new
+    end
+
+    def normalize_keys(keys)
+      Set.new(Array(keys).map { |key| normalize_key(key) }.reject(&:empty?))
+    end
+
+    def decode_component(name)
+      URI.decode_www_form_component(name)
+    rescue ArgumentError
+      name
+    end
+  end
+end