pinqloq 1.1.2

data/lib/pinqloq/log_entry.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "log_level"
+require_relative "log_source_type"
+
+module Pinqloq
+  class LogEntry
+    attr_accessor :log_level, :event, :date, :app_version_name, :device_identifier,
+                  :log_source_type, :collection_name, :correlation_id, :path, :metadata, :detail
+
+    def initialize(
+      event:,
+      log_level: LogLevel::INFORMATION,
+      date: nil,
+      app_version_name: nil,
+      device_identifier: nil,
+      log_source_type: LogSourceType::BACKEND,
+      collection_name: nil,
+      correlation_id: nil,
+      path: nil,
+      metadata: nil,
+      detail: nil
+    )
+      @event = event
+      @log_level = log_level
+      @date = date
+      @app_version_name = app_version_name
+      @device_identifier = device_identifier
+      @log_source_type = log_source_type
+      @collection_name = collection_name
+      @correlation_id = correlation_id
+      @path = path
+      @metadata = metadata
+      @detail = detail
+    end
+  end
+end

data/lib/pinqloq/log_failure_reason.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Pinqloq
+  module LogFailureReason
+    UNAUTHORIZED = :unauthorized
+    FORBIDDEN = :forbidden
+    QUEUE_FULL = :queue_full
+    HTTP_ERROR = :http_error
+    TIMEOUT = :timeout
+    NETWORK = :network
+  end
+
+  LogError = Struct.new(:reason, :status_code, :message, :cause, keyword_init: true)
+end

data/lib/pinqloq/log_level.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Pinqloq
+  module LogLevel
+    DEBUG = 1
+    INFORMATION = 2
+    WARNING = 3
+    ERROR = 4
+    FATAL = 5
+  end
+end

data/lib/pinqloq/log_source_type.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Pinqloq
+  module LogSourceType
+    DEVICE = 1
+    BACKEND = 2
+
+    NAMES = {
+      DEVICE => "Device",
+      BACKEND => "Backend"
+    }.freeze
+  end
+end

data/lib/pinqloq/logger.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Pinqloq
+  class Logger
+    DEVICE_IDENTIFIER_REQUIRED_MESSAGE =
+      "Pinqloq: device_identifier is required. Set it on the entry, or configure the global " \
+      "Options#device_identifier fallback."
+
+    def initialize(buffer, dispatcher, options)
+      @buffer = buffer
+      @dispatcher = dispatcher
+      @options = options
+    end
+
+    def enqueue(entry, on_sent: nil, on_failed: nil)
+      ensure_device_identifier!(entry)
+      written = @buffer.enqueue(entry, on_sent: on_sent, on_failed: on_failed)
+      @dispatcher.notify_enqueued
+      written
+    end
+
+    def enqueue_many(entries, on_sent: nil, on_failed: nil)
+      entries.each { |entry| ensure_device_identifier!(entry) }
+      written = @buffer.enqueue_many(entries, on_sent: on_sent, on_failed: on_failed)
+      @dispatcher.notify_enqueued
+      written
+    end
+
+    private
+
+    def ensure_device_identifier!(entry)
+      entry_value = entry.device_identifier&.strip
+      global_value = @options.device_identifier&.strip
+
+      return if (entry_value && !entry_value.empty?) || (global_value && !global_value.empty?)
+
+      raise DEVICE_IDENTIFIER_REQUIRED_MESSAGE
+    end
+  end
+end

data/lib/pinqloq/options.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Pinqloq
+  INGEST_BASE_ADDRESS = "https://pinqloq-external-api.pinqponq.io"
+
+  class Options
+    DEFAULT_BULK_PATH = "api/client-logs/bulk"
+    DEFAULT_BATCH_SIZE = 200
+    DEFAULT_FLUSH_INTERVAL = 2.0
+    DEFAULT_QUEUE_CAPACITY = 10_000
+    DEFAULT_HTTP_TIMEOUT = 10.0
+
+    attr_reader :secret_key, :api_logs_collection_name, :bulk_path, :batch_size,
+                :flush_interval, :queue_capacity, :http_timeout, :app_version_name, :device_identifier
+
+    def initialize(
+      secret_key:,
+      api_logs_collection_name: nil,
+      bulk_path: DEFAULT_BULK_PATH,
+      batch_size: DEFAULT_BATCH_SIZE,
+      flush_interval: DEFAULT_FLUSH_INTERVAL,
+      queue_capacity: DEFAULT_QUEUE_CAPACITY,
+      http_timeout: DEFAULT_HTTP_TIMEOUT,
+      app_version_name: nil,
+      device_identifier: nil
+    )
+      raise ArgumentError, "Pinqloq: secret_key is required." if secret_key.nil? || secret_key.strip.empty?
+
+      @secret_key = secret_key
+      @api_logs_collection_name = api_logs_collection_name
+      @bulk_path = bulk_path
+      @batch_size = [1, batch_size].max
+      @flush_interval = [0.001, flush_interval].max
+      @queue_capacity = [1, queue_capacity].max
+      @http_timeout = [0.001, http_timeout].max
+      @app_version_name = app_version_name
+      @device_identifier = device_identifier
+    end
+  end
+end

data/lib/pinqloq/rack/captured_body.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Pinqloq
+  module Rack
+    class CapturedBody
+      def initialize(body, max_bytes)
+        @body = body
+        @max_bytes = max_bytes
+        @chunks = []
+        @captured_bytes = 0
+      end
+
+      def each(&block)
+        @body.each do |chunk|
+          capture(chunk)
+          block.call(chunk)
+        end
+      end
+
+      def close
+        @body.close if @body.respond_to?(:close)
+      end
+
+      def captured_string
+        @chunks.join.dup.force_encoding("UTF-8")
+      end
+
+      private
+
+      def capture(chunk)
+        return if @captured_bytes >= @max_bytes
+
+        remaining = @max_bytes - @captured_bytes
+        slice = chunk.bytesize > remaining ? chunk.byteslice(0, remaining) : chunk
+
+        @chunks << slice
+        @captured_bytes += slice.bytesize
+      end
+    end
+  end
+end

data/lib/pinqloq/rack/path_match.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Pinqloq
+  module Rack
+    module PathMatch
+      module_function
+
+      def matches_any_prefix?(path, prefixes)
+        return false if prefixes.nil? || prefixes.empty?
+
+        lower_path = path.downcase
+        prefixes.any? { |prefix| matches_segment_prefix?(lower_path, prefix.downcase) }
+      end
+
+      def matches_segment_prefix?(lower_path, prefix)
+        normalized = prefix.start_with?("/") ? prefix : "/#{prefix}"
+        trimmed = (normalized.length > 1 && normalized.end_with?("/")) ? normalized[0..-2] : normalized
+
+        return false unless lower_path.start_with?(trimmed)
+
+        lower_path.length == trimmed.length || lower_path[trimmed.length] == "/"
+      end
+    end
+  end
+end

data/lib/pinqloq/rack/request_logging.rb

@@ -0,0 +1,240 @@
+# frozen_string_literal: true
+
+require "rack"
+require "securerandom"
+require "stringio"
+require_relative "path_match"
+require_relative "captured_body"
+require_relative "../redaction/redaction"
+require_relative "../log_level"
+require_relative "../log_source_type"
+require_relative "../internal/throttled_warn"
+
+module Pinqloq
+  module Rack
+    class RequestLogging
+      MAX_BODY_CHARACTERS = 32 * 1024
+      MAX_BODY_BYTES = 4 * MAX_BODY_CHARACTERS
+
+      DEVICE_IDENTIFIER_HEADER_NAME = "device-identifier"
+      CORRELATION_ID_HEADER_NAME = "correlation-id"
+
+      DEVICE_IDENTIFIER_REQUIRED_MESSAGE =
+        "Pinqloq: the required device_identifier could not be resolved. Send the " \
+        "'device-identifier' request header, or configure resolve_device_identifier, or set " \
+        "Options#device_identifier."
+
+      SERVER_ERROR_STATUS_THRESHOLD = 500
+      CLIENT_ERROR_STATUS_THRESHOLD = 400
+
+      SELECTOR_WARNING_THROTTLE_SECONDS = 60
+
+      def initialize(
+        app,
+        logger:,
+        pinqloq_options:,
+        exclude_paths: [],
+        resolve_device_identifier: nil,
+        resolve_app_version_name: nil,
+        metadata: {},
+        detail: {},
+        redact_fields: [],
+        redact_paths: []
+      )
+        @app = app
+        @logger = logger
+        @pinqloq_options = pinqloq_options
+        @exclude_paths = exclude_paths
+        @resolve_device_identifier = resolve_device_identifier
+        @resolve_app_version_name = resolve_app_version_name
+        @metadata_enrichers = metadata
+        @detail_enrichers = detail
+        @redact_fields = redact_fields
+        @redact_paths = redact_paths
+      end
+
+      def call(env)
+        request = ::Rack::Request.new(env)
+        path = request.path
+
+        return @app.call(env) if PathMatch.matches_any_prefix?(path, @exclude_paths)
+
+        device_identifier = resolve_device_identifier(request)
+        if device_identifier.nil? || device_identifier.strip.empty?
+          return [400, { "Content-Type" => "text/plain" }, [DEVICE_IDENTIFIER_REQUIRED_MESSAGE]]
+        end
+
+        redact_plan = build_redact_plan(path)
+        started_at = monotonic_now
+
+        request_headers = truncate(Redaction.serialize_headers(request_header_hash(env), redact_plan))
+        input_json = truncate(Redaction.apply_body_redaction(read_request_body(request), redact_plan))
+        correlation_id = resolve_correlation_id(request)
+        app_version_name = resolve_selector(@resolve_app_version_name, request, "resolve_app_version_name")
+
+        status, headers, body = @app.call(env)
+
+        captured_body = CapturedBody.new(body, MAX_BODY_BYTES)
+        wrapped_body = ::Rack::BodyProxy.new(captured_body) do
+          finish_log(
+            method: request.request_method,
+            path: path,
+            status: status,
+            headers: headers,
+            elapsed_ms: ((monotonic_now - started_at) * 1000).round,
+            device_identifier: device_identifier,
+            app_version_name: app_version_name,
+            correlation_id: correlation_id,
+            request_headers: request_headers,
+            input_json: input_json,
+            output_json: truncate(Redaction.apply_body_redaction(captured_body.captured_string, redact_plan)),
+            request: request,
+            redact_plan: redact_plan
+          )
+        end
+
+        [status, headers, wrapped_body]
+      end
+
+      private
+
+      def build_redact_plan(path)
+        if !@redact_paths.empty? && PathMatch.matches_any_prefix?(path, @redact_paths)
+          Redaction::Plan::ALL
+        else
+          Redaction::Plan.new(redact_all: false, declared_names: @redact_fields)
+        end
+      end
+
+      def finish_log(method:, path:, status:, headers:, elapsed_ms:, device_identifier:, app_version_name:,
+                      correlation_id:, request_headers:, input_json:, output_json:, request:, redact_plan:)
+        response_headers = truncate(Redaction.serialize_headers(headers, redact_plan))
+
+        metadata = {}
+        metadata["event"] = "#{method} #{path}".strip
+        apply_enrichers(metadata, @metadata_enrichers, request, status, headers)
+        resolved_event_name = metadata.delete("event")
+
+        metadata["method"] = method
+        metadata["statusCode"] = status.to_s
+        metadata["durationMs"] = elapsed_ms.to_s
+        metadata["RequestMethod"] = method
+        metadata["ResponseCode"] = status.to_s
+
+        detail = {}
+        apply_enrichers(detail, @detail_enrichers, request, status, headers)
+        detail["InputJson"] = input_json
+        detail["OutputJson"] = output_json
+        detail["RequestHeaders"] = request_headers
+        detail["ResponseHeaders"] = response_headers
+
+        @logger.enqueue(
+          Pinqloq::LogEntry.new(
+            event: resolved_event_name,
+            log_level: resolve_log_level(status),
+            device_identifier: device_identifier,
+            app_version_name: (app_version_name.nil? || app_version_name.empty?) ? nil : app_version_name,
+            log_source_type: LogSourceType::BACKEND,
+            correlation_id: correlation_id,
+            path: path,
+            metadata: metadata,
+            detail: detail
+          )
+        )
+      end
+
+      def resolve_log_level(status)
+        return LogLevel::ERROR if status >= SERVER_ERROR_STATUS_THRESHOLD
+        return LogLevel::WARNING if status >= CLIENT_ERROR_STATUS_THRESHOLD
+
+        LogLevel::INFORMATION
+      end
+
+      def resolve_device_identifier(request)
+        overridden = resolve_selector(@resolve_device_identifier, request, "resolve_device_identifier")
+        return overridden unless overridden.nil? || overridden.strip.empty?
+
+        header = request.get_header(rack_header_key(DEVICE_IDENTIFIER_HEADER_NAME))
+        return header unless header.nil? || header.strip.empty?
+
+        @pinqloq_options.device_identifier
+      end
+
+      def resolve_correlation_id(request)
+        header = request.get_header(rack_header_key(CORRELATION_ID_HEADER_NAME))
+        (header && !header.strip.empty?) ? header : SecureRandom.uuid
+      end
+
+      def resolve_selector(selector, request, throttle_key)
+        return nil unless selector
+
+        selector.call(request)
+      rescue StandardError => e
+        Internal::ThrottledWarn.warn_throttled(
+          throttle_key,
+          SELECTOR_WARNING_THROTTLE_SECONDS,
+          "Pinqloq: #{throttle_key} raised an exception; ignored. #{e.class}: #{e.message}"
+        )
+        nil
+      end
+
+      def apply_enrichers(target, enrichers, request, status, headers)
+        enrichers.each do |key, selector|
+          value =
+            begin
+              selector.call(request, status, headers)
+            rescue StandardError => e
+              Internal::ThrottledWarn.warn_throttled(
+                "enricher:#{key}",
+                SELECTOR_WARNING_THROTTLE_SECONDS,
+                "Pinqloq: the '#{key}' enricher raised an exception; ignored. #{e.class}: #{e.message}"
+              )
+              next
+            end
+
+          target[key.to_s] = value.to_s unless value.nil?
+        end
+      end
+
+      def request_header_hash(env)
+        headers = env.each_with_object({}) do |(key, value), acc|
+          next unless key.start_with?("HTTP_")
+
+          header_name = key.sub(/\AHTTP_/, "").tr("_", "-")
+          acc[header_name] = value
+        end
+
+        headers["Content-Type"] = env["CONTENT_TYPE"] if env["CONTENT_TYPE"]
+        headers["Content-Length"] = env["CONTENT_LENGTH"] if env["CONTENT_LENGTH"]
+        headers
+      end
+
+      def rack_header_key(header_name)
+        "HTTP_#{header_name.upcase.tr('-', '_')}"
+      end
+
+      def read_request_body(request)
+        input = request.get_header("rack.input")
+        return "" unless input
+
+        if input.respond_to?(:rewind)
+          body = input.read(MAX_BODY_BYTES) || ""
+          input.rewind
+          return body.force_encoding("UTF-8")
+        end
+
+        full_body = (input.read || "").force_encoding("UTF-8")
+        request.set_header("rack.input", StringIO.new(full_body))
+        full_body[0, MAX_BODY_BYTES] || ""
+      end
+
+      def truncate(value)
+        value.length <= MAX_BODY_CHARACTERS ? value : value[0, MAX_BODY_CHARACTERS]
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/pinqloq/redaction/redaction.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "redaction_plan"
+require_relative "../internal/throttled_warn"
+
+module Pinqloq
+  module Redaction
+    REDACTED_VALUE = "*****REDACTED*****"
+
+    UNPARSEABLE_SENSITIVE_BODY_VALUE =
+      "*****REDACTED: body carries a credential field and is not parseable JSON " \
+      "(non-JSON content type, or longer than the capture limit)*****"
+
+    MALFORMED_BODY_WARNING_THROTTLE_SECONDS = 60
+
+    class << self
+      def redact_properties(value, plan)
+        case value
+        when Array
+          value.map { |item| redact_properties(item, plan) }
+        when Hash
+          value.each_with_object({}) do |(key, item), result|
+            result[key] = plan.should_redact?(key.to_s) ? REDACTED_VALUE : redact_properties(item, plan)
+          end
+        else
+          value
+        end
+      end
+
+      def redact_fully(value)
+        case value
+        when Array
+          value.map { |item| redact_fully(item) }
+        when Hash
+          value.each_with_object({}) { |(key, item), result| result[key] = redact_fully(item) }
+        else
+          REDACTED_VALUE
+        end
+      end
+
+      def redact_json_properties(body, plan, sensitive)
+        return body if body.strip.empty?
+
+        JSON.generate(redact_properties(JSON.parse(body), plan))
+      rescue JSON::ParserError => e
+        Internal::ThrottledWarn.warn_throttled(
+          :redact_json_properties,
+          MALFORMED_BODY_WARNING_THROTTLE_SECONDS,
+          "Pinqloq: a captured body could not be parsed as JSON; falling back to whole-body handling. #{e.class}: #{e.message}"
+        )
+        sensitive ? UNPARSEABLE_SENSITIVE_BODY_VALUE : body
+      end
+
+      def redact_json_fully(body)
+        return body if body.strip.empty?
+
+        JSON.generate(redact_fully(JSON.parse(body)))
+      rescue JSON::ParserError => e
+        Internal::ThrottledWarn.warn_throttled(
+          :redact_json_fully,
+          MALFORMED_BODY_WARNING_THROTTLE_SECONDS,
+          "Pinqloq: a captured body under a redact_all plan could not be parsed as JSON; masking it wholesale. #{e.class}: #{e.message}"
+        )
+        REDACTED_VALUE
+      end
+
+      def apply_body_redaction(body, plan)
+        return redact_json_fully(body) if plan.redact_all
+
+        mentions_credential = Plan.contains_always_redacted_name?(body) || plan.contains_declared_name?(body)
+
+        return body if !plan.has_declared_redactions? && !mentions_credential
+
+        redact_json_properties(body, plan, mentions_credential)
+      end
+
+      def serialize_headers(headers, plan)
+        result = headers.each_with_object({}) do |(key, value), acc|
+          next if value.nil?
+
+          string_value = value.is_a?(Array) ? value.join(", ") : value.to_s
+          acc[key.to_s] = plan.should_redact?(key.to_s) ? REDACTED_VALUE : string_value
+        end
+
+        JSON.generate(result)
+      end
+    end
+  end
+end

data/lib/pinqloq/redaction/redaction_plan.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Pinqloq
+  module Redaction
+    ALWAYS_REDACTED_NAMES = Set.new(
+      [
+        "authorization",
+        "proxy-authorization",
+        "cookie",
+        "set-cookie",
+        "x-api-key",
+        "x-secret-key",
+        "x-auth-token",
+        "x-access-token",
+        "x-csrf-token",
+        "x-xsrf-token",
+        "secret_key",
+        "password",
+        "newpassword",
+        "oldpassword",
+        "currentpassword",
+        "passwordconfirmation",
+        "confirmpassword",
+        "secret",
+        "secretkey",
+        "clientsecret",
+        "apikey",
+        "accesstoken",
+        "refreshtoken",
+        "idtoken",
+        "token",
+        "otp",
+        "otpcode",
+        "verificationcode",
+        "pin",
+        "privatekey",
+        "cardnumber",
+        "cvv",
+        "cvc",
+        "securitycode",
+        "iban",
+        "ssn"
+      ]
+    ).freeze
+
+    module_function
+
+    def bounded_name?(body, index, length)
+      end_index = index + length
+
+      before = index.positive? ? body[index - 1] : nil
+      at_start = body[index]
+      after = end_index < body.length ? body[end_index] : nil
+
+      start_bounded = index.zero? || !letter_or_digit?(before) || (upper?(at_start) && !upper?(before))
+      end_bounded = end_index == body.length || !letter_or_digit?(after) || upper?(after)
+
+      start_bounded && end_bounded
+    end
+
+    def letter_or_digit?(char)
+      !char.nil? && char =~ /[A-Za-z0-9]/ ? true : false
+    end
+
+    def upper?(char)
+      !char.nil? && char =~ /[A-Z]/ ? true : false
+    end
+
+    def contains_whole_name?(body, names)
+      lower_body = body.downcase
+
+      names.each do |name|
+        next if name.empty?
+
+        search_from = 0
+        while search_from <= lower_body.length - name.length
+          index = lower_body.index(name, search_from)
+          break if index.nil?
+
+          return true if bounded_name?(body, index, name.length)
+
+          search_from = index + 1
+        end
+      end
+
+      false
+    end
+
+    class Plan
+      attr_reader :redact_all
+
+      def initialize(redact_all: false, declared_names: [])
+        @redact_all = redact_all
+        @declared_names = Set.new(declared_names.map(&:downcase))
+      end
+
+      def has_declared_redactions?
+        @redact_all || !@declared_names.empty?
+      end
+
+      def should_redact?(property_or_header_name)
+        lower = property_or_header_name.downcase
+        @redact_all || ALWAYS_REDACTED_NAMES.include?(lower) || @declared_names.include?(lower)
+      end
+
+      def contains_declared_name?(body)
+        Redaction.contains_whole_name?(body, @declared_names)
+      end
+
+      def self.contains_always_redacted_name?(body)
+        Redaction.contains_whole_name?(body, ALWAYS_REDACTED_NAMES)
+      end
+
+      NONE = new(redact_all: false, declared_names: [])
+      ALL = new(redact_all: true, declared_names: [])
+    end
+  end
+end

data/lib/pinqloq/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Pinqloq
+  VERSION = "1.1.2"
+end