prompt-sanitizer 0.1.0

data/lib/prompt_sanitizer/engines/secrets_engine.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+module PromptSanitizer
+  module Engines
+    # Secrets Engine — Layer 1 of prompt-sanitizer (secrets branch).
+    #
+    # Detects credentials, API keys, tokens, and connection strings that should
+    # never reach an LLM. Runs alongside the RegexEngine on every sanitize() call.
+    class SecretsEngine
+      Pattern = Struct.new(:entity_type, :regex, :confidence, :label, keyword_init: true)
+
+      PATTERNS = [
+        # ── JWT (header.payload.signature) ────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::JWT,
+          regex:       /eyJ[a-zA-Z0-9_\-]{10,}\.eyJ[a-zA-Z0-9_\-]{10,}\.[a-zA-Z0-9_\-]{10,}/,
+          confidence:  0.99,
+          label:       "JWT"
+        ),
+
+        # ── Bearer token ──────────────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::BEARER_TOKEN,
+          regex:       /(?:Authorization\s*:\s*)?Bearer\s+([a-zA-Z0-9_\-\.]{20,})/i,
+          confidence:  0.92,
+          label:       "Bearer token"
+        ),
+
+        # ── AWS Access Key ID ──────────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::AWS_ACCESS_KEY,
+          regex:       /(?<![A-Z0-9])(?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}(?![A-Z0-9])/,
+          confidence:  0.99,
+          label:       "AWS access key ID"
+        ),
+
+        # ── AWS Secret Access Key (context-anchored) ───────────────────────────
+        Pattern.new(
+          entity_type: EntityType::AWS_SECRET_KEY,
+          regex:       /(?:aws_secret(?:_access)?_key|secret_access_key)\s*[=:"'\s]\s*([a-zA-Z0-9+\/]{40})/i,
+          confidence:  0.97,
+          label:       "AWS secret access key"
+        ),
+
+        # ── OpenAI API key ─────────────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /sk-(?:proj-|org-)?[a-zA-Z0-9_\-T]{20,}/,
+          confidence:  0.97,
+          label:       "OpenAI API key"
+        ),
+
+        # ── Anthropic API key ──────────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /sk-ant-(?:api\d{2}-)?[a-zA-Z0-9_\-]{20,}/,
+          confidence:  0.99,
+          label:       "Anthropic API key"
+        ),
+
+        # ── GitHub Personal Access Token (classic) ─────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /ghp_[a-zA-Z0-9]{36}/,
+          confidence:  0.99,
+          label:       "GitHub PAT (classic)"
+        ),
+
+        # ── GitHub Fine-grained PAT ────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /github_pat_[a-zA-Z0-9_]{82}/,
+          confidence:  0.99,
+          label:       "GitHub fine-grained PAT"
+        ),
+
+        # ── GitHub OAuth / server-to-server / refresh ──────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /(?:gho|ghs|ghr)_[a-zA-Z0-9]{36}/,
+          confidence:  0.99,
+          label:       "GitHub OAuth/server token"
+        ),
+
+        # ── Slack tokens ───────────────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /xox[baprs]-(?:[0-9a-zA-Z]{4,}-)+[0-9a-zA-Z]{4,}/,
+          confidence:  0.98,
+          label:       "Slack token"
+        ),
+
+        # ── Stripe secret / publishable key ───────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /(?:sk|pk)_(?:live|test)_[a-zA-Z0-9]{24,}/,
+          confidence:  0.99,
+          label:       "Stripe API key"
+        ),
+
+        # ── Twilio Account SID ─────────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /AC[a-f0-9]{32}/,
+          confidence:  0.90,
+          label:       "Twilio Account SID"
+        ),
+
+        # ── Twilio Auth Token ──────────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /(?:auth_token|TWILIO_AUTH_TOKEN)\s*[=:"'\s]\s*([a-f0-9]{32})/i,
+          confidence:  0.97,
+          label:       "Twilio Auth Token"
+        ),
+
+        # ── Google API key ─────────────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /AIza[0-9A-Za-z_\-]{35}/,
+          confidence:  0.99,
+          label:       "Google API key"
+        ),
+
+        # ── SendGrid API key ───────────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /SG\.[a-zA-Z0-9_\-]{22}\.[a-zA-Z0-9_\-]{43}/,
+          confidence:  0.99,
+          label:       "SendGrid API key"
+        ),
+
+        # ── Mailchimp API key ──────────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /[a-f0-9]{32}-us\d{1,2}/,
+          confidence:  0.90,
+          label:       "Mailchimp API key"
+        ),
+
+        # ── HuggingFace token ──────────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /hf_[a-zA-Z0-9]{34,}/,
+          confidence:  0.99,
+          label:       "HuggingFace token"
+        ),
+
+        # ── PEM private key header ─────────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::PRIVATE_KEY,
+          regex:       /-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED |PGP )?PRIVATE KEY-----/,
+          confidence:  0.99,
+          label:       "PEM private key"
+        ),
+
+        # ── Database connection strings ────────────────────────────────────────
+        Pattern.new(
+          entity_type: EntityType::DB_CONNECTION,
+          regex:       /(?:postgres(?:ql)?|mysql|mariadb|mongodb(?:\+srv)?|rediss?|mssql|sqlserver|oracle|clickhouse|cassandra|couchdb|neo4j):\/\/[^\s'"`<>\n]{8,}/i,
+          confidence:  0.97,
+          label:       "Database connection string"
+        ),
+
+        # ── Generic secret assignment (SECRET_KEY=..., api_key: '...') ─────────
+        Pattern.new(
+          entity_type: EntityType::API_KEY,
+          regex:       /(?:secret[_\-]?key|api[_\-]?key|access[_\-]?token|auth[_\-]?token|private[_\-]?key|client[_\-]?secret)\s*[=:"'\s]+\s*([a-zA-Z0-9_\-\.+\/]{16,})/i,
+          confidence:  0.80,
+          label:       "Generic secret assignment"
+        ),
+      ].freeze
+
+      # ── Instance ─────────────────────────────────────────────────────────────
+
+      def initialize
+        @patterns = PATTERNS.dup
+      end
+
+      # Register a custom secret pattern at runtime.
+      def add_pattern(entity_type, regex, label: "custom secret", confidence: 0.85)
+        @patterns << Pattern.new(
+          entity_type: entity_type,
+          regex:       regex,
+          confidence:  confidence,
+          label:       label
+        )
+      end
+
+      # Scan +text+ for secrets and return an Array of DetectedEntity.
+      # When the pattern has a capturing group, the captured value and its
+      # offsets are used; otherwise the full match is used.
+      def detect(text)
+        safe_text = text.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
+        entities  = []
+
+        @patterns.each do |pat|
+          safe_text.scan(pat.regex) do
+            m = Regexp.last_match
+
+            # Use capture group 1 when present (strips surrounding context)
+            if m.captures.any?
+              value     = m[1]
+              start_pos = m.begin(1)
+              end_pos   = m.end(1)
+            else
+              value     = m[0]
+              start_pos = m.begin(0)
+              end_pos   = m.end(0)
+            end
+
+            next if value.nil? || value.empty?
+
+            entities << DetectedEntity.new(
+              entity_type: pat.entity_type,
+              original:    value,
+              replacement: nil,
+              start_pos:   start_pos,
+              end_pos:     end_pos,
+              confidence:  pat.confidence,
+              layer:       :secrets
+            )
+          end
+        end
+
+        entities
+      end
+    end
+  end
+end

data/lib/prompt_sanitizer/entities.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module PromptSanitizer
+  # All PII / sensitive entity types the gem can detect.
+  # Values are symbols — mirror the Python/JS string enum names.
+  module EntityType
+    # ── Personal identifiers ──────────────────────────────────────────────
+    PERSON        = :person
+    EMAIL         = :email
+    PHONE         = :phone
+    DATE_OF_BIRTH = :date_of_birth
+    AGE           = :age
+
+    # ── Location ──────────────────────────────────────────────────────────
+    ADDRESS       = :address
+    ZIP_CODE      = :zip_code
+
+    # ── Financial ─────────────────────────────────────────────────────────
+    CREDIT_CARD   = :credit_card
+    IBAN          = :iban
+    BANK_ACCOUNT  = :bank_account
+    CRYPTO_ADDRESS = :crypto_address
+
+    # ── Government IDs ────────────────────────────────────────────────────
+    SSN             = :ssn
+    PASSPORT        = :passport
+    DRIVING_LICENSE = :driving_license
+
+    # ── Network / Digital ─────────────────────────────────────────────────
+    IP_ADDRESS  = :ip_address
+    MAC_ADDRESS = :mac_address
+    URL         = :url
+
+    # ── Secrets & credentials ─────────────────────────────────────────────
+    API_KEY        = :api_key
+    JWT            = :jwt
+    BEARER_TOKEN   = :bearer_token
+    AWS_ACCESS_KEY = :aws_access_key
+    AWS_SECRET_KEY = :aws_secret_key
+    PRIVATE_KEY    = :private_key
+    DB_CONNECTION  = :db_connection
+
+    # ── Temporal ──────────────────────────────────────────────────────────
+    DATE = :date
+
+    # ── NER-only (detected by SMART / FULL mode) ──────────────────────────
+    ORGANIZATION = :organization
+    LOCATION     = :location
+    MISC         = :misc
+
+    # ── User-defined ──────────────────────────────────────────────────────
+    CUSTOM = :custom
+
+    ALL = constants(false).map { |c| const_get(c) }.freeze
+  end
+end

data/lib/prompt_sanitizer/integrations/action_controller.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module PromptSanitizer
+  module Integrations
+    # ActionController concern — adds PII sanitization helpers to controllers.
+    #
+    # Include in ApplicationController or a specific controller:
+    #
+    #   class ApplicationController < ActionController::Base
+    #     include PromptSanitizer::Integrations::ActionControllerConcern
+    #   end
+    #
+    # Then in any action:
+    #
+    #   # Sanitize specific params in-place before using them:
+    #   def create
+    #     sanitize_params!(:message, :prompt)
+    #     call_llm(params[:message])
+    #   end
+    #
+    #   # Multi-turn session scoped to one action (vault cleared after block):
+    #   def chat
+    #     with_pii_session do |sess|
+    #       clean    = sess.anonymize(params[:message])
+    #       response = call_llm(clean)
+    #       render json: { reply: sess.deanonymize(response) }
+    #     end
+    #   end
+    module ActionControllerConcern
+      # Sanitize one or more param keys in-place.
+      #
+      # @param keys [Array<Symbol, String>] param keys whose string values
+      #   should be sanitized. Nested paths not supported (flatten first).
+      # @return [void]
+      def sanitize_params!(*keys)
+        keys.flatten.each do |key|
+          next unless params[key].is_a?(String)
+
+          result = pii_sanitizer.sanitize(params[key])
+          params[key] = result.text
+        end
+      end
+
+      # Yield a Session scoped to this action's PII vault.
+      # The vault is always cleared after the block, even on exception.
+      #
+      # @yieldparam session [PromptSanitizer::Session]
+      # @return the block's return value
+      def with_pii_session(session_id: nil, &block)
+        id = session_id || "#{controller_name}##{action_name}"
+        pii_sanitizer.session(session_id: id, &block)
+      end
+
+      # The Sanitizer instance used by this controller.
+      # Defaults to the global PromptSanitizer.sanitizer.
+      # Override in a controller to use a custom instance.
+      #
+      # @return [PromptSanitizer::Sanitizer]
+      def pii_sanitizer
+        PromptSanitizer.sanitizer
+      end
+    end
+  end
+end

data/lib/prompt_sanitizer/integrations/active_job.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module PromptSanitizer
+  module Integrations
+    # ActiveJob concern — sanitize job arguments before the job is enqueued.
+    #
+    # Prevents PII from leaking into job queues (Sidekiq, Resque, GoodJob, etc.)
+    # where job arguments are serialized and often stored in plain text.
+    #
+    # == Usage
+    #
+    #   class SummarizeJob < ApplicationJob
+    #     include PromptSanitizer::Integrations::ActiveJobConcern
+    #     sanitize_argument :prompt
+    #
+    #     def perform(prompt:)
+    #       # `prompt` is already sanitized here
+    #       call_llm(prompt)
+    #     end
+    #   end
+    #
+    # == How it works
+    #
+    # +sanitize_argument+ registers an +around_perform+ callback that sanitizes
+    # the named keyword argument(s) before +perform+ is called and restores them
+    # in the job's return context if needed.
+    #
+    # The original PII never touches the queue backend — the redacted version is
+    # enqueued and the job works with sanitized text.
+    module ActiveJobConcern
+      extend ActiveSupport::Concern
+
+      included do
+        # Class-level list of keyword argument names to sanitize.
+        class_attribute :_pii_sanitized_arguments, default: []
+      end
+
+      class_methods do
+        # Declare which keyword arguments should be sanitized before perform.
+        #
+        # @param args [Array<Symbol>] keyword argument names
+        def sanitize_argument(*args)
+          self._pii_sanitized_arguments = _pii_sanitized_arguments | args.map(&:to_sym)
+
+          around_perform do |job, block|
+            sanitizer = PromptSanitizer.sanitizer
+            session   = sanitizer.session(session_id: job.job_id)
+
+            # Sanitize the declared keyword arguments in job_data / arguments.
+            # ActiveJob stores keyword args as the last Hash element in arguments.
+            kwargs = job.arguments.last
+            if kwargs.is_a?(Hash)
+              _pii_sanitized_arguments.each do |key|
+                str_key = key.to_s
+                if kwargs[key].is_a?(String)
+                  kwargs[key] = session.anonymize(kwargs[key])
+                elsif kwargs[str_key].is_a?(String)
+                  kwargs[str_key] = session.anonymize(kwargs[str_key])
+                end
+              end
+            end
+
+            block.call
+          ensure
+            session&.reset
+          end
+        end
+      end
+
+      # Manually sanitize a string using the global sanitizer.
+      #
+      # @param text [String]
+      # @return [String] sanitized text
+      def sanitize_pii(text)
+        PromptSanitizer.sanitizer.sanitize(text).text
+      end
+    end
+  end
+end

data/lib/prompt_sanitizer/integrations/middleware.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module PromptSanitizer
+  module Integrations
+    # Rack middleware — auto-sanitizes incoming JSON request bodies.
+    #
+    # Intercepts POST/PUT/PATCH requests with Content-Type: application/json
+    # and sanitizes the following payload keys before they reach the app:
+    #
+    # - +messages[*].content+  (OpenAI / Anthropic / LangChain style)
+    # - +prompt+, +input+, +inputs+, +text+, +query+
+    #
+    # The sanitized body is written back into the Rack env so that
+    # downstream controllers see clean payloads with no raw PII.
+    #
+    # Optionally, if +restore_response: true+, the middleware deanonymizes
+    # JSON response bodies using the same session vault before sending them
+    # to the client.
+    #
+    # == Usage (manually)
+    #
+    #   use PromptSanitizer::Integrations::SanitizerMiddleware,
+    #       sanitizer: PromptSanitizer.sanitizer,
+    #       routes:    ["/api/llm"],
+    #       restore_response: false
+    #
+    # == Usage (via Railtie config)
+    #
+    #   config.prompt_sanitizer.middleware        = true
+    #   config.prompt_sanitizer.middleware_routes = ["/api/llm"]
+    class SanitizerMiddleware
+      # Keys in a flat JSON body whose string values are sanitized.
+      BODY_KEYS = %w[prompt input inputs text query message content].freeze
+
+      # @param app              [#call]    next Rack app
+      # @param sanitizer        [Sanitizer] defaults to PromptSanitizer.sanitizer
+      # @param routes           [Array<String>, nil] path prefixes to match (nil = all)
+      # @param restore_response [Boolean]  deanonymize JSON response? (default false)
+      def initialize(app, sanitizer: nil, routes: nil, restore_response: false)
+        @app     = app
+        @san     = sanitizer || PromptSanitizer.sanitizer
+        @routes  = routes ? Array(routes) : nil
+        @restore = restore_response
+      end
+
+      def call(env)
+        req = Rack::Request.new(env)
+
+        session = nil
+
+        if should_process?(req)
+          session = _sanitize_request(req)
+        end
+
+        status, headers, body = @app.call(env)
+
+        if session && @restore
+          status, headers, body = _restore_response(status, headers, body, session)
+        end
+
+        [status, headers, body]
+      end
+
+      private
+
+      def should_process?(req)
+        return false unless %w[POST PUT PATCH].include?(req.request_method)
+        return false unless req.content_type&.include?("application/json")
+
+        return true if @routes.nil?
+
+        @routes.any? { |r| req.path.start_with?(r) }
+      end
+
+      def _sanitize_request(req)
+        raw = req.body.read
+        req.body.rewind
+        return nil if raw.nil? || raw.empty?
+
+        begin
+          payload = JSON.parse(raw)
+        rescue JSON::ParserError
+          return nil
+        end
+
+        session = @san.session
+        _sanitize_payload!(payload, session)
+
+        # Write the sanitized body back so downstream sees clean params.
+        sanitized_raw = JSON.generate(payload)
+        env = req.env
+        env["rack.input"]      = StringIO.new(sanitized_raw)
+        env["CONTENT_LENGTH"]  = sanitized_raw.bytesize.to_s
+        # Invalidate any already-parsed params cache.
+        env.delete("rack.request.form_hash")
+        env.delete("rack.request.form_input")
+
+        session
+      rescue StandardError
+        nil
+      end
+
+      def _sanitize_payload!(obj, session)
+        case obj
+        when Hash
+          # messages array (OpenAI / LangChain style)
+          if obj["messages"].is_a?(Array)
+            obj["messages"].each do |msg|
+              next unless msg.is_a?(Hash) && msg["content"].is_a?(String)
+
+              msg["content"] = session.anonymize(msg["content"])
+            end
+          end
+
+          BODY_KEYS.each do |key|
+            obj[key] = session.anonymize(obj[key]) if obj[key].is_a?(String)
+          end
+        when Array
+          obj.each { |item| _sanitize_payload!(item, session) }
+        end
+      end
+
+      def _restore_response(status, headers, body, session)
+        content_type = headers["Content-Type"] || headers["content-type"] || ""
+        return [status, headers, body] unless content_type.include?("application/json")
+
+        raw = +""
+        body.each { |chunk| raw << chunk }
+
+        begin
+          obj      = JSON.parse(raw)
+          restored = _restore_obj(obj, session)
+          new_body = JSON.generate(restored)
+          headers["Content-Length"] = new_body.bytesize.to_s
+          [status, headers, [new_body]]
+        rescue JSON::ParserError
+          [status, headers, [raw]]
+        end
+      ensure
+        body.close if body.respond_to?(:close)
+      end
+
+      def _restore_obj(obj, session)
+        case obj
+        when String then session.deanonymize(obj)
+        when Hash   then obj.transform_values { |v| _restore_obj(v, session) }
+        when Array  then obj.map { |v| _restore_obj(v, session) }
+        else             obj
+        end
+      end
+    end
+  end
+end

data/lib/prompt_sanitizer/modes.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module PromptSanitizer
+  module Mode
+    # Regex + secrets only. Sub-millisecond. Zero dependencies.
+    # Catches: email, phone, SSN, credit card, IBAN, IP, crypto, MAC, URL,
+    # API keys, JWTs, bearer tokens, AWS keys, DB connection strings,
+    # private keys, passport, driving licence, date-of-birth.
+    FAST = :fast
+
+    # FAST + NER (informers/distilbert or mitie). ~25–50 ms on CPU.
+    # Additionally catches: names, organisations, locations, misc entities.
+    # Requires: gem "informers"  (or gem "mitie")
+    SMART = :smart
+
+    # Everything in SMART plus synthetic replacements and audit logging.
+    # Requires: gem "informers", gem "faker"
+    FULL = :full
+
+    ALL = [FAST, SMART, FULL].freeze
+
+    def self.valid?(mode)
+      ALL.include?(mode)
+    end
+  end
+end

data/lib/prompt_sanitizer/railtie.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module PromptSanitizer
+  # Rails::Railtie — hooks prompt-sanitizer into the Rails boot process.
+  #
+  # Loaded automatically when Rails is present (detected in prompt_sanitizer.rb).
+  # Exposes a configuration namespace under +config.prompt_sanitizer+
+  # so Rails apps can tune the gem from an initializer without monkey-patching.
+  #
+  # Generated initializer (from +rails g prompt_sanitizer:install+)::
+  #
+  #   PromptSanitizer.configure do |c|
+  #     c.mode        = :smart
+  #     c.locale      = "en"
+  #     c.audit_log   = :memory
+  #   end
+  #
+  # Middleware (optional)::
+  #
+  #   config.prompt_sanitizer.middleware = true          # all routes
+  #   config.prompt_sanitizer.middleware_routes = ["/api/llm", "/chat"]
+  #   config.prompt_sanitizer.restore_response  = false
+  class Railtie < Rails::Railtie
+    # Expose a config object on the Rails::Application config.
+    config.prompt_sanitizer = ActiveSupport::OrderedOptions.new
+    config.prompt_sanitizer.middleware        = false
+    config.prompt_sanitizer.middleware_routes = nil
+    config.prompt_sanitizer.restore_response  = false
+
+    # After all initializers run, insert middleware if requested.
+    initializer "prompt_sanitizer.insert_middleware", after: :load_config_initializers do |app|
+      cfg = app.config.prompt_sanitizer
+      if cfg.middleware
+        require_relative "integrations/middleware"
+        app.middleware.use(
+          PromptSanitizer::Integrations::SanitizerMiddleware,
+          sanitizer:        PromptSanitizer.sanitizer,
+          routes:           cfg.middleware_routes,
+          restore_response: cfg.restore_response
+        )
+      end
+    end
+  end
+end