scrubber_rb 0.1.0

data/lib/scrubber/instance.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+module Scrubber
+  # A compiled, immutable scrubber.
+  #
+  # Compiling ~35 regexes into one automaton is the expensive part; scanning is
+  # cheap. Build one of these at boot and reuse it forever:
+  #
+  #   SCRUBBER = Scrubber.new(replacement: :hash)
+  #   SCRUBBER.scrub(line)
+  #
+  # Instances are frozen and hold no per-call state, so a single instance is
+  # safe to share across every thread in the process.
+  class Instance
+    REPLACEMENTS = %i[label mask hash remove].freeze
+
+    # Read in 1MB slices, and never cut closer than this to the end of the
+    # buffer, so a match straddling a chunk boundary still sees both halves.
+    CHUNK_SIZE = 1024 * 1024
+    CARRY_SIZE = 8 * 1024
+
+    attr_reader :detectors, :custom, :replacement, :hash_salt
+
+    def initialize(detectors: Scrubber::DEFAULTS, custom: {}, replacement: :label, hash_salt: nil)
+      @detectors = normalize_detectors(detectors)
+      @custom = normalize_custom(custom)
+      @replacement = normalize_replacement(replacement)
+      @hash_salt = hash_salt&.to_s
+
+      @native = Native.new(
+        @detectors.map(&:to_s),
+        @custom.map { |name, regexp| [name.to_s, regexp.source, regexp.options] },
+        @replacement.to_s,
+        @hash_salt
+      )
+      freeze
+    end
+
+    # Redact every match in `text`, returning a new String with the same
+    # encoding. The input is never mutated, and frozen input is fine.
+    def scrub(text)
+      str = coerce(text)
+      replaced = @native.scrub(str)
+      return str.dup if replaced.nil?
+
+      replaced.force_encoding(str.encoding)
+    end
+
+    # Redact in place. Returns the same object, so it raises FrozenError on a
+    # frozen string exactly like every other Ruby bang method.
+    def scrub!(text)
+      str = coerce(text)
+      replaced = @native.scrub(str)
+      return str if replaced.nil?
+
+      str.replace(replaced.force_encoding(str.encoding))
+    end
+
+    # Find matches without replacing them.
+    #
+    #   Scrubber.detect("mail nik@example.com")
+    #   # => [#<Scrubber::Match type=:email 5...20 preview="n**@e******.com">]
+    #
+    # Offsets are character offsets into `text`, not byte offsets, so they index
+    # the Ruby string correctly even when it contains emoji or Devanagari.
+    def detect(text)
+      str = coerce(text)
+      # Only UTF-8 needs byte->character conversion; in single-byte encodings
+      # the two are the same number.
+      char_offsets = str.encoding == Encoding::UTF_8
+      @native.detect(str, char_offsets).map do |type, from, to, preview|
+        Match.new(type: type.to_sym, begin: from, end: to, preview: preview)
+      end
+    end
+
+    # True if anything at all would be redacted. Cheaper than `scrub` when you
+    # only need a yes/no (an audit check, a test assertion, a CI gate).
+    def match?(text)
+      # `detect` stops at the match list; it never builds the output string.
+      !@native.detect(coerce(text), false).empty?
+    end
+
+    # Stream a file through the engine.
+    #
+    # Reads in 1MB chunks and cuts each chunk at the last line break before an
+    # 8KB carry window, so a match spanning a chunk boundary is never sliced in
+    # half. Multi-line PEM blocks get an extra guard: if a chunk would end
+    # between BEGIN and END, the cut moves back before the BEGIN.
+    #
+    # Returns the number of bytes written.
+    def scrub_file(input_path, output_path, chunk_size: CHUNK_SIZE)
+      written = 0
+      File.open(input_path, "rb") do |input|
+        File.open(output_path, "wb") do |output|
+          each_safe_chunk(input, chunk_size) do |chunk|
+            written += output.write(@native.scrub(chunk) || chunk)
+          end
+        end
+      end
+      written
+    end
+
+    # How many compiled patterns back this instance. Detectors expand to more
+    # than one rule each in a few cases (`:api_key` alone is ~19).
+    def rule_count
+      @native.rule_count
+    end
+
+    def inspect
+      "#<Scrubber::Instance detectors=#{@detectors.size} rules=#{rule_count} " \
+        "replacement=#{@replacement.inspect}>"
+    end
+
+    # Config identity, used to memoize module-level `Scrubber.scrub` calls.
+    def cache_key
+      self.class.cache_key(
+        detectors: @detectors, custom: @custom,
+        replacement: @replacement, hash_salt: @hash_salt
+      )
+    end
+
+    def self.cache_key(detectors:, custom:, replacement:, hash_salt:)
+      customs = custom.to_a.map do |name, regexp|
+        source = regexp.respond_to?(:source) ? regexp.source : regexp.to_s
+        options = regexp.respond_to?(:options) ? regexp.options : 0
+        [name.to_sym, source, options]
+      end
+      [Array(detectors).map(&:to_sym).sort, customs.sort, replacement.to_sym, hash_salt&.to_s]
+    end
+
+    private
+
+    def coerce(text)
+      return text if text.is_a?(String)
+      raise TypeError, "expected a String, got #{text.class}" unless text.respond_to?(:to_str)
+
+      text.to_str
+    end
+
+    def normalize_detectors(detectors)
+      list = Array(detectors).map do |d|
+        unless d.respond_to?(:to_sym)
+          raise ConfigurationError,
+                "detector names must be Symbols or Strings, got #{d.class}"
+        end
+
+        d.to_sym
+      end
+      list.uniq.freeze
+    end
+
+    def normalize_custom(custom)
+      raise ConfigurationError, "custom: must be a Hash of name => Regexp" unless custom.respond_to?(:to_h)
+
+      custom.to_h.each_with_object({}) do |(name, regexp), out|
+        unless regexp.is_a?(Regexp)
+          raise ConfigurationError,
+                "custom detector #{name.inspect} must be a Regexp, got #{regexp.class}"
+        end
+
+        out[name.to_sym] = regexp
+      end.freeze
+    end
+
+    def normalize_replacement(replacement)
+      sym = replacement.to_sym
+      unless REPLACEMENTS.include?(sym)
+        raise ConfigurationError,
+              "unknown replacement #{replacement.inspect}. Expected one of: #{REPLACEMENTS.join(", ")}"
+      end
+
+      sym
+    end
+
+    # Yields byte slices that are safe to scan independently.
+    def each_safe_chunk(input, chunk_size)
+      buffer = +""
+      buffer.force_encoding(Encoding::BINARY)
+
+      while (chunk = input.read(chunk_size))
+        buffer << chunk
+        next if buffer.bytesize <= CARRY_SIZE
+
+        cut = safe_cut(buffer)
+        next if cut.zero?
+
+        yield buffer.byteslice(0, cut)
+        buffer = buffer.byteslice(cut, buffer.bytesize - cut)
+      end
+
+      yield buffer unless buffer.empty?
+    end
+
+    # Where can this buffer be cut without slicing a match in half?
+    def safe_cut(buffer)
+      limit = buffer.bytesize - CARRY_SIZE
+      return 0 if limit <= 0
+
+      cut = buffer.rindex("\n", limit - 1)
+      cut = cut ? cut + 1 : limit
+      pem_safe_cut(buffer, cut)
+    end
+
+    # A PEM block can be longer than the carry window, so cutting at a line
+    # break is not enough: if the head we are about to emit opens a block it
+    # does not close, back the cut up to just before that BEGIN.
+    def pem_safe_cut(buffer, cut)
+      # Give up on the guard rather than buffer a whole file: a PEM block this
+      # big is malformed, and unbounded memory growth is the worse failure.
+      return cut if buffer.bytesize > CHUNK_SIZE * 8
+
+      head = buffer.byteslice(0, cut)
+      opens = head.scan("-----BEGIN").size
+      closes = head.scan("-----END").size
+      return cut if opens <= closes
+
+      last_begin = head.rindex("-----BEGIN")
+      return cut if last_begin.nil? || last_begin.zero?
+
+      last_begin
+    end
+  end
+end

data/lib/scrubber/llm_guard.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Scrubber
+  # Redact prompts on their way to a third-party model API.
+  #
+  #   guard = Scrubber::LLMGuard.new(replacement: :hash)
+  #   chat.ask(guard.call(user_message))
+  #
+  # `:hash` is the interesting default here. With `[EMAIL:9f86d081]` tokens the
+  # model can still reason about "the same customer" across a conversation, and
+  # your logs of that conversation stay correlatable, without the value ever
+  # crossing the network.
+  #
+  # It accepts the shapes prompts actually come in:
+  #
+  #   guard.call("my email is nik@example.com")
+  #   guard.call([{ role: "user", content: "..." }])
+  #   guard.call({ role: "user", content: "..." })
+  #
+  # There is no framework hook here on purpose. `LLMGuard` is a plain callable,
+  # so it drops into RubyLLM, langchainrb, ruby-openai, anthropic-sdk-ruby, or a
+  # hand-rolled `Net::HTTP` call the same way. See the README for the RubyLLM
+  # wrapper pattern.
+  class LLMGuard
+    # Message hash keys whose values are prompt text.
+    CONTENT_KEYS = %w[content text prompt input].freeze
+
+    attr_reader :scrubber
+
+    def initialize(scrubber: nil, replacement: :hash, **options)
+      @scrubber = scrubber || Scrubber.new(replacement: replacement, **options)
+    end
+
+    # Redact `input`, preserving its shape.
+    def call(input)
+      case input
+      when String then @scrubber.scrub(input)
+      when Array  then input.map { |item| call(item) }
+      when Hash   then scrub_hash(input)
+      when Symbol, Numeric, NilClass, TrueClass, FalseClass then input
+      else
+        input.respond_to?(:to_str) ? @scrubber.scrub(input.to_str) : input
+      end
+    end
+    alias scrub call
+
+    # Usable directly as a block: `messages.map(&guard)`.
+    def to_proc
+      method(:call).to_proc
+    end
+
+    # What would have been redacted. Handy for a "we blocked N leaks today"
+    # metric, or for failing a test when a prompt builder regresses.
+    def findings(input)
+      case input
+      when String then @scrubber.detect(input)
+      when Array  then input.flat_map { |item| findings(item) }
+      when Hash   then content_values(input).flat_map { |v| findings(v) }
+      else []
+      end
+    end
+
+    private
+
+    def scrub_hash(hash)
+      hash.each_with_object(hash.class.new) do |(key, value), out|
+        out[key] = content_key?(key) || value.is_a?(Array) || value.is_a?(Hash) ? call(value) : value
+      end
+    end
+
+    def content_values(hash)
+      hash.filter_map { |key, value| value if content_key?(key) }
+    end
+
+    def content_key?(key)
+      CONTENT_KEYS.include?(key.to_s)
+    end
+  end
+end

data/lib/scrubber/log_formatter.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Scrubber
+  # Wraps an existing Logger formatter and redacts whatever it produces.
+  #
+  #   # config/initializers/scrubber.rb
+  #   Rails.logger.formatter = Scrubber::LogFormatter.new(Rails.logger.formatter)
+  #
+  # Wrapping the formatter rather than the logger means it catches everything:
+  # your own `Rails.logger.info`, Active Record's SQL echo, Rack's request
+  # lines, and the exception messages your error middleware logs on the way out.
+  #
+  # It runs on the log write path, so it is worth knowing what it costs: one
+  # Rust scan per line, which on typical log lines is a few microseconds. If
+  # your app is log-bound, narrow `detectors:` to what you actually care about.
+  class LogFormatter
+    attr_reader :scrubber
+
+    # @param formatter [#call] the formatter to wrap. Defaults to
+    #   `Logger::Formatter`, which is what a bare `Logger` uses.
+    # @param scrubber [Scrubber::Instance] a prebuilt engine, if you have one.
+    # @param options [Hash] otherwise, options for {Scrubber.new}.
+    def initialize(formatter = nil, scrubber: nil, **options)
+      @formatter = formatter || self.class.default_formatter
+      @scrubber = scrubber || Scrubber.new(**options)
+    end
+
+    def call(severity, time, progname, msg)
+      formatted = @formatter.call(severity, time, progname, msg)
+      return formatted unless formatted.is_a?(String)
+
+      @scrubber.scrub(formatted)
+    end
+
+    def self.default_formatter
+      require "logger"
+      ::Logger::Formatter.new
+    end
+  end
+end

data/lib/scrubber/match.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Scrubber
+  # One thing the engine found, without the thing itself.
+  #
+  # `begin` and `end` are character offsets into the string you passed, so
+  # `text[match.begin...match.end]` gives you the raw value back if you really
+  # want it. `preview` is already masked — it exists so you can log "we found an
+  # email here" without logging the email.
+  Match = Struct.new(:type, :begin, :end, :preview, keyword_init: true) do
+    # The span as a Range, ready for `String#[]`.
+    def to_range
+      self.begin...self.end
+    end
+
+    # Length of the match in characters.
+    def length
+      self.end - self.begin
+    end
+
+    def inspect
+      "#<Scrubber::Match type=#{type.inspect} #{self.begin}...#{self.end} " \
+        "preview=#{preview.inspect}>"
+    end
+    alias_method :to_s, :inspect
+  end
+end

data/lib/scrubber/middleware.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Scrubber
+  # Rack middleware that makes a redacted view of the request available to
+  # everything downstream, and redacts exception messages on the way out.
+  #
+  #   use Scrubber::Middleware, detectors: Scrubber::DEFAULTS
+  #
+  # == What it does not do
+  #
+  # It does not rewrite `QUERY_STRING`, `rack.input`, or `params`. Redacting the
+  # request the application is about to act on would break every login form in
+  # the world: the app needs the real password to check it. What leaks is not
+  # the request, it is the *record* of the request — the log line, the error
+  # tracker payload, the APM trace.
+  #
+  # So this middleware gives you redacted copies to record:
+  #
+  #   env["scrubber.query_string"]  # "user=x&password=[PASSWORD_PAIR]"
+  #   env["scrubber.params"]        # { "user" => "x", "password" => "[PASSWORD_PAIR]" }
+  #   env["scrubber.instance"]      # the engine, for anything else you log
+  #
+  # and it redacts the message of any exception raised further down the stack
+  # before re-raising it, so an error tracker that never heard of this gem still
+  # gets a clean payload.
+  class Middleware
+    QUERY_STRING = "QUERY_STRING"
+    ENV_QUERY = "scrubber.query_string"
+    ENV_PARAMS = "scrubber.params"
+    ENV_INSTANCE = "scrubber.instance"
+
+    attr_reader :scrubber
+
+    # @param app [#call] the next Rack app.
+    # @param scrubber [Scrubber::Instance] a prebuilt engine, if you have one.
+    # @param scrub_exceptions [Boolean] redact exception messages (default true).
+    # @param options [Hash] otherwise, options for {Scrubber.new}.
+    def initialize(app, scrubber: nil, scrub_exceptions: true, **options)
+      @app = app
+      @scrubber = scrubber || Scrubber.new(**options)
+      @scrub_exceptions = scrub_exceptions
+    end
+
+    def call(env)
+      annotate(env)
+      @app.call(env)
+    rescue StandardError => e
+      raise unless @scrub_exceptions
+
+      raise redacted(e)
+    end
+
+    private
+
+    def annotate(env)
+      env[ENV_INSTANCE] = @scrubber
+      query = env[QUERY_STRING]
+      return if query.nil? || query.empty?
+
+      # Decode *before* scrubbing. `email=nik%40example.com` does not look like
+      # an email address until the `%40` is a `@`, and a redactor that can be
+      # defeated by percent-encoding is not a redactor. The decoded key is put
+      # back in front of the value before scrubbing, because `:password_pair`
+      # needs that context to recognise `hunter2` as a password at all.
+      pairs = parse_pairs(query).map { |key, value| [key, scrub_value(key, value)] }
+
+      env[ENV_PARAMS] = pairs.to_h
+      # Rebuilt from the decoded pairs, so this is a string for logs to print,
+      # not a query string to re-issue a request with.
+      env[ENV_QUERY] = pairs.map { |key, value| "#{key}=#{value}" }.join("&")
+    end
+
+    # Deliberately not `Rack::Utils.parse_nested_query`: this is a logging aid,
+    # it must never raise on a malformed query string, and the gem must stay
+    # loadable without Rack.
+    def parse_pairs(query)
+      query.split("&").filter_map do |pair|
+        key, _, value = pair.partition("=")
+        next if key.empty?
+
+        [unescape(key), unescape(value)]
+      end
+    end
+
+    def scrub_value(key, value)
+      scrubbed = @scrubber.scrub("#{key}=#{value}")
+      prefix = "#{key}="
+      scrubbed.start_with?(prefix) ? scrubbed[prefix.length..] : scrubbed.partition("=").last
+    end
+
+    def unescape(str)
+      decoded = str.tr("+", " ").gsub(/%([0-9a-fA-F]{2})/) { [Regexp.last_match(1)].pack("H2") }
+      decoded.force_encoding(Encoding::UTF_8)
+      decoded.valid_encoding? ? decoded : decoded.force_encoding(Encoding::BINARY)
+    rescue StandardError
+      str
+    end
+
+    # Rebuild the exception with a redacted message, keeping its class and
+    # backtrace so error groupers still group it the same way.
+    def redacted(error)
+      message = error.message
+      return error unless message.is_a?(String)
+
+      clean = @scrubber.scrub(message)
+      return error if clean == message
+
+      replacement = rebuild(error, clean)
+      return error if replacement.nil?
+
+      replacement.set_backtrace(error.backtrace) if error.backtrace
+      replacement
+    end
+
+    # Some exception classes have a non-standard `initialize` and cannot be
+    # rebuilt from a message. Better the original than a crash in the middleware
+    # that was supposed to protect you.
+    def rebuild(error, message)
+      error.exception(message)
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/scrubber/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Scrubber
+  VERSION = "0.1.0"
+end

data/lib/scrubber_rb.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require_relative "scrubber/version"
+
+# Precompiled platform gems ship the extension under a Ruby ABI directory
+# (lib/scrubber_rb/3.3/scrubber_rb.so); a source build puts it one level up.
+begin
+  RUBY_VERSION =~ /(\d+\.\d+)/
+  require_relative "scrubber_rb/#{Regexp.last_match(1)}/scrubber_rb"
+rescue LoadError
+  require_relative "scrubber_rb/scrubber_rb"
+end
+
+require_relative "scrubber/match"
+require_relative "scrubber/instance"
+require_relative "scrubber/log_formatter"
+require_relative "scrubber/middleware"
+require_relative "scrubber/llm_guard"
+
+# Fast PII and secret redaction, with the scanning done in Rust.
+#
+#   Scrubber.scrub("contact nik@example.com")  # => "contact [EMAIL]"
+#
+# The module-level methods are the convenient path and memoize one compiled
+# engine per distinct configuration. For hot loops, build a {Scrubber::Instance}
+# yourself with {Scrubber.new} and keep it around.
+module Scrubber
+  # Detectors enabled unless you say otherwise. Sourced from the Rust registry
+  # so there is exactly one copy of this list in the project.
+  DEFAULTS = Native.default_detectors.map(&:to_sym).freeze
+
+  # Opt-in India pack, for DPDP Act workloads:
+  #
+  #   Scrubber.new(detectors: Scrubber::DEFAULTS + Scrubber::INDIA)
+  INDIA = Native.india_detectors.map(&:to_sym).freeze
+
+  # Every detector this build knows about.
+  ALL = Native.all_detectors.map(&:to_sym).freeze
+
+  class << self
+    # Build a reusable, frozen scrubber. See {Scrubber::Instance}.
+    def new(detectors: DEFAULTS, custom: {}, replacement: :label, hash_salt: nil)
+      Instance.new(
+        detectors: detectors, custom: custom,
+        replacement: replacement, hash_salt: hash_salt
+      )
+    end
+
+    # Redact `text` using a memoized engine for these options.
+    def scrub(text, **options)
+      engine(**options).scrub(text)
+    end
+
+    # Redact `text` in place.
+    def scrub!(text, **options)
+      engine(**options).scrub!(text)
+    end
+
+    # Locate matches without replacing them. Returns {Scrubber::Match} structs.
+    def detect(text, **options)
+      engine(**options).detect(text)
+    end
+
+    # True if anything would be redacted.
+    def match?(text, **options)
+      engine(**options).match?(text)
+    end
+
+    # Stream a file through the engine. Returns bytes written.
+    def scrub_file(input_path, output_path, **options)
+      chunk_size = options.delete(:chunk_size) || Instance::CHUNK_SIZE
+      engine(**options).scrub_file(input_path, output_path, chunk_size: chunk_size)
+    end
+
+    # The memoized {Scrubber::Instance} for a set of options. Exposed because
+    # asking for it explicitly beats accidentally rebuilding one per request.
+    def engine(detectors: DEFAULTS, custom: {}, replacement: :label, hash_salt: nil)
+      key = Instance.cache_key(
+        detectors: detectors, custom: custom,
+        replacement: replacement, hash_salt: hash_salt
+      )
+      # Double-checked: reads are lock-free on the happy path, and building the
+      # same engine twice under a race is harmless (they are identical and
+      # immutable), so the mutex only prevents wasted work.
+      cache[key] || cache_mutex.synchronize do
+        cache[key] ||= new(
+          detectors: detectors, custom: custom,
+          replacement: replacement, hash_salt: hash_salt
+        )
+      end
+    end
+
+    # Drop memoized engines. Only useful in tests.
+    def reset!
+      cache_mutex.synchronize { cache.clear }
+      nil
+    end
+
+    private
+
+    def cache
+      @cache ||= {}
+    end
+
+    def cache_mutex
+      @cache_mutex ||= Mutex.new
+    end
+  end
+end