pikuri-core 0.0.5 → 0.0.7

data/lib/pikuri/file_type.rb

@@ -1,10 +1,8 @@
 # frozen_string_literal: true
 
-require 'pdf-reader'
-
 module Pikuri
-  # Magic-byte content sniffing + text extraction, centralised. Three
-  # responsibilities:
+  # Magic-byte content sniffing, plus the path-aware front over the
+  # {Extractor} registry. Two responsibilities:
   #
   # * {.detect_mime} — recognise a file from its leading bytes. Returns
   #   a MIME String for formats pikuri knows how to handle specially
@@ -15,21 +13,16 @@ module Pikuri
   #   {.detect_mime}: a file can be both recognised (e.g. PDF) *and*
   #   binary. {.detect_mime} tells you what the bytes are;
   #   {.binary?} tells you whether they're safe to render as text.
-  # * {.read_as_text} — read a file and return its content as plain
-  #   UTF-8 text. PDFs go through +pdf-reader+ page-by-page; plain
-  #   text passes through; images / binaries / missing files raise.
-  #   The pure-extraction shape consumers like +Pikuri::VectorDb+'s
-  #   indexer want (no LLM-tool concerns — no paging, no line
-  #   numbering, no byte caps; just bytes-in-text-out).
-  # * {.read_as_text_paged} — the LLM-tool shape: the same
-  #   extraction as {.read_as_text}, but lazily windowed to a
-  #   line range with a byte cap, returning a {Page} value the
-  #   caller renders. Shared by +Workspace::Read+ and
-  #   +VectorDb::Tools::Read+ so the offset/limit/byte-cap windowing lives
-  #   in one tested place; each tool keeps its own presentation
-  #   (cat-n numbering, trailer wording, citation vs. path). Same
-  #   refusal contract as {.read_as_text} (raises on image / binary
-  #   / missing / malformed-PDF).
+  #
+  # On top of those sit the two +Pathname+ conveniences,
+  # {.read_as_text} (whole document, the {Pikuri::VectorDb} indexer's
+  # shape) and {.read_as_text_paged} (line-windowed, the Read tools'
+  # shape). Both are thin wrappers: they own the *path-level* refusals
+  # (missing file, directory, image) and the exception mapping, then
+  # hand the opened IO to {Extractor.extract} /
+  # {Extractor.extract_paged} — which format the bytes are and how
+  # they become text is entirely the registry's business, so a gem
+  # plugging a new extractor in extends these wrappers for free.
   #
   # {.detect_mime} and {.binary?} accept either a +String+ of bytes
   # (sample taken by the caller) or a +Pathname+ — when given a path,
@@ -37,8 +30,7 @@ module Pikuri
   # for the sniff itself. The Pathname form is the convenience path;
   # the bytes form is for callers that already have the sample or are
   # calling both methods on the same file and want to avoid a second
-  # open. {.read_as_text} takes a +Pathname+ only — there's no
-  # bytes-in shortcut because the PDF case needs to seek the file.
+  # open.
   #
   # == Why a separate module
   #
@@ -48,8 +40,7 @@ module Pikuri
   # {.binary?} reached for by {Workspace::Edit}. Collecting the
   # detection logic here lets {Read} focus on routing
   # (mime-to-formatter), {Edit} drop its cross-tool reach, and new
-  # tools (a future +Workspace::Diff+, an attachment-aware web fetcher,
-  # ...) share one set of magic-byte truths.
+  # tools share one set of magic-byte truths.
   #
   # == Deliberate non-goals
   #
@@ -94,58 +85,6 @@ module Pikuri
     #   with this five-byte ASCII sequence per ISO 32000-1 §7.5.2.
     PDF_MAGIC = '%PDF-'
 
-    # @return [Integer] default line-window size for
-    #   {.read_as_text_paged} when the caller omits +limit+.
-    PAGE_DEFAULT_LIMIT = 2000
-
-    # @return [Integer] default hard byte cap on the content collected
-    #   by a single {.read_as_text_paged} call. Bypassable by paging
-    #   via +offset+. The rendered output is slightly larger (line
-    #   numbering, trailer) — that's the caller's concern.
-    PAGE_MAX_BYTES = 50 * 1024
-
-    # @return [Integer] default per-line character cap;
-    #   {.read_as_text_paged} truncates longer lines and appends
-    #   {PAGE_LINE_TRUNCATION_MARKER}.
-    PAGE_MAX_LINE_LENGTH = 2000
-
-    # @return [String] suffix appended to a line truncated at
-    #   {PAGE_MAX_LINE_LENGTH}.
-    PAGE_LINE_TRUNCATION_MARKER = "... (line truncated to #{PAGE_MAX_LINE_LENGTH} chars)"
-
-    # One windowed slice of a document, returned by
-    # {.read_as_text_paged}. The caller turns this into an
-    # observation; this struct carries everything a trailer needs
-    # without the caller re-reading the file.
-    #
-    # == Fields
-    #
-    # * +lines+ — +Array<String>+, the collected window. Already
-    #   per-line truncated (with {PAGE_LINE_TRUNCATION_MARKER}); *not*
-    #   line-numbered — numbering is presentation the caller adds. For
-    #   a PDF the array includes +"--- Page N ---"+ marker lines (one
-    #   per page that contributed text), which count toward +limit+ /
-    #   the byte cap like any other line.
-    # * +start_line+ — the 1-indexed line number of +lines.first+
-    #   (i.e. the +offset+ the caller asked for). +lines.last+ is at
-    #   +start_line + lines.length - 1+.
-    # * +total_lines+ — total line count of the document when known,
-    #   else +nil+. Known when extraction reached EOF (so the caller
-    #   can print "of N"); +nil+ when the read stopped early — the
-    #   byte cap fired, or a PDF filled the window before its last
-    #   page (counting the rest would defeat the laziness).
-    # * +more+ — +true+ if content remains past this window (the
-    #   caller should offer +offset = start_line + lines.length+).
-    # * +byte_capped+ — +true+ if {PAGE_MAX_BYTES} (not the line
-    #   limit) was the stopping criterion.
-    # * +kind+ — +:text+ or +:pdf+; lets the caller word PDF-specific
-    #   trailers and the empty-document message.
-    #
-    # An empty document yields +lines: []+, +total_lines: 0+; an
-    # +offset+ past EOF yields +lines: []+ with +total_lines+ set to
-    # the real (non-zero) count — the caller distinguishes the two.
-    Page = Data.define(:lines, :start_line, :total_lines, :more, :byte_capped, :kind)
-
     # Recognise a file from its leading bytes. Returns the MIME type
     # as a String for formats pikuri handles specially, or +nil+ for
     # "unrecognised" — callers interpret +nil+ themselves (text,
@@ -197,19 +136,13 @@ module Pikuri
       non_printable.to_f / bytes.bytesize > BINARY_NONPRINTABLE_THRESHOLD
     end
 
-    # Read +path+ and return its content as plain UTF-8 text. Two
-    # extraction paths, picked by {.detect_mime}:
-    #
-    # * **PDF** — walked page-by-page via +pdf-reader+; each page's
-    #   extracted text is stripped and pages are joined with a blank
-    #   line. A scanned-image PDF (no extractable text) comes back as
-    #   the empty String — a deliberate silent skip, callers detect by
-    #   length if they care.
-    # * **Plain text** — anything that {.detect_mime} doesn't
-    #   recognise and that {.binary?} accepts. Read with UTF-8
-    #   encoding; behaviour on non-UTF-8 bytes is whatever +File.read+
-    #   does with +encoding: Encoding::UTF_8+ (which is "leave invalid
-    #   bytes in, let downstream decide").
+    # Read +path+ and return its content as plain UTF-8 text, routed
+    # through the {Extractor} registry: anything
+    # unrecognised-but-textual passes through verbatim
+    # ({Extractor::Passthrough}); with pikuri-pdf registered, PDFs
+    # are extracted with +"--- Page N ---"+ markers (a scanned-image
+    # PDF with no extractable text comes back as the empty String, a
+    # deliberate silent skip callers detect by length if they care).
     #
     # Refusal cases — all raise rather than returning a sentinel
     # because the callers are internal pikuri code, not an LLM
@@ -220,13 +153,11 @@ module Pikuri
     # * Path is a directory → +ArgumentError+.
     # * Image (PNG / JPEG / GIF / WebP per {.detect_mime}) →
     #   +ArgumentError+; images aren't text.
-    # * Binary content (per {.binary?}) and not a recognised MIME →
-    #   +ArgumentError+.
-    # * Malformed PDF — +pdf-reader+'s
-    #   +MalformedPDFError+ / +UnsupportedFeatureError+ /
-    #   +InvalidPageError+ are re-raised as a +RuntimeError+ with the
-    #   path included so callers don't need to know pdf-reader's
-    #   exception hierarchy.
+    # * Content no extractor claims (opaque binary) →
+    #   +ArgumentError+, mapped from {Extractor::Unsupported}.
+    # * Extraction failure (malformed PDF, ...) → +RuntimeError+ with
+    #   the path included, mapped from {Extractor::Error} so callers
+    #   don't need to know any extractor's exception hierarchy.
     #
     # @param path [Pathname] file to read.
     # @return [String] UTF-8 text. May be empty (empty text file, or
@@ -234,56 +165,26 @@ module Pikuri
     # @raise [ArgumentError] if +path+ isn't a +Pathname+, points at
     #   a directory, is an image, or is binary.
     # @raise [Errno::ENOENT] if +path+ doesn't exist.
-    # @raise [RuntimeError] on a malformed / unsupported PDF.
+    # @raise [RuntimeError] on an extraction failure (malformed /
+    #   unsupported PDF, ...).
     def read_as_text(path)
-      raise ArgumentError, "expected Pathname, got #{path.class}" unless path.is_a?(Pathname)
-      raise Errno::ENOENT, path.to_s unless path.exist?
-      raise ArgumentError, "#{path} is a directory" if path.directory?
-
-      mime = detect_mime(path)
-      return read_pdf_text(path) if mime == 'application/pdf'
-      raise ArgumentError, "#{path} is an image (#{mime}); cannot extract as text" if mime&.start_with?('image/')
-      raise ArgumentError, "#{path} appears to be binary; cannot extract as text" if binary?(path)
-
-      path.read(encoding: Encoding::UTF_8)
+      mime = guard_extractable(path)
+      path.open('rb') { |io| Extractor.extract(io, content_type: mime) }
+    rescue Extractor::Unsupported
+      raise ArgumentError, "#{path} appears to be binary; cannot extract as text"
+    rescue Extractor::Error => e
+      raise "Cannot extract text from #{path}: #{e.message}"
     end
 
-    # Walk a PDF page-by-page via +pdf-reader+, returning a single
-    # String with non-empty page texts joined by blank lines. Catches
-    # the three +PDF::Reader+ exceptions Workspace::Read also handles
-    # and re-raises them as +RuntimeError+ with the path included.
-    #
-    # @param path [Pathname]
-    # @return [String]
-    # @raise [RuntimeError] on malformed / unsupported PDF.
-    def read_pdf_text(path)
-      pages = path.open('rb') do |io|
-        ::PDF::Reader.new(io).pages.map { |p| p.text.strip }
-      end
-      pages.reject(&:empty?).join("\n\n")
-    rescue ::PDF::Reader::MalformedPDFError,
-           ::PDF::Reader::UnsupportedFeatureError,
-           ::PDF::Reader::InvalidPageError => e
-      raise "Cannot extract PDF text from #{path}: " \
-            "#{e.class.name.split('::').last}: #{e.message}"
-    end
-    private_class_method :read_pdf_text
-
-    # Extract +path+ as text and return a windowed {Page}: the lines
-    # from +offset+ (1-indexed) up to +limit+ of them, stopping early
-    # if +max_bytes+ is reached, with over-long lines truncated at
-    # +max_line_length+. Lazy by design — a text file is streamed
-    # line-by-line and a PDF is parsed page-by-page only until the
-    # window fills, so reading the first page of a 500-page PDF parses
-    # a handful of pages, not all of them.
-    #
-    # Same routing and refusal contract as {.read_as_text}: PDFs are
-    # extracted (with +"--- Page N ---"+ marker lines, unlike
-    # {.read_as_text}'s marker-free join — paging is a display path,
-    # the marker-free form stays the indexing path); images, binaries,
-    # directories, missing files, and malformed PDFs all raise rather
-    # than returning a sentinel. The LLM-facing callers map those into
-    # +"Error: ..."+ observations themselves.
+    # Extract +path+ and return a windowed {Extractor::Page}: the
+    # lines from +offset+ (1-indexed) up to +limit+ of them, stopping
+    # early if +max_bytes+ is reached, with over-long lines truncated
+    # at +max_line_length+. Same routing and refusal contract as
+    # {.read_as_text}; the windowing semantics (including the lazy
+    # +extract_lines+ consumption that stops parsing once the window
+    # fills) are {Extractor.extract_paged}'s.
+    # The LLM-facing callers map the exceptions into +"Error: ..."+
+    # observations themselves.
     #
     # @param path [Pathname] file to read.
     # @param offset [Integer] 1-indexed first line to include. The
@@ -292,141 +193,48 @@ module Pikuri
     #   validates +limit >= 1+.
     # @param max_bytes [Integer] hard byte cap on collected content.
     # @param max_line_length [Integer] per-line truncation threshold.
-    # @return [Page] the windowed slice.
+    # @return [Extractor::Page] the windowed slice.
     # @raise [ArgumentError] if +path+ isn't a +Pathname+, is a
     #   directory, an image, or binary.
     # @raise [Errno::ENOENT] if +path+ doesn't exist.
-    # @raise [RuntimeError] on a malformed / unsupported PDF.
-    def read_as_text_paged(path, offset: 1, limit: PAGE_DEFAULT_LIMIT,
-                           max_bytes: PAGE_MAX_BYTES, max_line_length: PAGE_MAX_LINE_LENGTH)
+    # @raise [RuntimeError] on an extraction failure (malformed /
+    #   unsupported PDF, ...).
+    def read_as_text_paged(path, offset: 1, limit: Extractor::PAGE_DEFAULT_LIMIT,
+                           max_bytes: Extractor::PAGE_MAX_BYTES,
+                           max_line_length: Extractor::PAGE_MAX_LINE_LENGTH)
+      mime = guard_extractable(path)
+      path.open('rb') do |io|
+        Extractor.extract_paged(io, content_type: mime, offset: offset, limit: limit,
+                                    max_bytes: max_bytes, max_line_length: max_line_length)
+      end
+    rescue Extractor::Unsupported
+      raise ArgumentError, "#{path} appears to be binary; cannot extract as text"
+    rescue Extractor::Error => e
+      raise "Cannot extract text from #{path}: #{e.message}"
+    end
+
+    # The shared path-level refusals for {.read_as_text} /
+    # {.read_as_text_paged}: must be an existing non-directory
+    # +Pathname+, and not an image (images are data for a vision
+    # model, never text). Returns the {.detect_mime} result so the
+    # caller can pass it to the {Extractor} as the content-type hint.
+    #
+    # @param path [Pathname]
+    # @return [String, nil] the sniffed MIME type.
+    # @raise [ArgumentError] on a non-Pathname, a directory, or an
+    #   image.
+    # @raise [Errno::ENOENT] if +path+ doesn't exist.
+    def guard_extractable(path)
       raise ArgumentError, "expected Pathname, got #{path.class}" unless path.is_a?(Pathname)
       raise Errno::ENOENT, path.to_s unless path.exist?
       raise ArgumentError, "#{path} is a directory" if path.directory?
 
       mime = detect_mime(path)
-      if mime == 'application/pdf'
-        return paged_pdf(path, offset: offset, limit: limit,
-                               max_bytes: max_bytes, max_line_length: max_line_length)
-      end
       raise ArgumentError, "#{path} is an image (#{mime}); cannot extract as text" if mime&.start_with?('image/')
-      raise ArgumentError, "#{path} appears to be binary; cannot extract as text" if binary?(path)
-
-      paged_text(path, offset: offset, limit: limit,
-                       max_bytes: max_bytes, max_line_length: max_line_length)
-    end
-
-    # Stream a text file line-by-line into a {Page}. Keeps counting
-    # lines past the collection window so +total_lines+ can report the
-    # real total when the line limit (not the byte cap) stopped
-    # collection; on the byte cap it breaks and leaves +total_lines+
-    # +nil+ (the rest of the file is never read).
-    #
-    # @return [Page] +kind: :text+.
-    def paged_text(path, offset:, limit:, max_bytes:, max_line_length:)
-      start_index = offset - 1
-      collected   = []
-      total_lines = 0
-      bytes       = 0
-      byte_capped = false
-      more        = false
-
-      path.each_line do |raw|
-        total_lines += 1
-        next if total_lines <= start_index
-
-        if collected.length >= limit
-          more = true
-          next
-        end
-
-        line = truncate_line(raw.chomp, max_line_length)
-        size = line.bytesize + 1 # +1 for the joining newline
-        if bytes + size > max_bytes
-          byte_capped = true
-          more = true
-          break
-        end
-        collected << line
-        bytes += size
-      end
-
-      Page.new(lines: collected, start_line: offset,
-               total_lines: byte_capped ? nil : total_lines,
-               more: more, byte_capped: byte_capped, kind: :text)
-    end
-    private_class_method :paged_text
-
-    # PDF counterpart to {paged_text}: walk +pdf-reader+'s lazy page
-    # iterator, emitting a +"--- Page N ---"+ header line then each
-    # line of the page's text, applying the same offset / limit /
-    # byte-cap contract. The +throw :done+ short-circuits both loops
-    # the moment the window fills, so parsing stops — which is why a
-    # PDF that stops early can't report +total_lines+ (it would have
-    # to parse every page to count).
-    #
-    # @return [Page] +kind: :pdf+.
-    # @raise [RuntimeError] on a malformed / unsupported PDF.
-    def paged_pdf(path, offset:, limit:, max_bytes:, max_line_length:)
-      start_index = offset - 1
-      collected   = []
-      total_lines = 0
-      bytes       = 0
-      byte_capped = false
-      more        = false
-
-      catch(:done) do
-        path.open('rb') do |io|
-          reader = ::PDF::Reader.new(io)
-          reader.pages.each_with_index do |page, idx|
-            text = page.text.strip
-            next if text.empty?
-
-            ["--- Page #{idx + 1} ---", *text.split("\n")].each do |raw|
-              total_lines += 1
-              next if total_lines <= start_index
-
-              if collected.length >= limit
-                more = true
-                throw :done
-              end
-
-              line = truncate_line(raw, max_line_length)
-              size = line.bytesize + 1
-              if bytes + size > max_bytes
-                byte_capped = true
-                more = true
-                throw :done
-              end
-              collected << line
-              bytes += size
-            end
-          end
-        end
-      end
-
-      Page.new(lines: collected, start_line: offset,
-               total_lines: more ? nil : total_lines,
-               more: more, byte_capped: byte_capped, kind: :pdf)
-    rescue ::PDF::Reader::MalformedPDFError,
-           ::PDF::Reader::InvalidPageError,
-           ::PDF::Reader::UnsupportedFeatureError => e
-      raise "Cannot extract PDF text from #{path}: " \
-            "#{e.class.name.split('::').last}: #{e.message}"
-    end
-    private_class_method :paged_pdf
-
-    # Truncate +line+ to +max_line_length+ chars, appending
-    # {PAGE_LINE_TRUNCATION_MARKER} when it overflows.
-    #
-    # @param line [String]
-    # @param max_line_length [Integer]
-    # @return [String]
-    def truncate_line(line, max_line_length)
-      return line if line.length <= max_line_length
 
-      line[0, max_line_length] + PAGE_LINE_TRUNCATION_MARKER
+      mime
     end
-    private_class_method :truncate_line
+    private_class_method :guard_extractable
 
     # Coerce an +input+ argument into a bytes String for the sniffs.
     # +String+ inputs are returned as-is (caller already sampled);

data/lib/pikuri/sanitizer.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+module Pikuri
+  # Renders attacker-controlled text safe to display, and reports *why*
+  # it was unsafe.
+  #
+  # Every string an LLM composes is untrusted: a bash command, a tool
+  # observation echoed back to the user, a description it wrote for a
+  # confirmation prompt. A model that is broken — or, far more likely,
+  # being driven by a prompt injection — can embed bytes that a terminal
+  # acts on rather than prints: a carriage return that overwrites the
+  # line the user just read, an ESC that recolors or repositions, a
+  # backspace that erases, a bidirectional override that reorders text so
+  # it reads differently than it runs, a zero-width character that hides
+  # in plain sight, or a Cyrillic +а+ masquerading as a Latin +a+. The
+  # whole point of a confirmation prompt collapses if the bytes the user
+  # approves are not the bytes that execute.
+  #
+  # {.sanitize} is the one chrome-independent primitive every renderer
+  # (terminal, TUI, web) routes through. It does two things and returns
+  # both as a {Result}:
+  #
+  # 1. *Neutralize* — make the dangerous bytes visible without changing
+  #    structure. Control bytes become +\xNN+, bidi/zero-width codepoints
+  #    become +\u{NNNN}+, tab becomes +\t+. Newlines are preserved
+  #    (multi-line commands are normal). This is *faithful, not
+  #    beautifying*: it never collapses runs of whitespace or rewrites a
+  #    tab to a space, because the user must see exactly what they are
+  #    approving — a Makefile's leading tab stays visibly a tab. A web
+  #    chrome composes +html_escape(sanitize(s).text)+; the HTML layer is
+  #    the caller's, not ours.
+  # 2. *Warn* — return a {Warning} per category detected, each a semantic
+  #    record (kind + offending tokens + a plain-English explanation).
+  #    Presentation is the chrome's: a terminal renders these bold yellow,
+  #    a web client a banner. The {Warning} carries no color or markup.
+  #
+  # == Scope (deliberately closed)
+  #
+  # Detection covers the *invisibility / cursor-control / reordering*
+  # attack classes completely, because each is a finite, enumerable set
+  # of codepoints: C0 controls, C1 controls (a second ANSI introducer on
+  # some emulators), DEL, the bidi overrides, and the zero-width
+  # characters. On top of that, {.sanitize} flags *mixed-script tokens* —
+  # a single word combining letters from Latin + Cyrillic + Greek, which
+  # is the signature of a homoglyph spoof and has near-zero false
+  # positives on real text (humans do not weld two alphabets inside one
+  # word; +café+ is all-Latin, +Москва+ all-Cyrillic, only +Pаypal+ mixes).
+  #
+  # Two confusable classes are explicitly *out of scope*, because
+  # detecting them needs Unicode confusables tables and produces heavy
+  # false positives on legitimate multilingual text:
+  #
+  # * *Whole-script* homoglyphs — an entirely-Cyrillic string that merely
+  #   looks Latin (no mixing to detect).
+  # * *Single-symbol* confusables — the Greek question mark +;+ (U+037E)
+  #   that looks like a semicolon, full-width forms, the division slash.
+  #
+  # "Solid" here means complete on the classes above, not exhaustive over
+  # all of Unicode.
+  module Sanitizer
+    # One reason a piece of text was flagged, ready for a chrome to
+    # render however it surfaces warnings (bold yellow line, web banner).
+    #
+    # * +kind+ — a {Symbol} category: +:backspace+, +:control_bytes+,
+    #   +:bidi+, +:zero_width+, or +:mixed_script+.
+    # * +offenders+ — the distinct offending tokens, in first-seen order:
+    #   the escaped forms (+"\\x1b"+, +"\\u{202e}"+) for byte categories,
+    #   the raw tokens (+"Pаypal"+) for +:mixed_script+.
+    # * +explanation+ — a one-line, chrome-agnostic English summary of
+    #   what the bytes can do.
+    Warning = Data.define(:kind, :offenders, :explanation)
+
+    # The output of {Sanitizer.sanitize}.
+    #
+    # * +text+ — the neutralized string, safe to print literally.
+    # * +warnings+ — {Array}<{Warning}>, empty when nothing was flagged.
+    Result = Data.define(:text, :warnings)
+
+    # Bidirectional-override codepoints: the explicit LRO/RLO/PDF/LRE/RLE
+    # set plus the isolate set (LRI/RLI/FSI/PDI). Reordering attacks.
+    BIDI_OVERRIDES = [*0x202a..0x202e, *0x2066..0x2069].freeze
+
+    # Zero-width and invisible codepoints: ZWSP, ZWNJ, ZWJ, and the BOM /
+    # zero-width no-break space.
+    ZERO_WIDTH = [0x200b, 0x200c, 0x200d, 0xfeff].freeze
+
+    # Codepoints {.sanitize} rewrites: C0 controls including tab (U+0009)
+    # but *excluding* newline (U+000A, which passes through untouched),
+    # C1 controls + DEL (U+007F–009F), the zero-width set, and the bidi
+    # overrides. Newline is the one control character a faithful render
+    # must keep, so the C0 range is split around it.
+    SUSPECT = /[\u0000-\u0009\u000b-\u001f\u007f-\u009f\u200b-\u200d\u202a-\u202e\u2066-\u2069\ufeff]/
+
+    # The three Latin-confusable scripts whose mixing inside one token
+    # signals a homoglyph spoof. Punctuation, digits and spaces are the
+    # +Common+ script and match none of these, so they never count toward
+    # the "two distinct scripts" threshold.
+    CONFUSABLE_SCRIPTS = { 'Latin' => /\p{Latin}/, 'Cyrillic' => /\p{Cyrillic}/, 'Greek' => /\p{Greek}/ }.freeze
+
+    # Neutralize +text+ for literal display and report what was flagged.
+    #
+    # @param text [String] attacker-controlled text (an LLM-composed
+    #   command, description, or tool observation), e.g.
+    #   +"echo hi\rrm -rf /"+
+    # @return [Result] the neutralized +text+ plus an {Array}<{Warning}>
+    #   (empty when clean)
+    def self.sanitize(text)
+      backspace  = false
+      control    = []
+      bidi       = []
+      zero_width = []
+
+      clean = text.gsub(SUSPECT) do |ch|
+        cp = ch.ord
+        if cp == 0x09
+          '\\t'
+        elsif cp == 0x08
+          backspace = true
+          '\\x08'
+        elsif BIDI_OVERRIDES.include?(cp)
+          format('\\u{%04x}', cp).tap { |t| bidi << t }
+        elsif ZERO_WIDTH.include?(cp)
+          format('\\u{%04x}', cp).tap { |t| zero_width << t }
+        else
+          format('\\x%02x', cp).tap { |t| control << t }
+        end
+      end
+
+      Result.new(text: clean, warnings: warnings_for(backspace, control, bidi, zero_width, mixed_script_tokens(text)))
+    end
+
+    # Tokens (whitespace-delimited runs) that combine letters from two or
+    # more of {CONFUSABLE_SCRIPTS} — the homoglyph-spoof signature.
+    #
+    # @param text [String]
+    # @return [Array<String>] distinct offending tokens, first-seen order
+    def self.mixed_script_tokens(text)
+      text.split(/\s+/).reject(&:empty?).select do |token|
+        CONFUSABLE_SCRIPTS.count { |_name, re| token.match?(re) } >= 2
+      end.uniq
+    end
+
+    # Assemble one {Warning} per non-empty category, in a stable order
+    # (most-deceptive first).
+    #
+    # @return [Array<Warning>]
+    def self.warnings_for(backspace, control, bidi, zero_width, mixed)
+      out = []
+      if backspace
+        out << Warning.new(kind: :backspace, offenders: ['\\x08'],
+                           explanation: 'Backspace characters present — the model may be trying to visually erase ' \
+                                        'part of the text after you have read it.')
+      end
+      unless bidi.empty?
+        out << Warning.new(kind: :bidi, offenders: bidi.uniq,
+                           explanation: "Bidirectional-override characters present (#{bidi.uniq.join(' ')}) — these " \
+                                        'can reorder how text is displayed so it reads differently than it runs.')
+      end
+      unless zero_width.empty?
+        out << Warning.new(kind: :zero_width, offenders: zero_width.uniq,
+                           explanation: "Zero-width / invisible characters present (#{zero_width.uniq.join(' ')}) — " \
+                                        'the text may contain characters you cannot see.')
+      end
+      unless control.empty?
+        out << Warning.new(kind: :control_bytes, offenders: control.uniq,
+                           explanation: "Non-printable control bytes present (#{control.uniq.join(' ')}) — in a " \
+                                        'terminal these can move the cursor, change colors, or hide output.')
+      end
+      unless mixed.empty?
+        out << Warning.new(kind: :mixed_script, offenders: mixed,
+                           explanation: "Mixed-script tokens present (#{mixed.join(', ')}) — letters from different " \
+                                        "alphabets are combined within one word, a classic homoglyph spoof (e.g. " \
+                                        "Cyrillic 'а' standing in for Latin 'a').")
+      end
+      out
+    end
+    private_class_method :warnings_for
+  end
+end