pikuri-core 0.0.4 → 0.0.6

data/lib/pikuri/extractor.rb

@@ -0,0 +1,314 @@
+# frozen_string_literal: true
+
+module Pikuri
+  # The format→text extraction seam: one registry of extractors that
+  # turn an +IO+ of some recognised format (HTML and plain text out
+  # of the box; PDF / office formats via the pikuri-pdf /
+  # pikuri-extractors plug-in gems) into Markdown-flavoured UTF-8
+  # text, consumed through two front doors:
+  #
+  # * {.extract} — the whole document as one String. The shape the
+  #   indexing / caching callers want ({Pikuri::VectorDb}'s indexer,
+  #   {Tool::WebScrape}'s URL cache): no windowing, no presentation.
+  # * {.extract_paged} — the LLM-tool shape: the same extraction,
+  #   windowed to a line range with a byte cap, returned as a {Page}
+  #   the caller renders. Backs +Workspace::Read+ and
+  #   +VectorDb::Tools::Read+ (via the {FileType} path wrappers) so
+  #   the offset/limit/byte-cap logic lives in one tested place.
+  #
+  # Both front doors — +Tool::Scraper+ dispatching on the HTTP
+  # +Content-Type+ header for the web tools, and {FileType} resolving
+  # local paths — route through this one registry, so both share one
+  # set of format truths and "support a new format" is a registry
+  # entry (pikuri-pdf and pikuri-extractors plug PDF and office
+  # formats in without pikuri-core knowing), not a new special case
+  # in two dispatchers.
+  #
+  # == The extractor duck type
+  #
+  # Each {.registry} entry implements three methods:
+  #
+  # * +matches?(sample:, content_type:)+ → +Boolean+ — claim the
+  #   content. +sample+ is the leading {FileType::SAMPLE_BYTES} bytes
+  #   (for magic-byte sniffs); +content_type+ is the normalized HTTP
+  #   +Content-Type+ for web content, the {FileType.detect_mime}
+  #   result for local files, and may be +nil+ ("no transport
+  #   metadata — sniff if you can").
+  # * +extract(io)+ → +String+ — the whole document as
+  #   Markdown-flavoured UTF-8 text. Raises {Error} on content the
+  #   extractor claimed but cannot parse (malformed PDF, ...).
+  # * +kind+ → +Symbol+ — a short tag (+:text+ / +:pdf+ / +:html+)
+  #   carried on {Page#kind} so rendering callers can word
+  #   format-specific trailers ("End of PDF", the scanned-image
+  #   hint) without re-sniffing.
+  #
+  # plus one *optional* method for formats whose lines can be
+  # produced incrementally:
+  #
+  # * +extract_lines(io)+ → +Enumerator<String>+ — the same content
+  #   as +extract+, as a lazy stream of already-+chomp+ed lines.
+  #   {.extract_paged} prefers this when present and stops consuming
+  #   the moment the window fills, so the rest of the document is
+  #   never parsed (pikuri-pdf's extractor: pdf-reader's page list
+  #   parses on access; {Passthrough}: the IO is read line-by-line).
+  #   The enumerator
+  #   must be consumed while +io+ is still open, and may raise
+  #   {Error} mid-iteration. Extractors that need the whole document
+  #   to produce anything ({HTML}: Readability walks the full DOM —
+  #   true of any subprocess-based extractor too) simply omit it;
+  #   {.extract_paged} then extracts in full and windows the result.
+  #
+  # Windowing itself (offset / limit / byte cap / line truncation) is
+  # presentation and deliberately lives once in {.extract_paged}, not
+  # per extractor — +extract_lines+ is line *production*, the only
+  # genuinely format-specific half of paging.
+  #
+  # == Errors
+  #
+  # Both failure modes are failures the *caller's* LLM can react to,
+  # so they share one rescuable root:
+  #
+  # * {Unsupported} — nothing in {.registry} claimed the content
+  #   (opaque binary, an unhandled content-type).
+  # * {Error} (the root) — an extractor claimed the content but the
+  #   parse failed (malformed PDF, ...).
+  #
+  # Callers map them to their own conventions:
+  # +Tool::Scraper+ re-raises both as +FetchError+;
+  # {FileType.read_as_text} maps {Unsupported} to the +ArgumentError+
+  # binary refusal and {Error} to a +RuntimeError+ carrying the path.
+  module Extractor
+    module_function
+
+    # Raised when an extractor claims content but fails to parse it
+    # (e.g. a malformed PDF). Message is LLM-presentable.
+    Error = Class.new(StandardError)
+
+    # Raised by {.extract} / {.extract_paged} when no registry entry
+    # claims the content. Subclass of {Error} so callers that don't
+    # care about the distinction rescue one class.
+    Unsupported = Class.new(Error)
+
+    # @return [Integer] default line-window size for {.extract_paged}
+    #   when the caller omits +limit+.
+    PAGE_DEFAULT_LIMIT = 2000
+
+    # @return [Integer] default hard byte cap on the content collected
+    #   by a single {.extract_paged} call. Bypassable by paging via
+    #   +offset+. The rendered output is slightly larger (line
+    #   numbering, trailer) — that's the caller's concern.
+    PAGE_MAX_BYTES = 50 * 1024
+
+    # @return [Integer] default per-line character cap;
+    #   {.extract_paged} truncates longer lines and appends
+    #   {PAGE_LINE_TRUNCATION_MARKER}.
+    PAGE_MAX_LINE_LENGTH = 2000
+
+    # @return [String] suffix appended to a line truncated at
+    #   {PAGE_MAX_LINE_LENGTH}.
+    PAGE_LINE_TRUNCATION_MARKER = "... (line truncated to #{PAGE_MAX_LINE_LENGTH} chars)"
+
+    # One windowed slice of a document, returned by {.extract_paged}.
+    # The caller turns this into an observation; this struct carries
+    # everything a trailer needs without the caller re-reading the
+    # document.
+    #
+    # == Fields
+    #
+    # * +lines+ — +Array<String>+, the collected window. Already
+    #   per-line truncated (with {PAGE_LINE_TRUNCATION_MARKER}); *not*
+    #   line-numbered — numbering is presentation the caller adds. For
+    #   a PDF the array includes the +"--- Page N ---"+ marker lines
+    #   pikuri-pdf's extractor emits, which count toward +limit+ / the
+    #   byte cap like any other line.
+    # * +start_line+ — the 1-indexed line number of +lines.first+
+    #   (i.e. the +offset+ the caller asked for). +lines.last+ is at
+    #   +start_line + lines.length - 1+.
+    # * +total_lines+ — total line count of the document when known,
+    #   else +nil+. Known when the read reached EOF, when the format
+    #   was extracted in full (no +extract_lines+ — e.g. HTML), or
+    #   when the lazy stream is cheap enough to count to the end
+    #   (plain text). +nil+ when a lazy stream stopped early — the
+    #   byte cap fired, or a PDF filled the window before its last
+    #   page (counting the rest would mean parsing every page,
+    #   defeating the laziness).
+    # * +more+ — +true+ if content remains past this window (the
+    #   caller should offer +offset = start_line + lines.length+).
+    # * +byte_capped+ — +true+ if the byte cap (not the line limit)
+    #   was the stopping criterion.
+    # * +kind+ — the matched extractor's +kind+ tag (+:text+ /
+    #   +:pdf+ / +:html+); lets the caller word format-specific
+    #   trailers and the empty-document message.
+    #
+    # An empty document yields +lines: []+, +total_lines: 0+; an
+    # +offset+ past EOF yields +lines: []+ with +total_lines+ set to
+    # the real (non-zero) count — the caller distinguishes the two.
+    Page = Data.define(:lines, :start_line, :total_lines, :more, :byte_capped, :kind)
+
+    # The extractor registry, consulted in order — first match wins.
+    # Core ships two entries: {HTML} matches on content-type, and
+    # {Passthrough} is the terminal plain-text arm. A gem adding a
+    # format picks its insertion point by the strength of its claim:
+    # a magic-byte sniff that never misfires on text goes at the
+    # *front* so it beats {HTML}'s content-type match even under a
+    # lying header (+registry.unshift(X)+ — pikuri-pdf does this);
+    # a content-type / weaker-sniff claimer inserts before the
+    # terminal entry (+registry.insert(-2, X)+ — pikuri-extractors
+    # does this).
+    #
+    # @return [Array<#matches?>] mutable, deliberately — this is the
+    #   plug-in seam.
+    def registry
+      @registry ||= [HTML, Passthrough]
+    end
+
+    # Extract the whole document behind +io+ as one Markdown-flavoured
+    # UTF-8 String. May be empty (empty text file, scanned-image PDF
+    # with no extractable text).
+    #
+    # @param io [IO, StringIO] seekable IO positioned at the start of
+    #   the content; this method reads a leading sample for the
+    #   +matches?+ sniff and rewinds before extracting.
+    # @param content_type [String, nil] normalized content-type when
+    #   the transport supplies one (HTTP header, {FileType.detect_mime}
+    #   result); +nil+ when unknown — extractors then rely on their
+    #   byte sniffs.
+    # @return [String]
+    # @raise [Unsupported] when no registry entry claims the content.
+    # @raise [Error] when the matched extractor cannot parse it.
+    def extract(io, content_type: nil)
+      extractor_for(io, content_type).extract(io)
+    end
+
+    # Extract +io+ and return a windowed {Page}: the lines from
+    # +offset+ (1-indexed) up to +limit+ of them, stopping early if
+    # +max_bytes+ is reached, with over-long lines truncated at
+    # +max_line_length+.
+    #
+    # Lazy where the format allows: extractors that implement
+    # +extract_lines+ (plain text; pikuri-pdf's PDF) are consumed
+    # only until the window fills — reading the first window of a
+    # 500-page PDF parses a handful of pages, and the first page of
+    # a gigabyte log never loads it. Extractors without it (HTML) are extracted
+    # in full and then windowed, which is also what makes their
+    # +total_lines+ always exact.
+    #
+    # @param io [IO, StringIO] seekable IO positioned at the start.
+    # @param content_type [String, nil] as for {.extract}.
+    # @param offset [Integer] 1-indexed first line to include. The
+    #   caller is responsible for validating +offset >= 1+.
+    # @param limit [Integer] maximum lines to collect. Caller
+    #   validates +limit >= 1+.
+    # @param max_bytes [Integer] hard byte cap on collected content.
+    # @param max_line_length [Integer] per-line truncation threshold.
+    # @return [Page] the windowed slice.
+    # @raise [Unsupported] when no registry entry claims the content.
+    # @raise [Error] when the matched extractor cannot parse it.
+    def extract_paged(io, content_type: nil, offset: 1, limit: PAGE_DEFAULT_LIMIT,
+                      max_bytes: PAGE_MAX_BYTES, max_line_length: PAGE_MAX_LINE_LENGTH)
+      extractor = extractor_for(io, content_type)
+      if extractor.respond_to?(:extract_lines)
+        # count_tail is a per-format economics call: once the window
+        # fills, counting the rest of a plain-text stream is a cheap
+        # sequential read (so the trailer can say "of N"), while for
+        # a PDF it would mean parsing every remaining page — exactly
+        # what extract_lines exists to avoid. Plugged-in extractors
+        # (pikuri-pdf's included) get the conservative default (stop
+        # early, total unknown).
+        window(extractor.extract_lines(io),
+               offset: offset, limit: limit, max_bytes: max_bytes,
+               max_line_length: max_line_length, kind: extractor.kind,
+               known_total: nil, count_tail: extractor.equal?(Passthrough))
+      else
+        lines = extractor.extract(io).split("\n")
+        window(lines, offset: offset, limit: limit, max_bytes: max_bytes,
+                      max_line_length: max_line_length, kind: extractor.kind,
+                      known_total: lines.length)
+      end
+    end
+
+    # Find the first registry entry claiming +io+'s content: read the
+    # leading {FileType::SAMPLE_BYTES} for the sniff, rewind, and ask
+    # each extractor in order.
+    #
+    # @param io [IO, StringIO] seekable IO positioned at the start.
+    # @param content_type [String, nil]
+    # @return [#extract] the matched extractor.
+    # @raise [Unsupported] when nothing matches.
+    def extractor_for(io, content_type)
+      sample = io.read(FileType::SAMPLE_BYTES) || +''
+      io.rewind
+      registry.find { |ex| ex.matches?(sample: sample, content_type: content_type) } ||
+        raise(Unsupported, 'no extractor for this content' \
+                           "#{content_type && !content_type.empty? ? " (content-type #{content_type.inspect})" : ''}")
+    end
+    private_class_method :extractor_for
+
+    # Collect a {Page} window out of +lines+ (an Array or a lazy
+    # Enumerator of already-+chomp+ed lines). +known_total+ is the
+    # full line count when the caller extracted everything up front
+    # (Array case), +nil+ for a lazy stream — then +total_lines+ is
+    # exact only if the iteration reached EOF: +count_tail+ keeps
+    # the loop counting (without collecting) past the line limit
+    # when consuming the rest of the stream is cheap; without it the
+    # loop breaks and leaves the total unknown. The byte cap always
+    # aborts the count.
+    #
+    # @param lines [Enumerable<String>]
+    # @param known_total [Integer, nil]
+    # @param count_tail [Boolean]
+    # @return [Page]
+    def window(lines, offset:, limit:, max_bytes:, max_line_length:, kind:,
+               known_total:, count_tail: false)
+      start_index   = offset - 1
+      collected     = []
+      seen          = 0
+      bytes         = 0
+      byte_capped   = false
+      more          = false
+      stopped_early = false
+
+      lines.each do |raw|
+        seen += 1
+        next if seen <= start_index
+
+        if collected.length >= limit
+          more = true
+          next if count_tail # keep counting so total_lines stays exact
+
+          stopped_early = true
+          break
+        end
+
+        line = truncate_line(raw, max_line_length)
+        size = line.bytesize + 1 # +1 for the joining newline
+        if bytes + size > max_bytes
+          byte_capped = true
+          more = true
+          stopped_early = true
+          break
+        end
+        collected << line
+        bytes += size
+      end
+
+      Page.new(lines: collected, start_line: offset,
+               total_lines: known_total || (stopped_early ? nil : seen),
+               more: more, byte_capped: byte_capped, kind: kind)
+    end
+    private_class_method :window
+
+    # Truncate +line+ to +max_line_length+ chars, appending
+    # {PAGE_LINE_TRUNCATION_MARKER} when it overflows.
+    #
+    # @param line [String]
+    # @param max_line_length [Integer]
+    # @return [String]
+    def truncate_line(line, max_line_length)
+      return line if line.length <= max_line_length
+
+      line[0, max_line_length] + PAGE_LINE_TRUNCATION_MARKER
+    end
+    private_class_method :truncate_line
+  end
+end

data/lib/pikuri/file_type.rb

@@ -1,10 +1,8 @@
 # frozen_string_literal: true
 
-require 'pdf-reader'
-
 module Pikuri
-  # Magic-byte content sniffing + text extraction, centralised. Three
-  # responsibilities:
+  # Magic-byte content sniffing, plus the path-aware front over the
+  # {Extractor} registry. Two responsibilities:
   #
   # * {.detect_mime} — recognise a file from its leading bytes. Returns
   #   a MIME String for formats pikuri knows how to handle specially
@@ -15,12 +13,16 @@ module Pikuri
   #   {.detect_mime}: a file can be both recognised (e.g. PDF) *and*
   #   binary. {.detect_mime} tells you what the bytes are;
   #   {.binary?} tells you whether they're safe to render as text.
-  # * {.read_as_text} — read a file and return its content as plain
-  #   UTF-8 text. PDFs go through +pdf-reader+ page-by-page; plain
-  #   text passes through; images / binaries / missing files raise.
-  #   The pure-extraction shape consumers like +Pikuri::VectorDb+'s
-  #   indexer want (no LLM-tool concerns — no paging, no line
-  #   numbering, no byte caps; just bytes-in-text-out).
+  #
+  # On top of those sit the two +Pathname+ conveniences,
+  # {.read_as_text} (whole document, the {Pikuri::VectorDb} indexer's
+  # shape) and {.read_as_text_paged} (line-windowed, the Read tools'
+  # shape). Both are thin wrappers: they own the *path-level* refusals
+  # (missing file, directory, image) and the exception mapping, then
+  # hand the opened IO to {Extractor.extract} /
+  # {Extractor.extract_paged} — which format the bytes are and how
+  # they become text is entirely the registry's business, so a gem
+  # plugging a new extractor in extends these wrappers for free.
   #
   # {.detect_mime} and {.binary?} accept either a +String+ of bytes
   # (sample taken by the caller) or a +Pathname+ — when given a path,
@@ -28,8 +30,7 @@ module Pikuri
   # for the sniff itself. The Pathname form is the convenience path;
   # the bytes form is for callers that already have the sample or are
   # calling both methods on the same file and want to avoid a second
-  # open. {.read_as_text} takes a +Pathname+ only — there's no
-  # bytes-in shortcut because the PDF case needs to seek the file.
+  # open.
   #
   # == Why a separate module
   #
@@ -39,8 +40,7 @@ module Pikuri
   # {.binary?} reached for by {Workspace::Edit}. Collecting the
   # detection logic here lets {Read} focus on routing
   # (mime-to-formatter), {Edit} drop its cross-tool reach, and new
-  # tools (a future +Workspace::Diff+, an attachment-aware web fetcher,
-  # ...) share one set of magic-byte truths.
+  # tools share one set of magic-byte truths.
   #
   # == Deliberate non-goals
   #
@@ -136,19 +136,13 @@ module Pikuri
       non_printable.to_f / bytes.bytesize > BINARY_NONPRINTABLE_THRESHOLD
     end
 
-    # Read +path+ and return its content as plain UTF-8 text. Two
-    # extraction paths, picked by {.detect_mime}:
-    #
-    # * **PDF** — walked page-by-page via +pdf-reader+; each page's
-    #   extracted text is stripped and pages are joined with a blank
-    #   line. A scanned-image PDF (no extractable text) comes back as
-    #   the empty String — a deliberate silent skip, callers detect by
-    #   length if they care.
-    # * **Plain text** — anything that {.detect_mime} doesn't
-    #   recognise and that {.binary?} accepts. Read with UTF-8
-    #   encoding; behaviour on non-UTF-8 bytes is whatever +File.read+
-    #   does with +encoding: Encoding::UTF_8+ (which is "leave invalid
-    #   bytes in, let downstream decide").
+    # Read +path+ and return its content as plain UTF-8 text, routed
+    # through the {Extractor} registry: anything
+    # unrecognised-but-textual passes through verbatim
+    # ({Extractor::Passthrough}); with pikuri-pdf registered, PDFs
+    # are extracted with +"--- Page N ---"+ markers (a scanned-image
+    # PDF with no extractable text comes back as the empty String, a
+    # deliberate silent skip callers detect by length if they care).
     #
     # Refusal cases — all raise rather than returning a sentinel
     # because the callers are internal pikuri code, not an LLM
@@ -159,13 +153,11 @@ module Pikuri
     # * Path is a directory → +ArgumentError+.
     # * Image (PNG / JPEG / GIF / WebP per {.detect_mime}) →
     #   +ArgumentError+; images aren't text.
-    # * Binary content (per {.binary?}) and not a recognised MIME →
-    #   +ArgumentError+.
-    # * Malformed PDF — +pdf-reader+'s
-    #   +MalformedPDFError+ / +UnsupportedFeatureError+ /
-    #   +InvalidPageError+ are re-raised as a +RuntimeError+ with the
-    #   path included so callers don't need to know pdf-reader's
-    #   exception hierarchy.
+    # * Content no extractor claims (opaque binary) →
+    #   +ArgumentError+, mapped from {Extractor::Unsupported}.
+    # * Extraction failure (malformed PDF, ...) → +RuntimeError+ with
+    #   the path included, mapped from {Extractor::Error} so callers
+    #   don't need to know any extractor's exception hierarchy.
     #
     # @param path [Pathname] file to read.
     # @return [String] UTF-8 text. May be empty (empty text file, or
@@ -173,40 +165,76 @@ module Pikuri
     # @raise [ArgumentError] if +path+ isn't a +Pathname+, points at
     #   a directory, is an image, or is binary.
     # @raise [Errno::ENOENT] if +path+ doesn't exist.
-    # @raise [RuntimeError] on a malformed / unsupported PDF.
+    # @raise [RuntimeError] on an extraction failure (malformed /
+    #   unsupported PDF, ...).
     def read_as_text(path)
+      mime = guard_extractable(path)
+      path.open('rb') { |io| Extractor.extract(io, content_type: mime) }
+    rescue Extractor::Unsupported
+      raise ArgumentError, "#{path} appears to be binary; cannot extract as text"
+    rescue Extractor::Error => e
+      raise "Cannot extract text from #{path}: #{e.message}"
+    end
+
+    # Extract +path+ and return a windowed {Extractor::Page}: the
+    # lines from +offset+ (1-indexed) up to +limit+ of them, stopping
+    # early if +max_bytes+ is reached, with over-long lines truncated
+    # at +max_line_length+. Same routing and refusal contract as
+    # {.read_as_text}; the windowing semantics (including the lazy
+    # +extract_lines+ consumption that stops parsing once the window
+    # fills) are {Extractor.extract_paged}'s.
+    # The LLM-facing callers map the exceptions into +"Error: ..."+
+    # observations themselves.
+    #
+    # @param path [Pathname] file to read.
+    # @param offset [Integer] 1-indexed first line to include. The
+    #   caller is responsible for validating +offset >= 1+.
+    # @param limit [Integer] maximum lines to collect. Caller
+    #   validates +limit >= 1+.
+    # @param max_bytes [Integer] hard byte cap on collected content.
+    # @param max_line_length [Integer] per-line truncation threshold.
+    # @return [Extractor::Page] the windowed slice.
+    # @raise [ArgumentError] if +path+ isn't a +Pathname+, is a
+    #   directory, an image, or binary.
+    # @raise [Errno::ENOENT] if +path+ doesn't exist.
+    # @raise [RuntimeError] on an extraction failure (malformed /
+    #   unsupported PDF, ...).
+    def read_as_text_paged(path, offset: 1, limit: Extractor::PAGE_DEFAULT_LIMIT,
+                           max_bytes: Extractor::PAGE_MAX_BYTES,
+                           max_line_length: Extractor::PAGE_MAX_LINE_LENGTH)
+      mime = guard_extractable(path)
+      path.open('rb') do |io|
+        Extractor.extract_paged(io, content_type: mime, offset: offset, limit: limit,
+                                    max_bytes: max_bytes, max_line_length: max_line_length)
+      end
+    rescue Extractor::Unsupported
+      raise ArgumentError, "#{path} appears to be binary; cannot extract as text"
+    rescue Extractor::Error => e
+      raise "Cannot extract text from #{path}: #{e.message}"
+    end
+
+    # The shared path-level refusals for {.read_as_text} /
+    # {.read_as_text_paged}: must be an existing non-directory
+    # +Pathname+, and not an image (images are data for a vision
+    # model, never text). Returns the {.detect_mime} result so the
+    # caller can pass it to the {Extractor} as the content-type hint.
+    #
+    # @param path [Pathname]
+    # @return [String, nil] the sniffed MIME type.
+    # @raise [ArgumentError] on a non-Pathname, a directory, or an
+    #   image.
+    # @raise [Errno::ENOENT] if +path+ doesn't exist.
+    def guard_extractable(path)
       raise ArgumentError, "expected Pathname, got #{path.class}" unless path.is_a?(Pathname)
       raise Errno::ENOENT, path.to_s unless path.exist?
       raise ArgumentError, "#{path} is a directory" if path.directory?
 
       mime = detect_mime(path)
-      return read_pdf_text(path) if mime == 'application/pdf'
       raise ArgumentError, "#{path} is an image (#{mime}); cannot extract as text" if mime&.start_with?('image/')
-      raise ArgumentError, "#{path} appears to be binary; cannot extract as text" if binary?(path)
 
-      path.read(encoding: Encoding::UTF_8)
-    end
-
-    # Walk a PDF page-by-page via +pdf-reader+, returning a single
-    # String with non-empty page texts joined by blank lines. Catches
-    # the three +PDF::Reader+ exceptions Workspace::Read also handles
-    # and re-raises them as +RuntimeError+ with the path included.
-    #
-    # @param path [Pathname]
-    # @return [String]
-    # @raise [RuntimeError] on malformed / unsupported PDF.
-    def read_pdf_text(path)
-      pages = path.open('rb') do |io|
-        ::PDF::Reader.new(io).pages.map { |p| p.text.strip }
-      end
-      pages.reject(&:empty?).join("\n\n")
-    rescue ::PDF::Reader::MalformedPDFError,
-           ::PDF::Reader::UnsupportedFeatureError,
-           ::PDF::Reader::InvalidPageError => e
-      raise "Cannot extract PDF text from #{path}: " \
-            "#{e.class.name.split('::').last}: #{e.message}"
+      mime
     end
-    private_class_method :read_pdf_text
+    private_class_method :guard_extractable
 
     # Coerce an +input+ argument into a bytes String for the sniffs.
     # +String+ inputs are returned as-is (caller already sampled);

data/lib/pikuri/finalizers.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Pikuri
+  # Process-global teardown registry: one +at_exit+ for the whole
+  # process, with everything that owns a resource needing orderly
+  # shutdown (agents, {VectorDb::Server::Chroma}, future background
+  # workers) registering here instead of growing its own +at_exit+.
+  # It is {Agent#on_close} promoted from per-agent to per-process —
+  # the same LIFO + per-handler-rescue + idempotent shape, one level
+  # up.
+  #
+  # == Why one chokepoint
+  #
+  # Independent +at_exit+ hooks fire in an order decided by file load
+  # order, which is invisible and fragile. Routing every teardown
+  # through one registry makes the order explicit and controllable:
+  # the SIGTERM-the-strays backstop ({Subprocess.cleanup!}) registers
+  # at *load* time, so it sits at the bottom of the LIFO stack and runs
+  # *last* — after agents and servers (which register at *construction*
+  # time) have closed gracefully, while the subprocess machinery they
+  # shell out to during close (e.g. {VectorDb::Server::Chroma#close}'s
+  # +docker stop+) is still live.
+  #
+  # == Contract
+  #
+  # A registrant MUST respond to +#close+, and +#close+ MUST be
+  # idempotent and tolerant of running at process exit — the host may
+  # also have closed it explicitly earlier. Pass a block instead for
+  # teardown that has no natural +#close+ (e.g.
+  # +Finalizers.register { Pikuri::Subprocess.cleanup! }+).
+  #
+  # == Order: LIFO
+  #
+  # Last registered, first closed — Ruby +ensure+ semantics. A
+  # registrant that depends on an earlier one (a background indexer
+  # writing into {VectorDb::Server::Chroma}) is registered later and so
+  # tears down first. Registration order is therefore dependency order;
+  # register the dependency before its dependents.
+  #
+  # == Errors are contained
+  #
+  # Each +#close+ runs inside its own +rescue+: a raise is logged via
+  # {Pikuri.logger_for} and the sweep continues, so one botched
+  # teardown can't strand the rest. {.run!} drains the registry, so a
+  # second call (an explicit one, then the +at_exit+) closes nothing.
+  module Finalizers
+    # @return [Logger] subsystem logger for contained teardown failures.
+    LOGGER = Pikuri.logger_for('Finalizers')
+
+    # Adapts a teardown block to the +#close+ protocol, so a block and a
+    # closeable object can share one registry.
+    Closer = Struct.new(:block) do
+      # @return [void]
+      def close
+        block.call
+      end
+    end
+
+    @registered = []
+    @mutex      = Mutex.new
+
+    class << self
+      # Register a closeable (or a block) to be torn down at process
+      # exit. Returns the registered handle so the caller can later
+      # {.unregister} it — a resource closed explicitly before exit
+      # should drop out so it can be garbage-collected rather than
+      # pinned alive until the process dies.
+      #
+      # @param closeable [#close, nil] resource to close at exit; omit
+      #   when passing a block
+      # @yield teardown to run at exit, for resources with no +#close+
+      # @return [#close] the registered handle — the object itself, or
+      #   the {Closer} wrapping the block; pass it to {.unregister}
+      # @raise [ArgumentError] if neither an object nor a block is given
+      def register(closeable = nil, &block)
+        unless closeable || block
+          raise ArgumentError, 'Finalizers.register requires an object or a block'
+        end
+
+        handle = closeable || Closer.new(block)
+        @mutex.synchronize { @registered << handle }
+        handle
+      end
+
+      # Drop a previously-registered handle. Idempotent — unregistering
+      # something already gone (or never registered) is a no-op.
+      #
+      # @param handle [#close] the value returned by {.register}
+      # @return [void]
+      def unregister(handle)
+        @mutex.synchronize { @registered.delete(handle) }
+        nil
+      end
+
+      # Close every registrant in LIFO order, each guarded by its own
+      # +rescue+. Wired to +at_exit+ at the bottom of this file.
+      # Draining the registry under the lock makes a repeat call a
+      # no-op and keeps it safe against a concurrent caller.
+      #
+      # @return [void]
+      def run!
+        handles = @mutex.synchronize do
+          taken = @registered.reverse
+          @registered.clear
+          taken
+        end
+
+        handles.each do |handle|
+          handle.close
+        rescue StandardError => e
+          LOGGER.warn("finalizer #{handle.class} raised #{e.class}: #{e.message}")
+        end
+      end
+    end
+  end
+end
+
+at_exit { Pikuri::Finalizers.run! }

data/lib/pikuri/paths.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Pikuri
+  # Standardized on-disk locations for pikuri's local state. Centralizes
+  # the XDG resolution so every component that caches to disk roots under
+  # one place instead of each re-deriving it — currently
+  # {Pikuri::VectorDb::Server::Chroma} (its corpus volume) and
+  # {Pikuri::Memory::Mem0Server} (its mem0 checkout + data volume).
+  module Paths
+    # Pikuri's cache root: +$XDG_CACHE_HOME/pikuri+ when +XDG_CACHE_HOME+
+    # is set and non-empty, else +~/.cache/pikuri+.
+    #
+    # A method, not a frozen constant, on purpose: a constant would
+    # snapshot +XDG_CACHE_HOME+ at +require+ time, which breaks
+    # env-stubbing in tests and ignores a runtime change in a long-lived
+    # process. The directory is *not* created — callers +mkdir_p+ the
+    # subdirectory they need (e.g. +Paths.cache.join('chroma')+,
+    # +Paths.cache.join('mem0')+).
+    #
+    # @return [Pathname] the +<cache home>/pikuri+ directory
+    def self.cache
+      home = ENV['XDG_CACHE_HOME']
+      home = File.expand_path('~/.cache') if home.nil? || home.empty?
+      Pathname.new(home).join('pikuri')
+    end
+  end
+end