pikuri-core 0.0.5 → 0.0.6

data/lib/pikuri/file_type.rb

@@ -1,10 +1,8 @@
 # frozen_string_literal: true
 
-require 'pdf-reader'
-
 module Pikuri
-  # Magic-byte content sniffing + text extraction, centralised. Three
-  # responsibilities:
+  # Magic-byte content sniffing, plus the path-aware front over the
+  # {Extractor} registry. Two responsibilities:
   #
   # * {.detect_mime} — recognise a file from its leading bytes. Returns
   #   a MIME String for formats pikuri knows how to handle specially
@@ -15,21 +13,16 @@ module Pikuri
   #   {.detect_mime}: a file can be both recognised (e.g. PDF) *and*
   #   binary. {.detect_mime} tells you what the bytes are;
   #   {.binary?} tells you whether they're safe to render as text.
-  # * {.read_as_text} — read a file and return its content as plain
-  #   UTF-8 text. PDFs go through +pdf-reader+ page-by-page; plain
-  #   text passes through; images / binaries / missing files raise.
-  #   The pure-extraction shape consumers like +Pikuri::VectorDb+'s
-  #   indexer want (no LLM-tool concerns — no paging, no line
-  #   numbering, no byte caps; just bytes-in-text-out).
-  # * {.read_as_text_paged} — the LLM-tool shape: the same
-  #   extraction as {.read_as_text}, but lazily windowed to a
-  #   line range with a byte cap, returning a {Page} value the
-  #   caller renders. Shared by +Workspace::Read+ and
-  #   +VectorDb::Tools::Read+ so the offset/limit/byte-cap windowing lives
-  #   in one tested place; each tool keeps its own presentation
-  #   (cat-n numbering, trailer wording, citation vs. path). Same
-  #   refusal contract as {.read_as_text} (raises on image / binary
-  #   / missing / malformed-PDF).
+  #
+  # On top of those sit the two +Pathname+ conveniences,
+  # {.read_as_text} (whole document, the {Pikuri::VectorDb} indexer's
+  # shape) and {.read_as_text_paged} (line-windowed, the Read tools'
+  # shape). Both are thin wrappers: they own the *path-level* refusals
+  # (missing file, directory, image) and the exception mapping, then
+  # hand the opened IO to {Extractor.extract} /
+  # {Extractor.extract_paged} — which format the bytes are and how
+  # they become text is entirely the registry's business, so a gem
+  # plugging a new extractor in extends these wrappers for free.
   #
   # {.detect_mime} and {.binary?} accept either a +String+ of bytes
   # (sample taken by the caller) or a +Pathname+ — when given a path,
@@ -37,8 +30,7 @@ module Pikuri
   # for the sniff itself. The Pathname form is the convenience path;
   # the bytes form is for callers that already have the sample or are
   # calling both methods on the same file and want to avoid a second
-  # open. {.read_as_text} takes a +Pathname+ only — there's no
-  # bytes-in shortcut because the PDF case needs to seek the file.
+  # open.
   #
   # == Why a separate module
   #
@@ -48,8 +40,7 @@ module Pikuri
   # {.binary?} reached for by {Workspace::Edit}. Collecting the
   # detection logic here lets {Read} focus on routing
   # (mime-to-formatter), {Edit} drop its cross-tool reach, and new
-  # tools (a future +Workspace::Diff+, an attachment-aware web fetcher,
-  # ...) share one set of magic-byte truths.
+  # tools share one set of magic-byte truths.
   #
   # == Deliberate non-goals
   #
@@ -94,58 +85,6 @@ module Pikuri
     #   with this five-byte ASCII sequence per ISO 32000-1 §7.5.2.
     PDF_MAGIC = '%PDF-'
 
-    # @return [Integer] default line-window size for
-    #   {.read_as_text_paged} when the caller omits +limit+.
-    PAGE_DEFAULT_LIMIT = 2000
-
-    # @return [Integer] default hard byte cap on the content collected
-    #   by a single {.read_as_text_paged} call. Bypassable by paging
-    #   via +offset+. The rendered output is slightly larger (line
-    #   numbering, trailer) — that's the caller's concern.
-    PAGE_MAX_BYTES = 50 * 1024
-
-    # @return [Integer] default per-line character cap;
-    #   {.read_as_text_paged} truncates longer lines and appends
-    #   {PAGE_LINE_TRUNCATION_MARKER}.
-    PAGE_MAX_LINE_LENGTH = 2000
-
-    # @return [String] suffix appended to a line truncated at
-    #   {PAGE_MAX_LINE_LENGTH}.
-    PAGE_LINE_TRUNCATION_MARKER = "... (line truncated to #{PAGE_MAX_LINE_LENGTH} chars)"
-
-    # One windowed slice of a document, returned by
-    # {.read_as_text_paged}. The caller turns this into an
-    # observation; this struct carries everything a trailer needs
-    # without the caller re-reading the file.
-    #
-    # == Fields
-    #
-    # * +lines+ — +Array<String>+, the collected window. Already
-    #   per-line truncated (with {PAGE_LINE_TRUNCATION_MARKER}); *not*
-    #   line-numbered — numbering is presentation the caller adds. For
-    #   a PDF the array includes +"--- Page N ---"+ marker lines (one
-    #   per page that contributed text), which count toward +limit+ /
-    #   the byte cap like any other line.
-    # * +start_line+ — the 1-indexed line number of +lines.first+
-    #   (i.e. the +offset+ the caller asked for). +lines.last+ is at
-    #   +start_line + lines.length - 1+.
-    # * +total_lines+ — total line count of the document when known,
-    #   else +nil+. Known when extraction reached EOF (so the caller
-    #   can print "of N"); +nil+ when the read stopped early — the
-    #   byte cap fired, or a PDF filled the window before its last
-    #   page (counting the rest would defeat the laziness).
-    # * +more+ — +true+ if content remains past this window (the
-    #   caller should offer +offset = start_line + lines.length+).
-    # * +byte_capped+ — +true+ if {PAGE_MAX_BYTES} (not the line
-    #   limit) was the stopping criterion.
-    # * +kind+ — +:text+ or +:pdf+; lets the caller word PDF-specific
-    #   trailers and the empty-document message.
-    #
-    # An empty document yields +lines: []+, +total_lines: 0+; an
-    # +offset+ past EOF yields +lines: []+ with +total_lines+ set to
-    # the real (non-zero) count — the caller distinguishes the two.
-    Page = Data.define(:lines, :start_line, :total_lines, :more, :byte_capped, :kind)
-
     # Recognise a file from its leading bytes. Returns the MIME type
     # as a String for formats pikuri handles specially, or +nil+ for
     # "unrecognised" — callers interpret +nil+ themselves (text,
@@ -197,19 +136,13 @@ module Pikuri
       non_printable.to_f / bytes.bytesize > BINARY_NONPRINTABLE_THRESHOLD
     end
 
-    # Read +path+ and return its content as plain UTF-8 text. Two
-    # extraction paths, picked by {.detect_mime}:
-    #
-    # * **PDF** — walked page-by-page via +pdf-reader+; each page's
-    #   extracted text is stripped and pages are joined with a blank
-    #   line. A scanned-image PDF (no extractable text) comes back as
-    #   the empty String — a deliberate silent skip, callers detect by
-    #   length if they care.
-    # * **Plain text** — anything that {.detect_mime} doesn't
-    #   recognise and that {.binary?} accepts. Read with UTF-8
-    #   encoding; behaviour on non-UTF-8 bytes is whatever +File.read+
-    #   does with +encoding: Encoding::UTF_8+ (which is "leave invalid
-    #   bytes in, let downstream decide").
+    # Read +path+ and return its content as plain UTF-8 text, routed
+    # through the {Extractor} registry: anything
+    # unrecognised-but-textual passes through verbatim
+    # ({Extractor::Passthrough}); with pikuri-pdf registered, PDFs
+    # are extracted with +"--- Page N ---"+ markers (a scanned-image
+    # PDF with no extractable text comes back as the empty String, a
+    # deliberate silent skip callers detect by length if they care).
     #
     # Refusal cases — all raise rather than returning a sentinel
     # because the callers are internal pikuri code, not an LLM
@@ -220,13 +153,11 @@ module Pikuri
     # * Path is a directory → +ArgumentError+.
     # * Image (PNG / JPEG / GIF / WebP per {.detect_mime}) →
     #   +ArgumentError+; images aren't text.
-    # * Binary content (per {.binary?}) and not a recognised MIME →
-    #   +ArgumentError+.
-    # * Malformed PDF — +pdf-reader+'s
-    #   +MalformedPDFError+ / +UnsupportedFeatureError+ /
-    #   +InvalidPageError+ are re-raised as a +RuntimeError+ with the
-    #   path included so callers don't need to know pdf-reader's
-    #   exception hierarchy.
+    # * Content no extractor claims (opaque binary) →
+    #   +ArgumentError+, mapped from {Extractor::Unsupported}.
+    # * Extraction failure (malformed PDF, ...) → +RuntimeError+ with
+    #   the path included, mapped from {Extractor::Error} so callers
+    #   don't need to know any extractor's exception hierarchy.
     #
     # @param path [Pathname] file to read.
     # @return [String] UTF-8 text. May be empty (empty text file, or
@@ -234,56 +165,26 @@ module Pikuri
     # @raise [ArgumentError] if +path+ isn't a +Pathname+, points at
     #   a directory, is an image, or is binary.
     # @raise [Errno::ENOENT] if +path+ doesn't exist.
-    # @raise [RuntimeError] on a malformed / unsupported PDF.
+    # @raise [RuntimeError] on an extraction failure (malformed /
+    #   unsupported PDF, ...).
     def read_as_text(path)
-      raise ArgumentError, "expected Pathname, got #{path.class}" unless path.is_a?(Pathname)
-      raise Errno::ENOENT, path.to_s unless path.exist?
-      raise ArgumentError, "#{path} is a directory" if path.directory?
-
-      mime = detect_mime(path)
-      return read_pdf_text(path) if mime == 'application/pdf'
-      raise ArgumentError, "#{path} is an image (#{mime}); cannot extract as text" if mime&.start_with?('image/')
-      raise ArgumentError, "#{path} appears to be binary; cannot extract as text" if binary?(path)
-
-      path.read(encoding: Encoding::UTF_8)
+      mime = guard_extractable(path)
+      path.open('rb') { |io| Extractor.extract(io, content_type: mime) }
+    rescue Extractor::Unsupported
+      raise ArgumentError, "#{path} appears to be binary; cannot extract as text"
+    rescue Extractor::Error => e
+      raise "Cannot extract text from #{path}: #{e.message}"
     end
 
-    # Walk a PDF page-by-page via +pdf-reader+, returning a single
-    # String with non-empty page texts joined by blank lines. Catches
-    # the three +PDF::Reader+ exceptions Workspace::Read also handles
-    # and re-raises them as +RuntimeError+ with the path included.
-    #
-    # @param path [Pathname]
-    # @return [String]
-    # @raise [RuntimeError] on malformed / unsupported PDF.
-    def read_pdf_text(path)
-      pages = path.open('rb') do |io|
-        ::PDF::Reader.new(io).pages.map { |p| p.text.strip }
-      end
-      pages.reject(&:empty?).join("\n\n")
-    rescue ::PDF::Reader::MalformedPDFError,
-           ::PDF::Reader::UnsupportedFeatureError,
-           ::PDF::Reader::InvalidPageError => e
-      raise "Cannot extract PDF text from #{path}: " \
-            "#{e.class.name.split('::').last}: #{e.message}"
-    end
-    private_class_method :read_pdf_text
-
-    # Extract +path+ as text and return a windowed {Page}: the lines
-    # from +offset+ (1-indexed) up to +limit+ of them, stopping early
-    # if +max_bytes+ is reached, with over-long lines truncated at
-    # +max_line_length+. Lazy by design — a text file is streamed
-    # line-by-line and a PDF is parsed page-by-page only until the
-    # window fills, so reading the first page of a 500-page PDF parses
-    # a handful of pages, not all of them.
-    #
-    # Same routing and refusal contract as {.read_as_text}: PDFs are
-    # extracted (with +"--- Page N ---"+ marker lines, unlike
-    # {.read_as_text}'s marker-free join — paging is a display path,
-    # the marker-free form stays the indexing path); images, binaries,
-    # directories, missing files, and malformed PDFs all raise rather
-    # than returning a sentinel. The LLM-facing callers map those into
-    # +"Error: ..."+ observations themselves.
+    # Extract +path+ and return a windowed {Extractor::Page}: the
+    # lines from +offset+ (1-indexed) up to +limit+ of them, stopping
+    # early if +max_bytes+ is reached, with over-long lines truncated
+    # at +max_line_length+. Same routing and refusal contract as
+    # {.read_as_text}; the windowing semantics (including the lazy
+    # +extract_lines+ consumption that stops parsing once the window
+    # fills) are {Extractor.extract_paged}'s.
+    # The LLM-facing callers map the exceptions into +"Error: ..."+
+    # observations themselves.
     #
     # @param path [Pathname] file to read.
     # @param offset [Integer] 1-indexed first line to include. The
@@ -292,141 +193,48 @@ module Pikuri
     #   validates +limit >= 1+.
     # @param max_bytes [Integer] hard byte cap on collected content.
     # @param max_line_length [Integer] per-line truncation threshold.
-    # @return [Page] the windowed slice.
+    # @return [Extractor::Page] the windowed slice.
     # @raise [ArgumentError] if +path+ isn't a +Pathname+, is a
     #   directory, an image, or binary.
     # @raise [Errno::ENOENT] if +path+ doesn't exist.
-    # @raise [RuntimeError] on a malformed / unsupported PDF.
-    def read_as_text_paged(path, offset: 1, limit: PAGE_DEFAULT_LIMIT,
-                           max_bytes: PAGE_MAX_BYTES, max_line_length: PAGE_MAX_LINE_LENGTH)
+    # @raise [RuntimeError] on an extraction failure (malformed /
+    #   unsupported PDF, ...).
+    def read_as_text_paged(path, offset: 1, limit: Extractor::PAGE_DEFAULT_LIMIT,
+                           max_bytes: Extractor::PAGE_MAX_BYTES,
+                           max_line_length: Extractor::PAGE_MAX_LINE_LENGTH)
+      mime = guard_extractable(path)
+      path.open('rb') do |io|
+        Extractor.extract_paged(io, content_type: mime, offset: offset, limit: limit,
+                                    max_bytes: max_bytes, max_line_length: max_line_length)
+      end
+    rescue Extractor::Unsupported
+      raise ArgumentError, "#{path} appears to be binary; cannot extract as text"
+    rescue Extractor::Error => e
+      raise "Cannot extract text from #{path}: #{e.message}"
+    end
+
+    # The shared path-level refusals for {.read_as_text} /
+    # {.read_as_text_paged}: must be an existing non-directory
+    # +Pathname+, and not an image (images are data for a vision
+    # model, never text). Returns the {.detect_mime} result so the
+    # caller can pass it to the {Extractor} as the content-type hint.
+    #
+    # @param path [Pathname]
+    # @return [String, nil] the sniffed MIME type.
+    # @raise [ArgumentError] on a non-Pathname, a directory, or an
+    #   image.
+    # @raise [Errno::ENOENT] if +path+ doesn't exist.
+    def guard_extractable(path)
       raise ArgumentError, "expected Pathname, got #{path.class}" unless path.is_a?(Pathname)
       raise Errno::ENOENT, path.to_s unless path.exist?
       raise ArgumentError, "#{path} is a directory" if path.directory?
 
       mime = detect_mime(path)
-      if mime == 'application/pdf'
-        return paged_pdf(path, offset: offset, limit: limit,
-                               max_bytes: max_bytes, max_line_length: max_line_length)
-      end
       raise ArgumentError, "#{path} is an image (#{mime}); cannot extract as text" if mime&.start_with?('image/')
-      raise ArgumentError, "#{path} appears to be binary; cannot extract as text" if binary?(path)
-
-      paged_text(path, offset: offset, limit: limit,
-                       max_bytes: max_bytes, max_line_length: max_line_length)
-    end
-
-    # Stream a text file line-by-line into a {Page}. Keeps counting
-    # lines past the collection window so +total_lines+ can report the
-    # real total when the line limit (not the byte cap) stopped
-    # collection; on the byte cap it breaks and leaves +total_lines+
-    # +nil+ (the rest of the file is never read).
-    #
-    # @return [Page] +kind: :text+.
-    def paged_text(path, offset:, limit:, max_bytes:, max_line_length:)
-      start_index = offset - 1
-      collected   = []
-      total_lines = 0
-      bytes       = 0
-      byte_capped = false
-      more        = false
-
-      path.each_line do |raw|
-        total_lines += 1
-        next if total_lines <= start_index
-
-        if collected.length >= limit
-          more = true
-          next
-        end
-
-        line = truncate_line(raw.chomp, max_line_length)
-        size = line.bytesize + 1 # +1 for the joining newline
-        if bytes + size > max_bytes
-          byte_capped = true
-          more = true
-          break
-        end
-        collected << line
-        bytes += size
-      end
-
-      Page.new(lines: collected, start_line: offset,
-               total_lines: byte_capped ? nil : total_lines,
-               more: more, byte_capped: byte_capped, kind: :text)
-    end
-    private_class_method :paged_text
-
-    # PDF counterpart to {paged_text}: walk +pdf-reader+'s lazy page
-    # iterator, emitting a +"--- Page N ---"+ header line then each
-    # line of the page's text, applying the same offset / limit /
-    # byte-cap contract. The +throw :done+ short-circuits both loops
-    # the moment the window fills, so parsing stops — which is why a
-    # PDF that stops early can't report +total_lines+ (it would have
-    # to parse every page to count).
-    #
-    # @return [Page] +kind: :pdf+.
-    # @raise [RuntimeError] on a malformed / unsupported PDF.
-    def paged_pdf(path, offset:, limit:, max_bytes:, max_line_length:)
-      start_index = offset - 1
-      collected   = []
-      total_lines = 0
-      bytes       = 0
-      byte_capped = false
-      more        = false
-
-      catch(:done) do
-        path.open('rb') do |io|
-          reader = ::PDF::Reader.new(io)
-          reader.pages.each_with_index do |page, idx|
-            text = page.text.strip
-            next if text.empty?
-
-            ["--- Page #{idx + 1} ---", *text.split("\n")].each do |raw|
-              total_lines += 1
-              next if total_lines <= start_index
-
-              if collected.length >= limit
-                more = true
-                throw :done
-              end
-
-              line = truncate_line(raw, max_line_length)
-              size = line.bytesize + 1
-              if bytes + size > max_bytes
-                byte_capped = true
-                more = true
-                throw :done
-              end
-              collected << line
-              bytes += size
-            end
-          end
-        end
-      end
-
-      Page.new(lines: collected, start_line: offset,
-               total_lines: more ? nil : total_lines,
-               more: more, byte_capped: byte_capped, kind: :pdf)
-    rescue ::PDF::Reader::MalformedPDFError,
-           ::PDF::Reader::InvalidPageError,
-           ::PDF::Reader::UnsupportedFeatureError => e
-      raise "Cannot extract PDF text from #{path}: " \
-            "#{e.class.name.split('::').last}: #{e.message}"
-    end
-    private_class_method :paged_pdf
-
-    # Truncate +line+ to +max_line_length+ chars, appending
-    # {PAGE_LINE_TRUNCATION_MARKER} when it overflows.
-    #
-    # @param line [String]
-    # @param max_line_length [Integer]
-    # @return [String]
-    def truncate_line(line, max_line_length)
-      return line if line.length <= max_line_length
 
-      line[0, max_line_length] + PAGE_LINE_TRUNCATION_MARKER
+      mime
     end
-    private_class_method :truncate_line
+    private_class_method :guard_extractable
 
     # Coerce an +input+ argument into a bytes String for the sniffs.
     # +String+ inputs are returned as-is (caller already sampled);

data/lib/pikuri/subprocess.rb

@@ -7,8 +7,10 @@ module Pikuri
   # Chokepoint for *all* subprocess spawning in pikuri. Forces a new
   # process group for each invocation, tracks pgids so descendants of
   # the direct child (commands backgrounded with +&+) can be cleaned
-  # up at process exit, and captures combined stdout+stderr through a
-  # single pipe.
+  # up at process exit. Two front doors: {.spawn} (combined
+  # stdout+stderr through a single pipe — the shell-command shape) and
+  # {.run} (stdin fed from a String or streamed from an IO, stdout
+  # redirected to a file, stderr captured — the filter shape).
   #
   # == Seam discipline
   #
@@ -91,6 +93,75 @@ module Pikuri
       new(io: io, wait_thr: wait_thr)
     end
 
+    # Run +argv+ as a one-shot *filter*: feed it +stdin_data+, redirect
+    # its stdout straight to +stdout+ (an open +File+), capture stderr
+    # through a pipe, and block until it exits. Built for the
+    # stdin→markdown document converters (pikuri-extractors), where
+    # {.spawn}'s shape is wrong twice over: it closes the child's stdin
+    # immediately, and it merges stderr onto stdout — fatal when stdout
+    # *is* the payload and a converter's warnings would corrupt it.
+    #
+    # Redirecting stdout to a file (not a pipe) is also what makes the
+    # I/O deadlock-free with one writer thread: the child never blocks
+    # writing output, so it keeps draining stdin, while the parent
+    # drains the (low-volume) stderr pipe. The returned
+    # {Result#output} is the captured *stderr* — the diagnostics — not
+    # the payload; the payload is in +stdout+, whose file offset is
+    # shared with the child, so rewind before reading it back.
+    #
+    # Same discipline as {.spawn}: new process group, registered for
+    # the exit sweep, no built-in timeout (wrap +argv+ with coreutils'
+    # +timeout+ — see the class docs).
+    #
+    # @param argv [Array<String>] command and arguments, passed to
+    #   +exec+ directly — no implicit shell.
+    # @param stdin_data [String, IO, StringIO] the child's stdin: a
+    #   String is written as-is, an IO is streamed through
+    #   +IO.copy_stream+ (so a large source file never materialises in
+    #   the Ruby heap) and read from its current position. Either way
+    #   stdin is closed (EOF) afterwards. May be empty.
+    # @param stdout [File] open writable file the child's stdout is
+    #   redirected to.
+    # @param chdir [String, Pathname] working directory.
+    # @param env [Hash{String=>String}] extra environment variables,
+    #   as for {.spawn}.
+    # @return [Result] +output+ is the captured stderr; +status+ the
+    #   child's exit status.
+    # @raise [SystemCallError] whatever an IO +stdin_data+ raises
+    #   mid-stream (disk error, closed handle) — re-raised here after
+    #   the child has been reaped.
+    def self.run(*argv, stdin_data:, stdout:, chdir:, env: {})
+      in_r, in_w = IO.pipe
+      err_r, err_w = IO.pipe
+      pid = Process.spawn(env, *argv, chdir: chdir.to_s, pgroup: true,
+                          in: in_r, out: stdout, err: err_w)
+      in_r.close
+      err_w.close
+      register(pid)
+      writer = Thread.new do
+        in_w.binmode
+        if stdin_data.respond_to?(:read)
+          IO.copy_stream(stdin_data, in_w)
+        else
+          in_w.write(stdin_data)
+        end
+      rescue Errno::EPIPE
+        nil # child exited without draining stdin; its status tells the story
+      ensure
+        in_w.close
+      end
+      stderr = err_r.read
+      err_r.close
+      # Reap before joining the writer: if an IO source raised
+      # mid-stream, #join re-raises it, and the child (already exited —
+      # err_r hit EOF) must not be left a zombie.
+      _, status = Process.waitpid2(pid)
+      writer.join
+      Result.new(output: stderr, status: status)
+    ensure
+      prune(pid) if pid
+    end
+
     # @return [Integer] direct child's pid
     attr_reader :pid