pikuri-extractors 0.0.7 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be1bcb56e2c29751356aaca763f9dc6d4ea4c91605255522c3fcdd7a80d7483b
-  data.tar.gz: e3f82c933661d434173898156294573a0a6ee1f9de56efe78b8b8e5af035ce60
+  metadata.gz: 987bc9fcc9828a020ffaabd52551f22c40d5862a807cb8a6b528dd807182ab7b
+  data.tar.gz: c2e1ac45003e57acf35efde390d200b12b447ce0249c43a8748eddf431bda4e3
 SHA512:
-  metadata.gz: 2218b261c7400bd30ea8f113ec3b2cc4530786bd4c7384cfba68fd9453f065110fe17bf60eaf35df5ac9cfa27df9682e3346a10104850d1578fd7a2487ca709b
-  data.tar.gz: 62b679db1d2d9f0410e1af73a2a05b0fb85d59761ac8ac08beea49dce526faeb2c5177917c9dd7d45fe2b1c71049b092639ed66d9dcce205992b18b4a84334b9
+  metadata.gz: 45e4ea2c9d7127f2d45621dab27e03d54d742aeaca310424c6cc21463993c08129dbc716dd9738d70daf09c6c0f55564762441472eeb9a8c1b30c95fcd4a9558
+  data.tar.gz: c64393a2f30e85803262021cc7756d38b3ffb773589d7a1f278eefe7454e77e8bf7a73f282e920e8f1e96b61144852fb8d244a7364007e64f5dcdbeda4e651dc

data/README.md

@@ -50,8 +50,9 @@ wiring.** This gem parses PDFs inside the sandbox (poppler is native
 code chewing attacker-controlled bytes — exactly what the container
 is for) but re-converts the whole document on every paged read;
 pikuri-pdf is in-process pure Ruby with lazy page-windowed reads and
-no infrastructure to set up. The guide wires pikuri-pdf in chapter 3
-and supersedes it with this gem in chapter 7's assistant.
+no infrastructure to set up. The guide wires pikuri-pdf in the
+vector-DB chapter and supersedes it with this gem in the
+universal-assistant chapter.
 
 ## Install
 

data/lib/pikuri/extractors/documents.rb

@@ -6,120 +6,35 @@ module Pikuri
   module Extractors
     # Document extractor for the {Pikuri::Extractor} registry:
     # DOCX / ODT / XLSX / legacy XLS / PPTX / EPUB / RTF / PDF →
-    # Markdown, by piping the document bytes through pandoc (ODF,
-    # RTF, EPUB, DOCX), markitdown (the OOXML spreadsheet /
-    # presentation arms), or pdftotext (PDF), selected per format.
+    # Markdown, by piping the document bytes through pandoc (ODF, RTF,
+    # EPUB, DOCX), markitdown (OOXML spreadsheet / presentation), or
+    # pdftotext (PDF), selected per format.
     #
-    # == Container first, host CLI second
+    # Prefers a one-shot, networkless docker container ({IMAGE}) — bytes
+    # in via stdin, Markdown out via stdout, no volume mounts — and falls
+    # back to host +pandoc+ / +markitdown+ / +pdftotext+ CLIs on the same
+    # stdin→stdout contract when docker is absent. Requiring the gem
+    # defines this class + the shared {DOCUMENTS} instance but registers
+    # nothing; a host opts in with +DOCUMENTS.register+.
     #
-    # The preferred converter is a one-shot, locally-built docker
-    # container ({IMAGE}, built from this gem's +docker/+ directory):
-    # +docker run --rm -i --network=none --read-only --cap-drop=ALL+,
-    # bytes in via stdin, Markdown out via stdout, **no volume
-    # mounts**. Two reasons this beats running a host-installed
-    # converter directly:
-    #
-    # * *Security.* These documents typically arrive via +fetch+ /
-    #   +web_scrape+ — untrusted bytes — and complex format parsers
-    #   are a classic exploitation surface. In the container the
-    #   parser sees no network and no host filesystem; the worst a
-    #   malicious document can do is produce garbage Markdown.
-    # * *Reproducibility.* The Dockerfile pins pandoc (via the base
-    #   image's apt) and markitdown (exact pip version); a host
-    #   install is whatever version the machine happens to have.
-    #
-    # When docker is unavailable (binary absent or daemon down), the
-    # extractor falls back to host-installed +pandoc+ / +markitdown+
-    # / +pdftotext+ CLIs — same stdin→stdout contract, one code path
-    # with two argv builders. Which arm was picked is logged once via
-    # +Pikuri.logger_for('Extractors')+.
-    #
-    # == Registration is explicit
-    #
-    # Requiring pikuri-extractors defines this class and the shared
-    # {DOCUMENTS} instance but registers nothing. A host script opts
-    # in with +Pikuri::Extractors::DOCUMENTS.register+, which inserts
-    # the instance before the registry's terminal +Passthrough+ entry
-    # (and after core's +HTML+ — and pikuri-pdf's front-inserted
-    # +PDF+, when the host registers that too — which keep winning
-    # their formats). Same opt-in philosophy as +c.add_extension+ —
-    # no behavior changes by require alone.
+    # The cross-cutting *why* — the container's security + reproducibility
+    # rationale, the PDF-arm trade-off against pikuri-pdf, the
+    # deliberately-unsupported formats (ODS/ODP, OCR/audio), and why
+    # paging re-converts each time — lives in
+    # +pikuri-extractors/DESIGN.md+.
     #
     # == Format detection
     #
-    # {#matches?} claims content by normalized content-type
-    # ({CONTENT_TYPES}) or by byte sniff: PDF's +%PDF-+ prefix, RTF's
-    # +{\\rtf+ prefix; for
-    # zip-based formats, ODF and EPUB mandate an uncompressed
-    # +mimetype+ first entry (so the literal mime string sits inside
-    # the leading sample), and OOXML is recognised by the
-    # +[Content_Types].xml+ entry plus a +word/+ / +ppt/+ / +xl/+
-    # entry-name scan. {#extract} re-sniffs (the registry duck type
-    # doesn't pass +content_type+ to +extract+); when content was
-    # claimed by content-type but the sniff is blind (legacy XLS — an
-    # OLE2 container whose discriminating directory sits at the end of
-    # the file, past the sample), the bytes go to markitdown with no
-    # format hint and its own magic-byte detection takes over. The
-    # consequence: a *local* +.xls+ (no transport content-type, sniff
-    # blind) is not claimed at all and keeps today's binary refusal.
-    # One ordering edge vs pikuri-pdf: this instance sits *after*
-    # core's +HTML+ in the registry, so a PDF served under a lying
-    # +text/html+ header goes to the HTML extractor (pikuri-pdf
-    # front-inserts and wins that case). Accepted — lying-header PDFs
-    # under specifically +text/html+ are rare.
-    #
-    # == PDF: this gem or pikuri-pdf — pick one per wiring
-    #
-    # The PDF arm (pdftotext, with {#pdf_page_lines} restoring the
-    # +"--- Page N ---"+ markers from pdftotext's +\f+ separators)
-    # makes this extractor a complete superset of pikuri-pdf's
-    # formats, so a host that registers {DOCUMENTS} does NOT also
-    # register +Extractors::PDF+ — one extractor per format keeps the
-    # registry's first-match-wins semantics legible. The trade per
-    # wiring:
-    #
-    # * *This gem* — PDF parsing happens inside the sandbox (poppler
-    #   is native code parsing attacker-controlled bytes; the
-    #   container is exactly the right place for it), one gem covers
-    #   every document format. Costs: docker (or host CLIs), no lazy
-    #   paging (each paged read re-converts the whole PDF), and the
-    #   generic +:document+ kind (the Read tools say "End of file",
-    #   not "End of PDF", and a scanned PDF reads as "(Empty file)"
-    #   rather than the scanned-image hint).
-    # * *pikuri-pdf* — in-process pure Ruby (no infrastructure), lazy
-    #   +extract_lines+ paging (a windowed read of a 500-page PDF
-    #   parses only its window), PDF-specific Read-tool wording.
-    #   Costs: pdf-reader's dependency subtree, parsing untrusted
-    #   bytes in-process (pure Ruby, so DoS at worst).
-    #
-    # The guide walks this as a progression: chapter 3 wires
-    # pikuri-pdf (no docker yet), chapter 7's assistant supersedes it
-    # with this extractor.
-    #
-    # == Deliberately out of scope
-    #
-    # * *ODS / ODP* — neither pandoc nor markitdown reads them; the
-    #   only converter that does (LibreOffice headless) costs a 2 GB+
-    #   image. Excluded rather than half-supported.
-    # * *Image OCR / audio transcription* — markitdown's optional
-    #   arms need model downloads; the converter image stays
-    #   networkless and small. A multi-modal main LLM is the pikuri
-    #   answer to images.
-    #
-    # == Paging economics
-    #
-    # A subprocess converter needs the whole document before it can
-    # emit anything, so there is no lazy parse: every
-    # +Extractor.extract_paged+ call (each +Read+ page of a long DOCX)
-    # re-runs the full conversion. Accepted — no result cache in v1.
-    # Both legs of one conversion still stream, though: the source +io+
-    # is handed to {Pikuri::Subprocess.run} and copied straight into
-    # the converter's stdin (+IO.copy_stream+ — a big local file never
-    # loads into the Ruby heap), and the converter's stdout lands in a
-    # Tempfile (also what makes the stdin/stdout pumping deadlock-free
-    # — see {Pikuri::Subprocess.run}) whose lines {#extract_lines}
-    # yields from disk — so neither the document nor the full Markdown
-    # String is ever resident during paging.
+    # {#matches?} claims content by content-type ({CONTENT_TYPES}) or byte
+    # sniff (see {#sniff}). {#extract} re-sniffs (the registry duck type
+    # doesn't pass +content_type+ to +extract+); when the sniff is blind
+    # — legacy XLS, an OLE2 container whose discriminating directory sits
+    # past the sample — bytes go to markitdown with no hint and its own
+    # detection takes over. Consequence: a *local* +.xls+ (no transport
+    # content-type, sniff blind) isn't claimed at all and keeps today's
+    # binary refusal. Ordering edge: this instance sits *after* core's
+    # +HTML+, so a PDF served under a lying +text/html+ header goes to
+    # HTML (pikuri-pdf front-inserts and wins that) — accepted, rare.
     class Documents
       # @return [Logger] gem-wide diagnostics logger.
       LOGGER = Pikuri.logger_for('Extractors')
@@ -145,14 +60,10 @@ module Pikuri
       #   claimed by content-type but the byte sniff is blind.
       AUTO = 'auto'
 
-      # @return [String] the PDF format tag. Singled out as a constant
-      #   because PDF is the one format whose converter output gets a
-      #   post-processing pass: pdftotext emits +\f+ between pages, and
-      #   {#extract} / {#extract_lines} turn those into the same
-      #   +"--- Page N ---"+ marker lines pikuri-pdf's extractor emits,
-      #   so page provenance (vectordb chunk citations, the Read tools'
-      #   page references) survives whichever PDF extractor a host
-      #   wires.
+      # @return [String] the PDF format tag. A constant because PDF is the
+      #   one format whose output gets a post-processing pass ({#pdf_page_lines}
+      #   restoring +"--- Page N ---"+ markers), so page provenance survives
+      #   whichever PDF extractor a host wires.
       PDF = 'pdf'
 
       # @return [Hash{String => String}] normalized content-type →
@@ -231,17 +142,14 @@ module Pikuri
         end
       end
 
-      # Same content as {#extract}, as a stream of +chomp+ed lines
-      # read off the converter's stdout Tempfile — the whole-document
-      # conversion still runs up front (subprocess converters can't
-      # parse lazily), but neither the document nor the Markdown ever
-      # materialises as one String: the conversion fires on first
-      # consumption, streaming +io+ into the converter. The enumerator
-      # owns the Tempfile and deletes it when iteration ends.
+      # Same content as {#extract}, as a stream of +chomp+ed lines off the
+      # converter's stdout Tempfile. The full conversion still runs up
+      # front (fired on first consumption), but neither the document nor
+      # the Markdown ever materialises as one String. The enumerator owns
+      # the Tempfile and deletes it when iteration ends.
       #
-      # @param io [IO, StringIO] seekable IO positioned at the start;
-      #   must remain open until the enumerator is consumed (same
-      #   contract as pikuri-pdf's lazy +extract_lines+).
+      # @param io [IO, StringIO] seekable IO positioned at the start; must
+      #   remain open until the enumerator is consumed.
       # @return [Enumerator<String>]
       # @raise [Pikuri::Extractor::Error] as for {#extract}, raised on
       #   first consumption.
@@ -257,9 +165,10 @@ module Pikuri
         end
       end
 
-      # Plug this extractor into {Pikuri::Extractor.registry}, before
-      # the terminal +Passthrough+ entry. Idempotent — a second call
-      # is a no-op.
+      # Plug this extractor into {Pikuri::Extractor.registry}, before the
+      # terminal +Passthrough+ entry but after core's +HTML+ and
+      # pikuri-pdf's front-inserted +PDF+ (both keep winning their
+      # formats). Idempotent — a second call is a no-op.
       #
       # @return [Documents] self, for chaining.
       def register

data/lib/pikuri-extractors.rb

@@ -2,19 +2,10 @@
 
 require 'pikuri-core'
 
-# Entry file for the pikuri-extractors gem. Sets up a dedicated
-# Zeitwerk loader rooted at this gem's +lib/+, contributing to the
-# shared +Pikuri::+ namespace alongside pikuri-core. After +require
-# 'pikuri-extractors'+, +Pikuri::Extractors::Documents+ and the shared
-# +Pikuri::Extractors::DOCUMENTS+ instance are defined — but *nothing
-# is registered*: extractors plug into +Pikuri::Extractor.registry+
-# only when the host script calls their +#register+ explicitly, so a
-# +bin/pikuri-*+ picks which extractors it wires in (same opt-in
-# philosophy as +c.add_extension+).
-#
-# The loader is per-gem (not shared with pikuri-core's loader) so each
-# gem owns its own +lib/+ tree and the cooperation between gems is via
-# the Pikuri namespace alone.
+# Entry file for the pikuri-extractors gem: sets up the per-gem Zeitwerk
+# loader. Requiring it defines {Pikuri::Extractors::Documents} + the
+# shared {Pikuri::Extractors::DOCUMENTS} but registers nothing — a host
+# opts in with +DOCUMENTS.register+ (see there).
 module Pikuri
   module Extractors
     LOADER = Zeitwerk::Loader.new

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pikuri-extractors
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.1.0
 platform: ruby
 authors:
 - Martin Vysny
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.7
+        version: 0.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.7
+        version: 0.1.0
 description: |
   pikuri-extractors plugs additional document formats into
   pikuri-core's +Pikuri::Extractor+ registry. The bundled