pikuri-core 0.0.5 → 0.0.7

data/lib/pikuri/extractor/html.rb

@@ -0,0 +1,303 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'nokogiri'
+require 'readability'
+require 'reverse_markdown'
+
+module Pikuri
+  module Extractor
+    # HTML → Markdown extractor.
+    #
+    # Matched by content-type only (+text/html+ /
+    # +application/xhtml+xml+) — deliberately no byte sniff. The web
+    # path always has the header; for local files a sniff would route
+    # +Workspace::Read+ of an +.html+ source file through readability
+    # extraction, when a developer reading an HTML file wants the
+    # source. Local HTML stays on the {Passthrough} arm until a
+    # consumer genuinely needs otherwise.
+    #
+    # Always renders both views of the page when available:
+    #
+    # 1. JSON-LD section. Any +<script type="application/ld+json">+ node
+    #    whose +@type+ matches a substantive schema.org content type
+    #    (Product, Article, Recipe, ...) is rendered as a header — title,
+    #    metadata bullets (brand, SKU, price, rating, author, published),
+    #    and the +articleBody+/+description+ copy when present.
+    # 2. Readability section. The page is run through +Readability+ +
+    #    +reverse_markdown+, with a +<main>+/+<article>+ fallback for
+    #    pages whose content sits mostly outside +<p>+ tags.
+    #
+    # Concatenated with a horizontal rule, so the LLM gets both the
+    # structured metadata and the rendered body and can pick whichever
+    # is more useful for the task. Trades some duplication (when a
+    # publisher embeds the article body in JSON-LD AND in HTML) for
+    # fewer type-based heuristics on which branch should win — the
+    # earlier "is this Article's +description+ a teaser or the real
+    # body?" carve-out is no longer needed because both end up in
+    # the output regardless.
+    module HTML
+      # @return [Array<String>] content-types this extractor claims.
+      CONTENT_TYPES = %w[text/html application/xhtml+xml].freeze
+
+      # @return [Array<String>] schema.org +@type+ values that we treat
+      #   as "the primary entity of this page" when picking a JSON-LD
+      #   node to render. Order does not matter — the first matching
+      #   node wins. Skips noise nodes (Organization, BreadcrumbList,
+      #   WebSite, ...) that ship on most pages but carry no page
+      #   content.
+      INTERESTING_TYPES = %w[
+        Product Article NewsArticle BlogPosting Recipe Event Book Movie
+      ].freeze
+
+      # @return [Array<String>] HTML tags preserved by the readability
+      #   pass. Anything outside this list is stripped before Markdown
+      #   conversion.
+      READABILITY_TAGS = %w[
+        h1 h2 h3 h4 h5 h6 p div span ul ol li blockquote pre code a img
+        strong em b i br hr table thead tbody tr td th
+      ].freeze
+
+      # @return [Array<String>] HTML attributes preserved by the
+      #   readability pass; everything else (class, id, style, data-*)
+      #   is dropped before Markdown conversion
+      READABILITY_ATTRS = %w[href src alt title].freeze
+
+      # @return [Float] minimum +<main>+/+<article>+ to Readability
+      #   text-length ratio that triggers the semantic-container
+      #   fallback in {.readability_to_markdown}. Picked low enough to
+      #   catch the failure mode (Readability collapsing a page that
+      #   uses divs/lists instead of +<p>+ — e.g. +vaadin.com/company+,
+      #   ~5x) but high enough that pages where both produce
+      #   comparable output keep Readability's noise filtering.
+      MAIN_FALLBACK_RATIO = 2.0
+
+      # @return [Integer] minimum text length the
+      #   +<main>+/+<article>+ container must hold before the fallback
+      #   in {.readability_to_markdown} can fire. Below this, the
+      #   ratio comparison is dominated by noise and we'd swap on
+      #   tiny pages where Readability is doing the right thing.
+      MAIN_FALLBACK_MIN_CHARS = 500
+
+      # @return [Symbol] {Page#kind} tag.
+      def self.kind
+        :html
+      end
+
+      # @param sample [String] leading bytes of the content (unused —
+      #   see the no-sniff rationale in the module doc).
+      # @param content_type [String, nil] normalized content-type.
+      # @return [Boolean]
+      def self.matches?(sample:, content_type:)
+        CONTENT_TYPES.include?(content_type)
+      end
+
+      # Render the HTML document behind +io+ as Markdown by emitting
+      # both the JSON-LD section (when an interesting node is present)
+      # and the readability / +<main>+ section, joined by a horizontal
+      # rule. Either section may be missing — pages with no JSON-LD
+      # return only the readability output, and a malformed page with
+      # no extractable body returns only the JSON-LD render.
+      #
+      # @param io [IO, StringIO] IO over the HTML document.
+      # @return [String] Markdown representation
+      def self.extract(io)
+        html = io.read
+        sections = [jsonld_section(html), readability_to_markdown(html)]
+        sections.reject! { |s| s.nil? || s.strip.empty? }
+        sections.join("\n\n---\n\n")
+      end
+
+      # Pick the first JSON-LD node whose +@type+ matches one of
+      # {INTERESTING_TYPES} and render it as Markdown. Returns +nil+
+      # when no such node exists, in which case {.extract} emits only
+      # the readability section.
+      #
+      # No content-field gating: a node carrying just +name+/+author+/
+      # +datePublished+ still renders (as a metadata-only header),
+      # because the readability pass independently produces the page
+      # body. That is the trade-off that lets us drop the type-based
+      # "is this teaser or article copy?" heuristics — duplication is
+      # acceptable when both views are available, and the LLM can
+      # pick whichever it needs.
+      #
+      # @param html [String] HTML document body
+      # @return [String, nil] Markdown render of the picked JSON-LD
+      #   node, or +nil+ when nothing matched
+      def self.jsonld_section(html)
+        node = parse_jsonld(html).find do |n|
+          Array(n['@type']).any? { |t| INTERESTING_TYPES.include?(t) }
+        end
+        node ? jsonld_to_markdown(node) : nil
+      end
+
+      # Collect every JSON-LD payload embedded in +html+, flattening
+      # +@graph+ wrappers so callers see one flat array of schema.org
+      # nodes. Malformed JSON blocks are silently skipped — sites
+      # frequently ship broken JSON-LD and we only need at least one
+      # parseable block.
+      #
+      # @param html [String] HTML document body
+      # @return [Array<Hash>] parsed JSON-LD nodes; possibly empty
+      def self.parse_jsonld(html)
+        doc = Nokogiri::HTML(html)
+        blobs = doc.css('script[type="application/ld+json"]').map(&:text)
+
+        blobs.flat_map do |raw|
+          parsed = begin
+            JSON.parse(raw)
+          rescue JSON::ParserError
+            nil
+          end
+          next [] unless parsed
+
+          nodes = parsed.is_a?(Array) ? parsed : [parsed]
+          nodes.flat_map { |n| n['@graph'].is_a?(Array) ? n['@graph'] : [n] }
+        end
+      end
+
+      # Render a single JSON-LD +node+ as Markdown: a top-level title
+      # from +name+/+headline+, a bullet list of common useful fields
+      # (brand, SKU, price, rating, author, published date, ...), the
+      # body copy, and the lead image.
+      #
+      # When the node carries +articleBody+ (the full publisher-supplied
+      # article text), that wins over +description+ — the description
+      # is typically a lede teaser and would just repeat the article's
+      # opening lines.
+      #
+      # @param node [Hash] JSON-LD node, typically picked by
+      #   {.jsonld_section}
+      # @return [String] Markdown representation
+      def self.jsonld_to_markdown(node)
+        out = +''
+        name = node['name'] || node['headline']
+        out << "# #{name}\n\n" if name
+
+        offer  = first_obj(node['offers'])
+        rating = first_obj(node['aggregateRating'])
+        brand  = first_obj_or_string(node['brand'])
+        author = first_obj_or_string(node['author'])
+
+        brand_name  = brand.is_a?(Hash)  ? brand['name']  : brand
+        author_name = author.is_a?(Hash) ? author['name'] : author
+
+        fields = {
+          'Brand'        => brand_name,
+          'SKU'          => node['sku'],
+          'GTIN'         => node['gtin13'] || node['gtin'],
+          'Price'        => [offer['price'], offer['priceCurrency']].compact.join(' '),
+          'Availability' => offer['availability'],
+          'Rating'       => rating['ratingValue'],
+          'Reviews'      => rating['reviewCount'],
+          'Author'       => author_name,
+          'Published'    => node['datePublished']
+        }.reject { |_, v| v.nil? || v.to_s.strip.empty? }
+
+        unless fields.empty?
+          fields.each { |k, v| out << "- **#{k}:** #{v}\n" }
+          out << "\n"
+        end
+
+        if (body = node['articleBody'] || node['description'])
+          out << "#{body}\n\n"
+        end
+
+        if (img = node['image'])
+          img = img.first if img.is_a?(Array)
+          img = img['url'] if img.is_a?(Hash)
+          out << "![image](#{img})\n\n" if img
+        end
+
+        out
+      end
+
+      # Run +Readability+ over +html+ to isolate the main content node,
+      # then convert that to Markdown via +reverse_markdown+. The page
+      # +<title>+ is rendered as a top-level heading.
+      #
+      # When the page uses semantic HTML5 (+<main>+ or +<article>+) but
+      # leaves most of its content outside +<p>+ tags — divs, lists,
+      # spans — Readability's paragraph-density scoring collapses the
+      # extraction to a sliver of the page. In that case we render the
+      # +<main>+/+<article>+ container directly. The fallback only
+      # fires when the container holds substantially more text than
+      # Readability picked up (see {MAIN_FALLBACK_RATIO} /
+      # {MAIN_FALLBACK_MIN_CHARS}); on pages where both agree we keep
+      # Readability so its noise filtering still strips nav/ads/etc.
+      #
+      # @param html [String] HTML document body
+      # @return [String] Markdown representation
+      def self.readability_to_markdown(html)
+        rdoc = Readability::Document.new(
+          html,
+          tags: READABILITY_TAGS,
+          attributes: READABILITY_ATTRS,
+          remove_empty_nodes: true
+        )
+        readability_html = rdoc.content
+        title = rdoc.title
+
+        body_html = main_fallback_html(html, readability_html) || readability_html
+        body = ReverseMarkdown.convert(body_html, unknown_tags: :bypass, github_flavored: true)
+
+        out = +''
+        out << "# #{title.strip}\n\n" if title && !title.strip.empty?
+        out << body
+        out
+      end
+
+      # If +html+ has a +<main>+ or +<article>+ element holding
+      # substantially more text than Readability extracted, return that
+      # container's HTML so the caller can render it instead. Returns
+      # +nil+ when the fallback should not fire — when there is no
+      # semantic container, when it's too small to be meaningful, or
+      # when Readability's output is already comparable.
+      #
+      # @param html [String] full HTML document body, used to locate
+      #   the +<main>+/+<article>+ container
+      # @param readability_html [String] HTML produced by
+      #   +Readability::Document#content+, used as the comparison
+      #   baseline
+      # @return [String, nil] container HTML when the fallback should
+      #   fire, +nil+ otherwise
+      def self.main_fallback_html(html, readability_html)
+        doc = Nokogiri::HTML(html)
+        container = doc.at_css('main') || doc.at_css('article')
+        return nil unless container
+
+        container_text_len = container.text.gsub(/\s+/, ' ').strip.length
+        return nil if container_text_len < MAIN_FALLBACK_MIN_CHARS
+
+        readability_text_len = Nokogiri::HTML(readability_html).text.gsub(/\s+/, ' ').strip.length
+        return nil if container_text_len < MAIN_FALLBACK_RATIO * readability_text_len
+
+        container.to_html
+      end
+      private_class_method :main_fallback_html
+
+      # JSON-LD fields can be a string, hash, or array of either.
+      # Normalize to a single hash (the first one if it's a list) so
+      # callers can +.dig+ safely.
+      #
+      # @param value [Object] raw JSON-LD field value
+      # @return [Hash] empty hash when +value+ does not contain a hash
+      def self.first_obj(value)
+        value = value.first if value.is_a?(Array)
+        value.is_a?(Hash) ? value : {}
+      end
+      private_class_method :first_obj
+
+      # Same idea as {.first_obj} but preserves a bare string (e.g.
+      # +brand: "Apple"+) instead of replacing it with +{}+.
+      #
+      # @param value [Object] raw JSON-LD field value
+      # @return [String, Hash, nil]
+      def self.first_obj_or_string(value)
+        value = value.first if value.is_a?(Array)
+        value
+      end
+      private_class_method :first_obj_or_string
+    end
+  end
+end

data/lib/pikuri/extractor/passthrough.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Pikuri
+  module Extractor
+    # The terminal plain-text arm of the registry: content that *is*
+    # already text needs no extraction, so it passes through verbatim
+    # (forced to UTF-8 — invalid bytes are left in for downstream to
+    # deal with, matching what +File.read+ with a UTF-8 encoding does).
+    # Markdown, source files, JSON, robots.txt all land here.
+    #
+    # Matching is split by whether the transport supplied a
+    # content-type:
+    #
+    # * With a content-type (the web path): claim +text/*+ only.
+    #   A non-text type that no earlier extractor claimed is *not*
+    #   second-guessed by sniffing — a server declaring
+    #   +application/octet-stream+ gets the {Unsupported} refusal the
+    #   LLM can react to, same as before this registry existed.
+    # * Without one (the local-file path, where {FileType.detect_mime}
+    #   returned +nil+ for "unrecognised"): claim anything that passes
+    #   the {FileType.binary?} heuristic on the sample. Opaque
+    #   binaries stay unclaimed and surface as {Unsupported}.
+    module Passthrough
+      # @return [Symbol] {Page#kind} tag.
+      def self.kind
+        :text
+      end
+
+      # @param sample [String] leading bytes of the content.
+      # @param content_type [String, nil] normalized content-type,
+      #   +nil+ when the transport has none.
+      # @return [Boolean]
+      def self.matches?(sample:, content_type:)
+        return content_type.start_with?('text/') unless content_type.nil?
+
+        !FileType.binary?(sample)
+      end
+
+      # @param io [IO, StringIO] IO over the text content.
+      # @return [String] the content, tagged UTF-8. Deliberately NOT
+      #   derived from {.extract_lines} — a passthrough must stay
+      #   verbatim (trailing newline, CRLF line endings), which a
+      #   join of chomped lines would silently normalize away.
+      def self.extract(io)
+        io.read.force_encoding(Encoding::UTF_8)
+      end
+
+      # The lazy line stream for {Extractor.extract_paged}: the IO is
+      # read line-by-line, so a window over the head of a gigabyte
+      # log never loads the rest. Consuming the whole stream is a
+      # cheap sequential read — which is why the paging window counts
+      # this stream's tail for an exact +total_lines+ (see
+      # {Extractor.extract_paged}).
+      #
+      # @param io [IO, StringIO] IO over the text content; must
+      #   remain open while the enumerator is consumed.
+      # @return [Enumerator::Lazy<String>] chomped lines, tagged
+      #   UTF-8.
+      def self.extract_lines(io)
+        io.each_line.lazy.map { |raw| raw.chomp.force_encoding(Encoding::UTF_8) }
+      end
+    end
+  end
+end

data/lib/pikuri/extractor.rb

@@ -0,0 +1,314 @@
+# frozen_string_literal: true
+
+module Pikuri
+  # The format→text extraction seam: one registry of extractors that
+  # turn an +IO+ of some recognised format (HTML and plain text out
+  # of the box; PDF / office formats via the pikuri-pdf /
+  # pikuri-extractors plug-in gems) into Markdown-flavoured UTF-8
+  # text, consumed through two front doors:
+  #
+  # * {.extract} — the whole document as one String. The shape the
+  #   indexing / caching callers want ({Pikuri::VectorDb}'s indexer,
+  #   {Tool::WebScrape}'s URL cache): no windowing, no presentation.
+  # * {.extract_paged} — the LLM-tool shape: the same extraction,
+  #   windowed to a line range with a byte cap, returned as a {Page}
+  #   the caller renders. Backs +Workspace::Read+ and
+  #   +VectorDb::Tools::Read+ (via the {FileType} path wrappers) so
+  #   the offset/limit/byte-cap logic lives in one tested place.
+  #
+  # Both front doors — +Tool::Scraper+ dispatching on the HTTP
+  # +Content-Type+ header for the web tools, and {FileType} resolving
+  # local paths — route through this one registry, so both share one
+  # set of format truths and "support a new format" is a registry
+  # entry (pikuri-pdf and pikuri-extractors plug PDF and office
+  # formats in without pikuri-core knowing), not a new special case
+  # in two dispatchers.
+  #
+  # == The extractor duck type
+  #
+  # Each {.registry} entry implements three methods:
+  #
+  # * +matches?(sample:, content_type:)+ → +Boolean+ — claim the
+  #   content. +sample+ is the leading {FileType::SAMPLE_BYTES} bytes
+  #   (for magic-byte sniffs); +content_type+ is the normalized HTTP
+  #   +Content-Type+ for web content, the {FileType.detect_mime}
+  #   result for local files, and may be +nil+ ("no transport
+  #   metadata — sniff if you can").
+  # * +extract(io)+ → +String+ — the whole document as
+  #   Markdown-flavoured UTF-8 text. Raises {Error} on content the
+  #   extractor claimed but cannot parse (malformed PDF, ...).
+  # * +kind+ → +Symbol+ — a short tag (+:text+ / +:pdf+ / +:html+)
+  #   carried on {Page#kind} so rendering callers can word
+  #   format-specific trailers ("End of PDF", the scanned-image
+  #   hint) without re-sniffing.
+  #
+  # plus one *optional* method for formats whose lines can be
+  # produced incrementally:
+  #
+  # * +extract_lines(io)+ → +Enumerator<String>+ — the same content
+  #   as +extract+, as a lazy stream of already-+chomp+ed lines.
+  #   {.extract_paged} prefers this when present and stops consuming
+  #   the moment the window fills, so the rest of the document is
+  #   never parsed (pikuri-pdf's extractor: pdf-reader's page list
+  #   parses on access; {Passthrough}: the IO is read line-by-line).
+  #   The enumerator
+  #   must be consumed while +io+ is still open, and may raise
+  #   {Error} mid-iteration. Extractors that need the whole document
+  #   to produce anything ({HTML}: Readability walks the full DOM —
+  #   true of any subprocess-based extractor too) simply omit it;
+  #   {.extract_paged} then extracts in full and windows the result.
+  #
+  # Windowing itself (offset / limit / byte cap / line truncation) is
+  # presentation and deliberately lives once in {.extract_paged}, not
+  # per extractor — +extract_lines+ is line *production*, the only
+  # genuinely format-specific half of paging.
+  #
+  # == Errors
+  #
+  # Both failure modes are failures the *caller's* LLM can react to,
+  # so they share one rescuable root:
+  #
+  # * {Unsupported} — nothing in {.registry} claimed the content
+  #   (opaque binary, an unhandled content-type).
+  # * {Error} (the root) — an extractor claimed the content but the
+  #   parse failed (malformed PDF, ...).
+  #
+  # Callers map them to their own conventions:
+  # +Tool::Scraper+ re-raises both as +FetchError+;
+  # {FileType.read_as_text} maps {Unsupported} to the +ArgumentError+
+  # binary refusal and {Error} to a +RuntimeError+ carrying the path.
+  module Extractor
+    module_function
+
+    # Raised when an extractor claims content but fails to parse it
+    # (e.g. a malformed PDF). Message is LLM-presentable.
+    Error = Class.new(StandardError)
+
+    # Raised by {.extract} / {.extract_paged} when no registry entry
+    # claims the content. Subclass of {Error} so callers that don't
+    # care about the distinction rescue one class.
+    Unsupported = Class.new(Error)
+
+    # @return [Integer] default line-window size for {.extract_paged}
+    #   when the caller omits +limit+.
+    PAGE_DEFAULT_LIMIT = 2000
+
+    # @return [Integer] default hard byte cap on the content collected
+    #   by a single {.extract_paged} call. Bypassable by paging via
+    #   +offset+. The rendered output is slightly larger (line
+    #   numbering, trailer) — that's the caller's concern.
+    PAGE_MAX_BYTES = 50 * 1024
+
+    # @return [Integer] default per-line character cap;
+    #   {.extract_paged} truncates longer lines and appends
+    #   {PAGE_LINE_TRUNCATION_MARKER}.
+    PAGE_MAX_LINE_LENGTH = 2000
+
+    # @return [String] suffix appended to a line truncated at
+    #   {PAGE_MAX_LINE_LENGTH}.
+    PAGE_LINE_TRUNCATION_MARKER = "... (line truncated to #{PAGE_MAX_LINE_LENGTH} chars)"
+
+    # One windowed slice of a document, returned by {.extract_paged}.
+    # The caller turns this into an observation; this struct carries
+    # everything a trailer needs without the caller re-reading the
+    # document.
+    #
+    # == Fields
+    #
+    # * +lines+ — +Array<String>+, the collected window. Already
+    #   per-line truncated (with {PAGE_LINE_TRUNCATION_MARKER}); *not*
+    #   line-numbered — numbering is presentation the caller adds. For
+    #   a PDF the array includes the +"--- Page N ---"+ marker lines
+    #   pikuri-pdf's extractor emits, which count toward +limit+ / the
+    #   byte cap like any other line.
+    # * +start_line+ — the 1-indexed line number of +lines.first+
+    #   (i.e. the +offset+ the caller asked for). +lines.last+ is at
+    #   +start_line + lines.length - 1+.
+    # * +total_lines+ — total line count of the document when known,
+    #   else +nil+. Known when the read reached EOF, when the format
+    #   was extracted in full (no +extract_lines+ — e.g. HTML), or
+    #   when the lazy stream is cheap enough to count to the end
+    #   (plain text). +nil+ when a lazy stream stopped early — the
+    #   byte cap fired, or a PDF filled the window before its last
+    #   page (counting the rest would mean parsing every page,
+    #   defeating the laziness).
+    # * +more+ — +true+ if content remains past this window (the
+    #   caller should offer +offset = start_line + lines.length+).
+    # * +byte_capped+ — +true+ if the byte cap (not the line limit)
+    #   was the stopping criterion.
+    # * +kind+ — the matched extractor's +kind+ tag (+:text+ /
+    #   +:pdf+ / +:html+); lets the caller word format-specific
+    #   trailers and the empty-document message.
+    #
+    # An empty document yields +lines: []+, +total_lines: 0+; an
+    # +offset+ past EOF yields +lines: []+ with +total_lines+ set to
+    # the real (non-zero) count — the caller distinguishes the two.
+    Page = Data.define(:lines, :start_line, :total_lines, :more, :byte_capped, :kind)
+
+    # The extractor registry, consulted in order — first match wins.
+    # Core ships two entries: {HTML} matches on content-type, and
+    # {Passthrough} is the terminal plain-text arm. A gem adding a
+    # format picks its insertion point by the strength of its claim:
+    # a magic-byte sniff that never misfires on text goes at the
+    # *front* so it beats {HTML}'s content-type match even under a
+    # lying header (+registry.unshift(X)+ — pikuri-pdf does this);
+    # a content-type / weaker-sniff claimer inserts before the
+    # terminal entry (+registry.insert(-2, X)+ — pikuri-extractors
+    # does this).
+    #
+    # @return [Array<#matches?>] mutable, deliberately — this is the
+    #   plug-in seam.
+    def registry
+      @registry ||= [HTML, Passthrough]
+    end
+
+    # Extract the whole document behind +io+ as one Markdown-flavoured
+    # UTF-8 String. May be empty (empty text file, scanned-image PDF
+    # with no extractable text).
+    #
+    # @param io [IO, StringIO] seekable IO positioned at the start of
+    #   the content; this method reads a leading sample for the
+    #   +matches?+ sniff and rewinds before extracting.
+    # @param content_type [String, nil] normalized content-type when
+    #   the transport supplies one (HTTP header, {FileType.detect_mime}
+    #   result); +nil+ when unknown — extractors then rely on their
+    #   byte sniffs.
+    # @return [String]
+    # @raise [Unsupported] when no registry entry claims the content.
+    # @raise [Error] when the matched extractor cannot parse it.
+    def extract(io, content_type: nil)
+      extractor_for(io, content_type).extract(io)
+    end
+
+    # Extract +io+ and return a windowed {Page}: the lines from
+    # +offset+ (1-indexed) up to +limit+ of them, stopping early if
+    # +max_bytes+ is reached, with over-long lines truncated at
+    # +max_line_length+.
+    #
+    # Lazy where the format allows: extractors that implement
+    # +extract_lines+ (plain text; pikuri-pdf's PDF) are consumed
+    # only until the window fills — reading the first window of a
+    # 500-page PDF parses a handful of pages, and the first page of
+    # a gigabyte log never loads it. Extractors without it (HTML) are extracted
+    # in full and then windowed, which is also what makes their
+    # +total_lines+ always exact.
+    #
+    # @param io [IO, StringIO] seekable IO positioned at the start.
+    # @param content_type [String, nil] as for {.extract}.
+    # @param offset [Integer] 1-indexed first line to include. The
+    #   caller is responsible for validating +offset >= 1+.
+    # @param limit [Integer] maximum lines to collect. Caller
+    #   validates +limit >= 1+.
+    # @param max_bytes [Integer] hard byte cap on collected content.
+    # @param max_line_length [Integer] per-line truncation threshold.
+    # @return [Page] the windowed slice.
+    # @raise [Unsupported] when no registry entry claims the content.
+    # @raise [Error] when the matched extractor cannot parse it.
+    def extract_paged(io, content_type: nil, offset: 1, limit: PAGE_DEFAULT_LIMIT,
+                      max_bytes: PAGE_MAX_BYTES, max_line_length: PAGE_MAX_LINE_LENGTH)
+      extractor = extractor_for(io, content_type)
+      if extractor.respond_to?(:extract_lines)
+        # count_tail is a per-format economics call: once the window
+        # fills, counting the rest of a plain-text stream is a cheap
+        # sequential read (so the trailer can say "of N"), while for
+        # a PDF it would mean parsing every remaining page — exactly
+        # what extract_lines exists to avoid. Plugged-in extractors
+        # (pikuri-pdf's included) get the conservative default (stop
+        # early, total unknown).
+        window(extractor.extract_lines(io),
+               offset: offset, limit: limit, max_bytes: max_bytes,
+               max_line_length: max_line_length, kind: extractor.kind,
+               known_total: nil, count_tail: extractor.equal?(Passthrough))
+      else
+        lines = extractor.extract(io).split("\n")
+        window(lines, offset: offset, limit: limit, max_bytes: max_bytes,
+                      max_line_length: max_line_length, kind: extractor.kind,
+                      known_total: lines.length)
+      end
+    end
+
+    # Find the first registry entry claiming +io+'s content: read the
+    # leading {FileType::SAMPLE_BYTES} for the sniff, rewind, and ask
+    # each extractor in order.
+    #
+    # @param io [IO, StringIO] seekable IO positioned at the start.
+    # @param content_type [String, nil]
+    # @return [#extract] the matched extractor.
+    # @raise [Unsupported] when nothing matches.
+    def extractor_for(io, content_type)
+      sample = io.read(FileType::SAMPLE_BYTES) || +''
+      io.rewind
+      registry.find { |ex| ex.matches?(sample: sample, content_type: content_type) } ||
+        raise(Unsupported, 'no extractor for this content' \
+                           "#{content_type && !content_type.empty? ? " (content-type #{content_type.inspect})" : ''}")
+    end
+    private_class_method :extractor_for
+
+    # Collect a {Page} window out of +lines+ (an Array or a lazy
+    # Enumerator of already-+chomp+ed lines). +known_total+ is the
+    # full line count when the caller extracted everything up front
+    # (Array case), +nil+ for a lazy stream — then +total_lines+ is
+    # exact only if the iteration reached EOF: +count_tail+ keeps
+    # the loop counting (without collecting) past the line limit
+    # when consuming the rest of the stream is cheap; without it the
+    # loop breaks and leaves the total unknown. The byte cap always
+    # aborts the count.
+    #
+    # @param lines [Enumerable<String>]
+    # @param known_total [Integer, nil]
+    # @param count_tail [Boolean]
+    # @return [Page]
+    def window(lines, offset:, limit:, max_bytes:, max_line_length:, kind:,
+               known_total:, count_tail: false)
+      start_index   = offset - 1
+      collected     = []
+      seen          = 0
+      bytes         = 0
+      byte_capped   = false
+      more          = false
+      stopped_early = false
+
+      lines.each do |raw|
+        seen += 1
+        next if seen <= start_index
+
+        if collected.length >= limit
+          more = true
+          next if count_tail # keep counting so total_lines stays exact
+
+          stopped_early = true
+          break
+        end
+
+        line = truncate_line(raw, max_line_length)
+        size = line.bytesize + 1 # +1 for the joining newline
+        if bytes + size > max_bytes
+          byte_capped = true
+          more = true
+          stopped_early = true
+          break
+        end
+        collected << line
+        bytes += size
+      end
+
+      Page.new(lines: collected, start_line: offset,
+               total_lines: known_total || (stopped_early ? nil : seen),
+               more: more, byte_capped: byte_capped, kind: kind)
+    end
+    private_class_method :window
+
+    # Truncate +line+ to +max_line_length+ chars, appending
+    # {PAGE_LINE_TRUNCATION_MARKER} when it overflows.
+    #
+    # @param line [String]
+    # @param max_line_length [Integer]
+    # @return [String]
+    def truncate_line(line, max_line_length)
+      return line if line.length <= max_line_length
+
+      line[0, max_line_length] + PAGE_LINE_TRUNCATION_MARKER
+    end
+    private_class_method :truncate_line
+  end
+end