pikuri-core 0.0.4 → 0.0.6

data/lib/pikuri/agent.rb

@@ -94,8 +94,16 @@ module Pikuri
     #   queue is drained on every +after_tool_result+, each item
     #   appended as a +role: :user+ message and emitted as
     #   {Event::UserTurn} with +mid_loop: true+
+    # @param on_user_message [Proc, nil] when set, called with each
+    #   drained interloper +content+ String *after* it is appended
+    #   to the chat — the per-turn {Extension#on_user_message}
+    #   dispatch (prefetch + recording). Threaded through here rather
+    #   than fired inline so {Synthesizer.run}, which reuses this
+    #   wiring without an interloper or memory, simply passes +nil+.
+    #   Only consulted when +interloper+ is also set.
     # @return [void]
-    def self.wire_chat(chat, listeners:, step_limit: nil, cancellable: nil, interloper: nil)
+    def self.wire_chat(chat, listeners:, step_limit: nil, cancellable: nil, interloper: nil,
+                       on_user_message: nil)
       chat.after_message do |msg|
         emit_after_message(msg, listeners)
       end
@@ -106,7 +114,7 @@ module Pikuri
       end
       chat.after_tool_result do |result|
         listeners.emit(Event::ToolResult.new(content: result))
-        drain_interloper(interloper, chat, listeners) if interloper
+        drain_interloper(interloper, chat, listeners, on_user_message) if interloper
       end
     end
 
@@ -216,18 +224,29 @@ module Pikuri
 
     # Drain the interloper queue: for each pending item, append a
     # +role: :user+ message to the chat history so the next
-    # round-trip sees it, then emit an {Event::UserTurn} with
-    # +mid_loop: true+ to the listener stream so renderers see
-    # the injection.
+    # round-trip sees it, emit an {Event::UserTurn} with
+    # +mid_loop: true+ to the listener stream so renderers see the
+    # injection, then run the per-turn {Extension#on_user_message}
+    # dispatch (so mid-loop injections are prefetched + recorded
+    # exactly like initial turns).
+    #
+    # The dispatch runs *after* the +:user+ append so any
+    # +<memory-context>+ it injects lands as a +:system+ message
+    # right behind the user turn it annotates — the same
+    # append-at-the-tail ordering {#run_loop} produces for initial
+    # turns.
     #
     # @param interloper [Control::Interloper]
     # @param chat [RubyLLM::Chat]
     # @param listeners [ListenerList]
+    # @param on_user_message [Proc, nil] per-content dispatch; +nil+
+    #   skips it (e.g. an interloper with no memory extension wired)
     # @return [void]
-    def self.drain_interloper(interloper, chat, listeners)
+    def self.drain_interloper(interloper, chat, listeners, on_user_message = nil)
       interloper.drain!.each do |content|
         chat.add_message(role: :user, content: content)
         listeners.emit(Event::UserTurn.new(content: content, mid_loop: true))
+        on_user_message&.call(content)
       end
     end
     private_class_method :drain_interloper
@@ -381,71 +400,31 @@ module Pikuri
       @streaming = streaming
       @synth_answer = nil
       @on_close_handlers = []
-
-      # Single Configurator funnel for everything the block adds —
-      # tools, listeners, system-prompt snippets, extensions, and
-      # on_close handlers. See {Configurator} for the per-method
-      # contract.
-      configurator = Configurator.new(
-        transport: @transport,
-        system_prompt_base: system_prompt,
-        id: @id,
-        streaming: @streaming,
-        step_limit: @step_limit,
-        cancellable: @cancellable,
-        interloper: @interloper
-      )
-
-      block&.call(configurator)
-
-      @tools = configurator.tools.dup
-      @sub_agent_tools = configurator.sub_agent_tools.dup
-      @listeners = ListenerList.new(configurator.listeners)
-      configurator.system_prompt_additions.each do |snippet|
-        @system_prompt = "#{@system_prompt}\n\n#{snippet}"
+      # Stashed for {#run_configure}, which runs the failure-prone
+      # build phase below out of a separate method.
+      @block = block
+      @context_window = context_window
+      @llama_probe_url = llama_probe_url
+
+      # Register *before* the build phase so a mid-construction raise
+      # is still recoverable: extensions arm their cleanup via
+      # +c.on_close+ (which writes straight to +@on_close_handlers+,
+      # see {Configurator}), and the rescue below fires whatever was
+      # armed before the failure. On the happy path this registration
+      # is the at-exit backstop if the host forgets {#close}; an
+      # explicit {#close} unregisters, so the agent isn't pinned alive
+      # until process exit.
+      Pikuri::Finalizers.register(self)
+
+      begin
+        run_configure
+      rescue StandardError
+        # Half-built agent (e.g. an extension's +configure+ raised
+        # Cancelled mid-spawn). Fire the handlers armed so far, drop
+        # out of the registry, and re-raise — no partial state leaks.
+        close
+        raise
       end
-      @on_close_handlers.concat(configurator.on_close_handlers)
-      @extensions = configurator.extensions.dup
-
-      @chat = RubyLLM.chat(**@transport.to_h)
-      @chat.with_instructions(@system_prompt)
-      @tools.each { |t| @chat.with_tool(t.to_ruby_llm_tool) }
-
-      @context_window_cap = ContextWindowDetector.new(
-        override: context_window,
-        ruby_llm_reported: @chat.model.context_window,
-        llama_probe_url: llama_probe_url
-      ).detect
-
-      self.class.wire_chat(
-        @chat,
-        listeners: @listeners,
-        step_limit: @step_limit,
-        cancellable: @cancellable,
-        interloper: @interloper
-      )
-
-      # One-shot context-window cap: lets every listener that
-      # cares (notably TokenLog) pick the value off the stream
-      # before any Tokens event arrives.
-      @listeners.emit(Event::ContextCap.new(cap: @context_window_cap))
-
-      # Bind sweep — each extension gets its chance to install
-      # per-agent state (dynamic tools via #internal_add_tool,
-      # per-agent close hooks via #on_close, etc.) now that the
-      # chat is fully wired. See IDEAS.md §"Extension protocol
-      # design" for what #configure vs #bind are each for.
-      @extensions.each { |ext| ext.bind(self) }
-
-      # Fallback cleanup: if the host forgets to call #close, the
-      # at_exit hook fires it on process exit. Idempotent, so an
-      # explicit close earlier makes this a no-op. The closure
-      # captures self, which keeps the agent reachable until
-      # process exit — fine for the handful of agents a typical
-      # host creates; if pikuri grows a long-running host that
-      # constructs many short-lived agents, switch to a single
-      # process-global registry that close-then-removes.
-      at_exit { close }
     end
 
     # @return [RubyLLM::Chat] underlying chat; the extension seam
@@ -601,13 +580,23 @@ module Pikuri
         if user_message.nil? || user_message.to_s.strip.empty?
 
       @synth_answer = nil
-      @listeners.emit(Event::UserTurn.new(content: user_message, mid_loop: false))
       @step_limit&.reset!
       @cancellable&.reset!
+      # Append the user turn, emit it, then run the memory dispatch — so
+      # any <memory-context> the dispatch injects lands as a :system
+      # message *after* the user turn it annotates (append-only at the
+      # tail; see {#dispatch_ext_on_user_message}). `ask` would bundle the
+      # user-message append with completion atomically, leaving no seam to
+      # inject between them, so the two halves run explicitly here:
+      # add_message + complete (the exact pair `ask` is sugar for). A raw
+      # String content matches the interloper drain path.
+      @chat.add_message(role: :user, content: user_message)
+      @listeners.emit(Event::UserTurn.new(content: user_message, mid_loop: false))
+      dispatch_ext_on_user_message(user_message)
       if @streaming
-        @chat.ask(user_message, &self.class.streaming_block(listeners: @listeners, cancellable: @cancellable))
+        @chat.complete(&self.class.streaming_block(listeners: @listeners, cancellable: @cancellable))
       else
-        @chat.ask(user_message)
+        @chat.complete
       end
       nil
     rescue Control::Cancellable::Cancelled
@@ -661,6 +650,10 @@ module Pikuri
       return if @closed
 
       @closed = true
+      # Drop out of the process-global registry first: a deliberate
+      # close means this agent no longer needs the at-exit fallback,
+      # and removing the reference lets it be garbage-collected.
+      Pikuri::Finalizers.unregister(self)
       @on_close_handlers.reverse_each do |handler|
         handler.call
       rescue StandardError => e
@@ -719,5 +712,113 @@ module Pikuri
     def to_s
       "Agent(id=#{@id}, model=#{model}, tools=#{@tools.size}, listeners=#{@listeners})"
     end
+
+    private
+
+    # The failure-prone build phase, split out of {#initialize} so the
+    # constructor can wrap it in a rescue and self-heal. Funnels the
+    # +Agent.new+ block through a single {Configurator} — tools,
+    # listeners, system-prompt snippets, extensions, and +on_close+
+    # handlers — then wires the chat and runs the extension +bind+
+    # sweep. The Configurator's +on_close_sink:+ is +@on_close_handlers+
+    # itself, so a handler an extension arms via +c.on_close+ is live on
+    # the agent the instant it's registered — that's what lets the
+    # constructor's rescue close a half-built agent.
+    #
+    # @return [void]
+    def run_configure
+      configurator = Configurator.new(
+        transport: @transport,
+        system_prompt_base: @system_prompt,
+        id: @id,
+        streaming: @streaming,
+        step_limit: @step_limit,
+        cancellable: @cancellable,
+        interloper: @interloper,
+        on_close_sink: @on_close_handlers
+      )
+
+      @block&.call(configurator)
+
+      @tools = configurator.tools.dup
+      @sub_agent_tools = configurator.sub_agent_tools.dup
+      @listeners = ListenerList.new(configurator.listeners)
+      configurator.system_prompt_additions.each do |snippet|
+        @system_prompt = "#{@system_prompt}\n\n#{snippet}"
+      end
+      @extensions = configurator.extensions.dup
+
+      @chat = RubyLLM.chat(**@transport.to_h)
+      @chat.with_instructions(@system_prompt)
+      @tools.each { |t| @chat.with_tool(t.to_ruby_llm_tool) }
+
+      @context_window_cap = ContextWindowDetector.new(
+        override: @context_window,
+        ruby_llm_reported: @chat.model.context_window,
+        llama_probe_url: @llama_probe_url,
+        model_id: @chat.model.id
+      ).detect
+
+      self.class.wire_chat(
+        @chat,
+        listeners: @listeners,
+        step_limit: @step_limit,
+        cancellable: @cancellable,
+        interloper: @interloper,
+        on_user_message: method(:dispatch_ext_on_user_message)
+      )
+
+      # One-shot context-window cap: lets every listener that
+      # cares (notably TokenLog) pick the value off the stream
+      # before any Tokens event arrives.
+      @listeners.emit(Event::ContextCap.new(cap: @context_window_cap))
+
+      # Bind sweep — each extension gets its chance to install
+      # per-agent state (dynamic tools via #internal_add_tool,
+      # per-agent close hooks via #on_close, etc.) now that the
+      # chat is fully wired. See IDEAS.md §"Extension protocol
+      # design" for what #configure vs #bind are each for.
+      @extensions.each { |ext| ext.bind(self) }
+    end
+
+    # Fire the per-turn {Extension#on_user_message} hook on every
+    # extension that defines it, appending any returned
+    # +<memory-context>+ block to the chat as a +role: :system+
+    # message right after the user turn it annotates (callers append
+    # the +:user+ message first; this runs last). The system role is
+    # load-bearing — it tags the block as recalled reference (not new
+    # input) and keeps it excludable from a later extraction pass.
+    # See {Extension#on_user_message}.
+    #
+    # Each injected block also emits an {Event::SystemInjected} at
+    # this site, so the listener stream mirrors the log growth (the
+    # Terminal renders it; otherwise an injection would be invisible
+    # except as a downstream echo in the assistant's reasoning).
+    #
+    # Private and the single place the chat log grows by a memory
+    # block — keeps "what mutates the log, when" one grep in this
+    # file. Fired from {#run_loop} (initial turn) and, via the
+    # +on_user_message:+ proc threaded into {.wire_chat}, from
+    # {.drain_interloper} (mid-loop interlopers). Called on every
+    # extension unconditionally — same as {Extension#configure} /
+    # {Extension#bind}: the hook is part of the protocol and the
+    # {Extension} module supplies a no-op default, so any extension
+    # that includes the module responds. An extension is "opted out"
+    # by leaving the default in place (it returns +nil+, injecting
+    # nothing), not by omitting the method.
+    #
+    # @param content [String] the incoming user message
+    # @return [void]
+    def dispatch_ext_on_user_message(content)
+      @extensions.each do |ext|
+        message = ext.on_user_message(self, content)
+        next unless message.is_a?(String) && !message.strip.empty?
+
+        block = message.strip
+        @chat.add_message(role: :system, content: block)
+        @listeners.emit(Event::SystemInjected.new(content: block))
+      end
+      nil
+    end
   end
 end

data/lib/pikuri/extractor/html.rb

@@ -0,0 +1,303 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'nokogiri'
+require 'readability'
+require 'reverse_markdown'
+
+module Pikuri
+  module Extractor
+    # HTML → Markdown extractor.
+    #
+    # Matched by content-type only (+text/html+ /
+    # +application/xhtml+xml+) — deliberately no byte sniff. The web
+    # path always has the header; for local files a sniff would route
+    # +Workspace::Read+ of an +.html+ source file through readability
+    # extraction, when a developer reading an HTML file wants the
+    # source. Local HTML stays on the {Passthrough} arm until a
+    # consumer genuinely needs otherwise.
+    #
+    # Always renders both views of the page when available:
+    #
+    # 1. JSON-LD section. Any +<script type="application/ld+json">+ node
+    #    whose +@type+ matches a substantive schema.org content type
+    #    (Product, Article, Recipe, ...) is rendered as a header — title,
+    #    metadata bullets (brand, SKU, price, rating, author, published),
+    #    and the +articleBody+/+description+ copy when present.
+    # 2. Readability section. The page is run through +Readability+ +
+    #    +reverse_markdown+, with a +<main>+/+<article>+ fallback for
+    #    pages whose content sits mostly outside +<p>+ tags.
+    #
+    # Concatenated with a horizontal rule, so the LLM gets both the
+    # structured metadata and the rendered body and can pick whichever
+    # is more useful for the task. Trades some duplication (when a
+    # publisher embeds the article body in JSON-LD AND in HTML) for
+    # fewer type-based heuristics on which branch should win — the
+    # earlier "is this Article's +description+ a teaser or the real
+    # body?" carve-out is no longer needed because both end up in
+    # the output regardless.
+    module HTML
+      # @return [Array<String>] content-types this extractor claims.
+      CONTENT_TYPES = %w[text/html application/xhtml+xml].freeze
+
+      # @return [Array<String>] schema.org +@type+ values that we treat
+      #   as "the primary entity of this page" when picking a JSON-LD
+      #   node to render. Order does not matter — the first matching
+      #   node wins. Skips noise nodes (Organization, BreadcrumbList,
+      #   WebSite, ...) that ship on most pages but carry no page
+      #   content.
+      INTERESTING_TYPES = %w[
+        Product Article NewsArticle BlogPosting Recipe Event Book Movie
+      ].freeze
+
+      # @return [Array<String>] HTML tags preserved by the readability
+      #   pass. Anything outside this list is stripped before Markdown
+      #   conversion.
+      READABILITY_TAGS = %w[
+        h1 h2 h3 h4 h5 h6 p div span ul ol li blockquote pre code a img
+        strong em b i br hr table thead tbody tr td th
+      ].freeze
+
+      # @return [Array<String>] HTML attributes preserved by the
+      #   readability pass; everything else (class, id, style, data-*)
+      #   is dropped before Markdown conversion
+      READABILITY_ATTRS = %w[href src alt title].freeze
+
+      # @return [Float] minimum +<main>+/+<article>+ to Readability
+      #   text-length ratio that triggers the semantic-container
+      #   fallback in {.readability_to_markdown}. Picked low enough to
+      #   catch the failure mode (Readability collapsing a page that
+      #   uses divs/lists instead of +<p>+ — e.g. +vaadin.com/company+,
+      #   ~5x) but high enough that pages where both produce
+      #   comparable output keep Readability's noise filtering.
+      MAIN_FALLBACK_RATIO = 2.0
+
+      # @return [Integer] minimum text length the
+      #   +<main>+/+<article>+ container must hold before the fallback
+      #   in {.readability_to_markdown} can fire. Below this, the
+      #   ratio comparison is dominated by noise and we'd swap on
+      #   tiny pages where Readability is doing the right thing.
+      MAIN_FALLBACK_MIN_CHARS = 500
+
+      # @return [Symbol] {Page#kind} tag.
+      def self.kind
+        :html
+      end
+
+      # @param sample [String] leading bytes of the content (unused —
+      #   see the no-sniff rationale in the module doc).
+      # @param content_type [String, nil] normalized content-type.
+      # @return [Boolean]
+      def self.matches?(sample:, content_type:)
+        CONTENT_TYPES.include?(content_type)
+      end
+
+      # Render the HTML document behind +io+ as Markdown by emitting
+      # both the JSON-LD section (when an interesting node is present)
+      # and the readability / +<main>+ section, joined by a horizontal
+      # rule. Either section may be missing — pages with no JSON-LD
+      # return only the readability output, and a malformed page with
+      # no extractable body returns only the JSON-LD render.
+      #
+      # @param io [IO, StringIO] IO over the HTML document.
+      # @return [String] Markdown representation
+      def self.extract(io)
+        html = io.read
+        sections = [jsonld_section(html), readability_to_markdown(html)]
+        sections.reject! { |s| s.nil? || s.strip.empty? }
+        sections.join("\n\n---\n\n")
+      end
+
+      # Pick the first JSON-LD node whose +@type+ matches one of
+      # {INTERESTING_TYPES} and render it as Markdown. Returns +nil+
+      # when no such node exists, in which case {.extract} emits only
+      # the readability section.
+      #
+      # No content-field gating: a node carrying just +name+/+author+/
+      # +datePublished+ still renders (as a metadata-only header),
+      # because the readability pass independently produces the page
+      # body. That is the trade-off that lets us drop the type-based
+      # "is this teaser or article copy?" heuristics — duplication is
+      # acceptable when both views are available, and the LLM can
+      # pick whichever it needs.
+      #
+      # @param html [String] HTML document body
+      # @return [String, nil] Markdown render of the picked JSON-LD
+      #   node, or +nil+ when nothing matched
+      def self.jsonld_section(html)
+        node = parse_jsonld(html).find do |n|
+          Array(n['@type']).any? { |t| INTERESTING_TYPES.include?(t) }
+        end
+        node ? jsonld_to_markdown(node) : nil
+      end
+
+      # Collect every JSON-LD payload embedded in +html+, flattening
+      # +@graph+ wrappers so callers see one flat array of schema.org
+      # nodes. Malformed JSON blocks are silently skipped — sites
+      # frequently ship broken JSON-LD and we only need at least one
+      # parseable block.
+      #
+      # @param html [String] HTML document body
+      # @return [Array<Hash>] parsed JSON-LD nodes; possibly empty
+      def self.parse_jsonld(html)
+        doc = Nokogiri::HTML(html)
+        blobs = doc.css('script[type="application/ld+json"]').map(&:text)
+
+        blobs.flat_map do |raw|
+          parsed = begin
+            JSON.parse(raw)
+          rescue JSON::ParserError
+            nil
+          end
+          next [] unless parsed
+
+          nodes = parsed.is_a?(Array) ? parsed : [parsed]
+          nodes.flat_map { |n| n['@graph'].is_a?(Array) ? n['@graph'] : [n] }
+        end
+      end
+
+      # Render a single JSON-LD +node+ as Markdown: a top-level title
+      # from +name+/+headline+, a bullet list of common useful fields
+      # (brand, SKU, price, rating, author, published date, ...), the
+      # body copy, and the lead image.
+      #
+      # When the node carries +articleBody+ (the full publisher-supplied
+      # article text), that wins over +description+ — the description
+      # is typically a lede teaser and would just repeat the article's
+      # opening lines.
+      #
+      # @param node [Hash] JSON-LD node, typically picked by
+      #   {.jsonld_section}
+      # @return [String] Markdown representation
+      def self.jsonld_to_markdown(node)
+        out = +''
+        name = node['name'] || node['headline']
+        out << "# #{name}\n\n" if name
+
+        offer  = first_obj(node['offers'])
+        rating = first_obj(node['aggregateRating'])
+        brand  = first_obj_or_string(node['brand'])
+        author = first_obj_or_string(node['author'])
+
+        brand_name  = brand.is_a?(Hash)  ? brand['name']  : brand
+        author_name = author.is_a?(Hash) ? author['name'] : author
+
+        fields = {
+          'Brand'        => brand_name,
+          'SKU'          => node['sku'],
+          'GTIN'         => node['gtin13'] || node['gtin'],
+          'Price'        => [offer['price'], offer['priceCurrency']].compact.join(' '),
+          'Availability' => offer['availability'],
+          'Rating'       => rating['ratingValue'],
+          'Reviews'      => rating['reviewCount'],
+          'Author'       => author_name,
+          'Published'    => node['datePublished']
+        }.reject { |_, v| v.nil? || v.to_s.strip.empty? }
+
+        unless fields.empty?
+          fields.each { |k, v| out << "- **#{k}:** #{v}\n" }
+          out << "\n"
+        end
+
+        if (body = node['articleBody'] || node['description'])
+          out << "#{body}\n\n"
+        end
+
+        if (img = node['image'])
+          img = img.first if img.is_a?(Array)
+          img = img['url'] if img.is_a?(Hash)
+          out << "![image](#{img})\n\n" if img
+        end
+
+        out
+      end
+
+      # Run +Readability+ over +html+ to isolate the main content node,
+      # then convert that to Markdown via +reverse_markdown+. The page
+      # +<title>+ is rendered as a top-level heading.
+      #
+      # When the page uses semantic HTML5 (+<main>+ or +<article>+) but
+      # leaves most of its content outside +<p>+ tags — divs, lists,
+      # spans — Readability's paragraph-density scoring collapses the
+      # extraction to a sliver of the page. In that case we render the
+      # +<main>+/+<article>+ container directly. The fallback only
+      # fires when the container holds substantially more text than
+      # Readability picked up (see {MAIN_FALLBACK_RATIO} /
+      # {MAIN_FALLBACK_MIN_CHARS}); on pages where both agree we keep
+      # Readability so its noise filtering still strips nav/ads/etc.
+      #
+      # @param html [String] HTML document body
+      # @return [String] Markdown representation
+      def self.readability_to_markdown(html)
+        rdoc = Readability::Document.new(
+          html,
+          tags: READABILITY_TAGS,
+          attributes: READABILITY_ATTRS,
+          remove_empty_nodes: true
+        )
+        readability_html = rdoc.content
+        title = rdoc.title
+
+        body_html = main_fallback_html(html, readability_html) || readability_html
+        body = ReverseMarkdown.convert(body_html, unknown_tags: :bypass, github_flavored: true)
+
+        out = +''
+        out << "# #{title.strip}\n\n" if title && !title.strip.empty?
+        out << body
+        out
+      end
+
+      # If +html+ has a +<main>+ or +<article>+ element holding
+      # substantially more text than Readability extracted, return that
+      # container's HTML so the caller can render it instead. Returns
+      # +nil+ when the fallback should not fire — when there is no
+      # semantic container, when it's too small to be meaningful, or
+      # when Readability's output is already comparable.
+      #
+      # @param html [String] full HTML document body, used to locate
+      #   the +<main>+/+<article>+ container
+      # @param readability_html [String] HTML produced by
+      #   +Readability::Document#content+, used as the comparison
+      #   baseline
+      # @return [String, nil] container HTML when the fallback should
+      #   fire, +nil+ otherwise
+      def self.main_fallback_html(html, readability_html)
+        doc = Nokogiri::HTML(html)
+        container = doc.at_css('main') || doc.at_css('article')
+        return nil unless container
+
+        container_text_len = container.text.gsub(/\s+/, ' ').strip.length
+        return nil if container_text_len < MAIN_FALLBACK_MIN_CHARS
+
+        readability_text_len = Nokogiri::HTML(readability_html).text.gsub(/\s+/, ' ').strip.length
+        return nil if container_text_len < MAIN_FALLBACK_RATIO * readability_text_len
+
+        container.to_html
+      end
+      private_class_method :main_fallback_html
+
+      # JSON-LD fields can be a string, hash, or array of either.
+      # Normalize to a single hash (the first one if it's a list) so
+      # callers can +.dig+ safely.
+      #
+      # @param value [Object] raw JSON-LD field value
+      # @return [Hash] empty hash when +value+ does not contain a hash
+      def self.first_obj(value)
+        value = value.first if value.is_a?(Array)
+        value.is_a?(Hash) ? value : {}
+      end
+      private_class_method :first_obj
+
+      # Same idea as {.first_obj} but preserves a bare string (e.g.
+      # +brand: "Apple"+) instead of replacing it with +{}+.
+      #
+      # @param value [Object] raw JSON-LD field value
+      # @return [String, Hash, nil]
+      def self.first_obj_or_string(value)
+        value = value.first if value.is_a?(Array)
+        value
+      end
+      private_class_method :first_obj_or_string
+    end
+  end
+end

data/lib/pikuri/extractor/passthrough.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Pikuri
+  module Extractor
+    # The terminal plain-text arm of the registry: content that *is*
+    # already text needs no extraction, so it passes through verbatim
+    # (forced to UTF-8 — invalid bytes are left in for downstream to
+    # deal with, matching what +File.read+ with a UTF-8 encoding does).
+    # Markdown, source files, JSON, robots.txt all land here.
+    #
+    # Matching is split by whether the transport supplied a
+    # content-type:
+    #
+    # * With a content-type (the web path): claim +text/*+ only.
+    #   A non-text type that no earlier extractor claimed is *not*
+    #   second-guessed by sniffing — a server declaring
+    #   +application/octet-stream+ gets the {Unsupported} refusal the
+    #   LLM can react to, same as before this registry existed.
+    # * Without one (the local-file path, where {FileType.detect_mime}
+    #   returned +nil+ for "unrecognised"): claim anything that passes
+    #   the {FileType.binary?} heuristic on the sample. Opaque
+    #   binaries stay unclaimed and surface as {Unsupported}.
+    module Passthrough
+      # @return [Symbol] {Page#kind} tag.
+      def self.kind
+        :text
+      end
+
+      # @param sample [String] leading bytes of the content.
+      # @param content_type [String, nil] normalized content-type,
+      #   +nil+ when the transport has none.
+      # @return [Boolean]
+      def self.matches?(sample:, content_type:)
+        return content_type.start_with?('text/') unless content_type.nil?
+
+        !FileType.binary?(sample)
+      end
+
+      # @param io [IO, StringIO] IO over the text content.
+      # @return [String] the content, tagged UTF-8. Deliberately NOT
+      #   derived from {.extract_lines} — a passthrough must stay
+      #   verbatim (trailing newline, CRLF line endings), which a
+      #   join of chomped lines would silently normalize away.
+      def self.extract(io)
+        io.read.force_encoding(Encoding::UTF_8)
+      end
+
+      # The lazy line stream for {Extractor.extract_paged}: the IO is
+      # read line-by-line, so a window over the head of a gigabyte
+      # log never loads the rest. Consuming the whole stream is a
+      # cheap sequential read — which is why the paging window counts
+      # this stream's tail for an exact +total_lines+ (see
+      # {Extractor.extract_paged}).
+      #
+      # @param io [IO, StringIO] IO over the text content; must
+      #   remain open while the enumerator is consumed.
+      # @return [Enumerator::Lazy<String>] chomped lines, tagged
+      #   UTF-8.
+      def self.extract_lines(io)
+        io.each_line.lazy.map { |raw| raw.chomp.force_encoding(Encoding::UTF_8) }
+      end
+    end
+  end
+end