pikuri-core 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 914f6e02052e97773e630e32bc9689bf7edc0eee4e962419aa41a15ca4afa667
-  data.tar.gz: e9a68c3813a0bbcd292757ee2cb4dbce6df7abc890e0d4afa6cf51cae77a6247
+  metadata.gz: ac822a7bd46228f2eea2994c2e1428e3aa90c269e6ebafd603474fb630ba34ee
+  data.tar.gz: 00c69d139bc38c1a881bf87980970672517db51f59469aef266009a803a874db
 SHA512:
-  metadata.gz: 3f0bdf7af9a4a85d3669b01f0929b92fdb2acbc9c7cb083611c4acaecc4d3b5b37a3e3d97a1fd060bec143016f4db00968707d149f127eb1dea5a7243dd51a73
-  data.tar.gz: cf92ad370fed6574f9cd7cc44fbd93e36c6bb03c6d56555267dc0a22a723ca9ce298231495b902570bfbbdde6b5e6b319860b2b625818eb2bdbb51d5dd2346e2
+  metadata.gz: d148d78b2027d747ef10f4dd7a19252f66bb1b5f99e8eef763149bf3e93ae608a3f7cf739b02ed4d92ab3ab39d8fe6888615a5acaf90dcb44ca783058a95a716
+  data.tar.gz: 32ef75bbd6d825970e5a1e6b5e27cf0fc812980e87e5ddd59b14f2c10772c91cf93003c2ebe2c9501f5e9682e726f77c4c387c7c83b467b9aa57dc91a9a178f0

data/lib/pikuri/agent/listener/terminal.rb

@@ -1,21 +1,32 @@
 # frozen_string_literal: true
 
 require 'rainbow'
-require 'tty-markdown'
 
 module Pikuri
   class Agent
     module Listener
       # Terminal renderer for the normalized event stream: dim grey
-      # reasoning, Markdown-rendered assistant content, cyan tool-
-      # call and tool-result lines, yellow fallback notice, red
-      # cancelled notice. An {Event::SystemInjected} block (recalled
+      # reasoning, assistant content printed raw (Markdown as-is),
+      # cyan tool-call and tool-result lines, yellow fallback
+      # notice, red cancelled notice. An {Event::SystemInjected} block (recalled
       # memory / context an extension injected) renders dim grey
       # with a +⊕+ marker. {Event::UserTurn} is intentionally silent
       # (the terminal user just typed the message, so re-rendering
       # it adds nothing); {Event::Tokens} and {Event::ContextCap}
       # are silent too (their consumer is {TokenLog}).
       #
+      # Assistant Markdown deliberately prints raw, with no
+      # Markdown-to-ANSI rendering. A renderer (+tty-markdown+)
+      # used to sit on the non-streaming path; it was dropped:
+      # rendering can never apply to the streaming path anyway
+      # (half-finished Markdown — broken code fences, half-built
+      # tables — doesn't render), the gem hadn't shipped a release
+      # since 2023 (its known ANSI-in-table crashes forced a
+      # rescue-and-degrade carve-out here), and it pulled seven
+      # transitive gems into the audit surface. Raw Markdown is
+      # perfectly readable in a terminal; proper rendering belongs
+      # to a richer host (the planned pikuri-tui).
+      #
       # Optionally prepends a fixed number of leading spaces to
       # every rendered line via the +padding:+ kwarg. Sub-agents
       # get a fresh padded instance through {#for_sub_agent}
@@ -30,11 +41,8 @@ module Pikuri
       # - {Event::ThinkingDelta} fragments print live in the same
       #   dim grey as the non-streaming {Event::Thinking}, with no
       #   trailing newline so the next fragment continues the line.
-      # - {Event::AssistantDelta} fragments print live, *raw* — no
-      #   Markdown render. tty-markdown can't render half-finished
-      #   Markdown (broken code blocks, half-rendered tables), so
-      #   the live stream gives up formatting in exchange for
-      #   liveness.
+      # - {Event::AssistantDelta} fragments print live the same
+      #   way, uncolored.
       # - {Event::Thinking} and {Event::Assistant} bookends print
       #   a single blank line as a stream terminator, not their
       #   content (the content already landed via the deltas). The
@@ -45,15 +53,6 @@ module Pikuri
       # the deltas are silently ignored and the bookend events
       # render the full text the way they always have.
       class Terminal < Base
-        # Subsystem logger; set its level with +PIKURI_LOG_TERMINAL+
-        # or the global +PIKURI_LOG+. Used for the narrow rescue
-        # around third-party rendering (+tty-markdown+ choking on
-        # assistant output) — see the CLAUDE.md "secondary to the
-        # loop" carve-out.
-        #
-        # @return [Logger]
-        LOGGER = Pikuri.logger_for('Terminal')
-
         # Cap, in characters, applied to tool-result content
         # rendered to the terminal. Anything longer is truncated
         # with a marker that reports the original byte size so the
@@ -119,7 +118,7 @@ module Pikuri
             if @streaming
               terminate_stream
             else
-              println(indent(render_markdown(content)))
+              println(indent(content))
             end
           in Event::ThinkingDelta(content:)
             stream_fragment(Rainbow(content).color(85, 85, 85)) if @streaming
@@ -228,23 +227,6 @@ module Pikuri
           text.to_s.each_line.map { |line| prefix + line }.join
         end
 
-        # Render assistant Markdown for the terminal, degrading to
-        # the raw string when the renderer raises. tty-markdown /
-        # strings have known bugs around ANSI inside tables (e.g.
-        # +Strings::Wrap.insert_ansi+ raising +IndexError+); we'd
-        # rather show ugly Markdown than abort an in-flight
-        # conversation.
-        #
-        # @param content [String] assistant Markdown
-        # @return [String] rendered ANSI text, or +content+
-        #   unchanged on render failure
-        def render_markdown(content)
-          TTY::Markdown.parse(content)
-        rescue StandardError => e
-          LOGGER.warn("TTY::Markdown render failed (#{e.class}: #{e.message}); falling back to raw text")
-          content
-        end
-
         # Flatten whitespace and cap to {MAX_TOOL_RESULT_CHARS}. The
         # cap keeps multi-screen dumps (rendered HTML, PDF text)
         # from drowning the terminal stream; the byte-count suffix

data/lib/pikuri/extractor/html.rb

@@ -0,0 +1,303 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'nokogiri'
+require 'readability'
+require 'reverse_markdown'
+
+module Pikuri
+  module Extractor
+    # HTML → Markdown extractor.
+    #
+    # Matched by content-type only (+text/html+ /
+    # +application/xhtml+xml+) — deliberately no byte sniff. The web
+    # path always has the header; for local files a sniff would route
+    # +Workspace::Read+ of an +.html+ source file through readability
+    # extraction, when a developer reading an HTML file wants the
+    # source. Local HTML stays on the {Passthrough} arm until a
+    # consumer genuinely needs otherwise.
+    #
+    # Always renders both views of the page when available:
+    #
+    # 1. JSON-LD section. Any +<script type="application/ld+json">+ node
+    #    whose +@type+ matches a substantive schema.org content type
+    #    (Product, Article, Recipe, ...) is rendered as a header — title,
+    #    metadata bullets (brand, SKU, price, rating, author, published),
+    #    and the +articleBody+/+description+ copy when present.
+    # 2. Readability section. The page is run through +Readability+ +
+    #    +reverse_markdown+, with a +<main>+/+<article>+ fallback for
+    #    pages whose content sits mostly outside +<p>+ tags.
+    #
+    # Concatenated with a horizontal rule, so the LLM gets both the
+    # structured metadata and the rendered body and can pick whichever
+    # is more useful for the task. Trades some duplication (when a
+    # publisher embeds the article body in JSON-LD AND in HTML) for
+    # fewer type-based heuristics on which branch should win — the
+    # earlier "is this Article's +description+ a teaser or the real
+    # body?" carve-out is no longer needed because both end up in
+    # the output regardless.
+    module HTML
+      # @return [Array<String>] content-types this extractor claims.
+      CONTENT_TYPES = %w[text/html application/xhtml+xml].freeze
+
+      # @return [Array<String>] schema.org +@type+ values that we treat
+      #   as "the primary entity of this page" when picking a JSON-LD
+      #   node to render. Order does not matter — the first matching
+      #   node wins. Skips noise nodes (Organization, BreadcrumbList,
+      #   WebSite, ...) that ship on most pages but carry no page
+      #   content.
+      INTERESTING_TYPES = %w[
+        Product Article NewsArticle BlogPosting Recipe Event Book Movie
+      ].freeze
+
+      # @return [Array<String>] HTML tags preserved by the readability
+      #   pass. Anything outside this list is stripped before Markdown
+      #   conversion.
+      READABILITY_TAGS = %w[
+        h1 h2 h3 h4 h5 h6 p div span ul ol li blockquote pre code a img
+        strong em b i br hr table thead tbody tr td th
+      ].freeze
+
+      # @return [Array<String>] HTML attributes preserved by the
+      #   readability pass; everything else (class, id, style, data-*)
+      #   is dropped before Markdown conversion
+      READABILITY_ATTRS = %w[href src alt title].freeze
+
+      # @return [Float] minimum +<main>+/+<article>+ to Readability
+      #   text-length ratio that triggers the semantic-container
+      #   fallback in {.readability_to_markdown}. Picked low enough to
+      #   catch the failure mode (Readability collapsing a page that
+      #   uses divs/lists instead of +<p>+ — e.g. +vaadin.com/company+,
+      #   ~5x) but high enough that pages where both produce
+      #   comparable output keep Readability's noise filtering.
+      MAIN_FALLBACK_RATIO = 2.0
+
+      # @return [Integer] minimum text length the
+      #   +<main>+/+<article>+ container must hold before the fallback
+      #   in {.readability_to_markdown} can fire. Below this, the
+      #   ratio comparison is dominated by noise and we'd swap on
+      #   tiny pages where Readability is doing the right thing.
+      MAIN_FALLBACK_MIN_CHARS = 500
+
+      # @return [Symbol] {Page#kind} tag.
+      def self.kind
+        :html
+      end
+
+      # @param sample [String] leading bytes of the content (unused —
+      #   see the no-sniff rationale in the module doc).
+      # @param content_type [String, nil] normalized content-type.
+      # @return [Boolean]
+      def self.matches?(sample:, content_type:)
+        CONTENT_TYPES.include?(content_type)
+      end
+
+      # Render the HTML document behind +io+ as Markdown by emitting
+      # both the JSON-LD section (when an interesting node is present)
+      # and the readability / +<main>+ section, joined by a horizontal
+      # rule. Either section may be missing — pages with no JSON-LD
+      # return only the readability output, and a malformed page with
+      # no extractable body returns only the JSON-LD render.
+      #
+      # @param io [IO, StringIO] IO over the HTML document.
+      # @return [String] Markdown representation
+      def self.extract(io)
+        html = io.read
+        sections = [jsonld_section(html), readability_to_markdown(html)]
+        sections.reject! { |s| s.nil? || s.strip.empty? }
+        sections.join("\n\n---\n\n")
+      end
+
+      # Pick the first JSON-LD node whose +@type+ matches one of
+      # {INTERESTING_TYPES} and render it as Markdown. Returns +nil+
+      # when no such node exists, in which case {.extract} emits only
+      # the readability section.
+      #
+      # No content-field gating: a node carrying just +name+/+author+/
+      # +datePublished+ still renders (as a metadata-only header),
+      # because the readability pass independently produces the page
+      # body. That is the trade-off that lets us drop the type-based
+      # "is this teaser or article copy?" heuristics — duplication is
+      # acceptable when both views are available, and the LLM can
+      # pick whichever it needs.
+      #
+      # @param html [String] HTML document body
+      # @return [String, nil] Markdown render of the picked JSON-LD
+      #   node, or +nil+ when nothing matched
+      def self.jsonld_section(html)
+        node = parse_jsonld(html).find do |n|
+          Array(n['@type']).any? { |t| INTERESTING_TYPES.include?(t) }
+        end
+        node ? jsonld_to_markdown(node) : nil
+      end
+
+      # Collect every JSON-LD payload embedded in +html+, flattening
+      # +@graph+ wrappers so callers see one flat array of schema.org
+      # nodes. Malformed JSON blocks are silently skipped — sites
+      # frequently ship broken JSON-LD and we only need at least one
+      # parseable block.
+      #
+      # @param html [String] HTML document body
+      # @return [Array<Hash>] parsed JSON-LD nodes; possibly empty
+      def self.parse_jsonld(html)
+        doc = Nokogiri::HTML(html)
+        blobs = doc.css('script[type="application/ld+json"]').map(&:text)
+
+        blobs.flat_map do |raw|
+          parsed = begin
+            JSON.parse(raw)
+          rescue JSON::ParserError
+            nil
+          end
+          next [] unless parsed
+
+          nodes = parsed.is_a?(Array) ? parsed : [parsed]
+          nodes.flat_map { |n| n['@graph'].is_a?(Array) ? n['@graph'] : [n] }
+        end
+      end
+
+      # Render a single JSON-LD +node+ as Markdown: a top-level title
+      # from +name+/+headline+, a bullet list of common useful fields
+      # (brand, SKU, price, rating, author, published date, ...), the
+      # body copy, and the lead image.
+      #
+      # When the node carries +articleBody+ (the full publisher-supplied
+      # article text), that wins over +description+ — the description
+      # is typically a lede teaser and would just repeat the article's
+      # opening lines.
+      #
+      # @param node [Hash] JSON-LD node, typically picked by
+      #   {.jsonld_section}
+      # @return [String] Markdown representation
+      def self.jsonld_to_markdown(node)
+        out = +''
+        name = node['name'] || node['headline']
+        out << "# #{name}\n\n" if name
+
+        offer  = first_obj(node['offers'])
+        rating = first_obj(node['aggregateRating'])
+        brand  = first_obj_or_string(node['brand'])
+        author = first_obj_or_string(node['author'])
+
+        brand_name  = brand.is_a?(Hash)  ? brand['name']  : brand
+        author_name = author.is_a?(Hash) ? author['name'] : author
+
+        fields = {
+          'Brand'        => brand_name,
+          'SKU'          => node['sku'],
+          'GTIN'         => node['gtin13'] || node['gtin'],
+          'Price'        => [offer['price'], offer['priceCurrency']].compact.join(' '),
+          'Availability' => offer['availability'],
+          'Rating'       => rating['ratingValue'],
+          'Reviews'      => rating['reviewCount'],
+          'Author'       => author_name,
+          'Published'    => node['datePublished']
+        }.reject { |_, v| v.nil? || v.to_s.strip.empty? }
+
+        unless fields.empty?
+          fields.each { |k, v| out << "- **#{k}:** #{v}\n" }
+          out << "\n"
+        end
+
+        if (body = node['articleBody'] || node['description'])
+          out << "#{body}\n\n"
+        end
+
+        if (img = node['image'])
+          img = img.first if img.is_a?(Array)
+          img = img['url'] if img.is_a?(Hash)
+          out << "![image](#{img})\n\n" if img
+        end
+
+        out
+      end
+
+      # Run +Readability+ over +html+ to isolate the main content node,
+      # then convert that to Markdown via +reverse_markdown+. The page
+      # +<title>+ is rendered as a top-level heading.
+      #
+      # When the page uses semantic HTML5 (+<main>+ or +<article>+) but
+      # leaves most of its content outside +<p>+ tags — divs, lists,
+      # spans — Readability's paragraph-density scoring collapses the
+      # extraction to a sliver of the page. In that case we render the
+      # +<main>+/+<article>+ container directly. The fallback only
+      # fires when the container holds substantially more text than
+      # Readability picked up (see {MAIN_FALLBACK_RATIO} /
+      # {MAIN_FALLBACK_MIN_CHARS}); on pages where both agree we keep
+      # Readability so its noise filtering still strips nav/ads/etc.
+      #
+      # @param html [String] HTML document body
+      # @return [String] Markdown representation
+      def self.readability_to_markdown(html)
+        rdoc = Readability::Document.new(
+          html,
+          tags: READABILITY_TAGS,
+          attributes: READABILITY_ATTRS,
+          remove_empty_nodes: true
+        )
+        readability_html = rdoc.content
+        title = rdoc.title
+
+        body_html = main_fallback_html(html, readability_html) || readability_html
+        body = ReverseMarkdown.convert(body_html, unknown_tags: :bypass, github_flavored: true)
+
+        out = +''
+        out << "# #{title.strip}\n\n" if title && !title.strip.empty?
+        out << body
+        out
+      end
+
+      # If +html+ has a +<main>+ or +<article>+ element holding
+      # substantially more text than Readability extracted, return that
+      # container's HTML so the caller can render it instead. Returns
+      # +nil+ when the fallback should not fire — when there is no
+      # semantic container, when it's too small to be meaningful, or
+      # when Readability's output is already comparable.
+      #
+      # @param html [String] full HTML document body, used to locate
+      #   the +<main>+/+<article>+ container
+      # @param readability_html [String] HTML produced by
+      #   +Readability::Document#content+, used as the comparison
+      #   baseline
+      # @return [String, nil] container HTML when the fallback should
+      #   fire, +nil+ otherwise
+      def self.main_fallback_html(html, readability_html)
+        doc = Nokogiri::HTML(html)
+        container = doc.at_css('main') || doc.at_css('article')
+        return nil unless container
+
+        container_text_len = container.text.gsub(/\s+/, ' ').strip.length
+        return nil if container_text_len < MAIN_FALLBACK_MIN_CHARS
+
+        readability_text_len = Nokogiri::HTML(readability_html).text.gsub(/\s+/, ' ').strip.length
+        return nil if container_text_len < MAIN_FALLBACK_RATIO * readability_text_len
+
+        container.to_html
+      end
+      private_class_method :main_fallback_html
+
+      # JSON-LD fields can be a string, hash, or array of either.
+      # Normalize to a single hash (the first one if it's a list) so
+      # callers can +.dig+ safely.
+      #
+      # @param value [Object] raw JSON-LD field value
+      # @return [Hash] empty hash when +value+ does not contain a hash
+      def self.first_obj(value)
+        value = value.first if value.is_a?(Array)
+        value.is_a?(Hash) ? value : {}
+      end
+      private_class_method :first_obj
+
+      # Same idea as {.first_obj} but preserves a bare string (e.g.
+      # +brand: "Apple"+) instead of replacing it with +{}+.
+      #
+      # @param value [Object] raw JSON-LD field value
+      # @return [String, Hash, nil]
+      def self.first_obj_or_string(value)
+        value = value.first if value.is_a?(Array)
+        value
+      end
+      private_class_method :first_obj_or_string
+    end
+  end
+end

data/lib/pikuri/extractor/passthrough.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Pikuri
+  module Extractor
+    # The terminal plain-text arm of the registry: content that *is*
+    # already text needs no extraction, so it passes through verbatim
+    # (forced to UTF-8 — invalid bytes are left in for downstream to
+    # deal with, matching what +File.read+ with a UTF-8 encoding does).
+    # Markdown, source files, JSON, robots.txt all land here.
+    #
+    # Matching is split by whether the transport supplied a
+    # content-type:
+    #
+    # * With a content-type (the web path): claim +text/*+ only.
+    #   A non-text type that no earlier extractor claimed is *not*
+    #   second-guessed by sniffing — a server declaring
+    #   +application/octet-stream+ gets the {Unsupported} refusal the
+    #   LLM can react to, same as before this registry existed.
+    # * Without one (the local-file path, where {FileType.detect_mime}
+    #   returned +nil+ for "unrecognised"): claim anything that passes
+    #   the {FileType.binary?} heuristic on the sample. Opaque
+    #   binaries stay unclaimed and surface as {Unsupported}.
+    module Passthrough
+      # @return [Symbol] {Page#kind} tag.
+      def self.kind
+        :text
+      end
+
+      # @param sample [String] leading bytes of the content.
+      # @param content_type [String, nil] normalized content-type,
+      #   +nil+ when the transport has none.
+      # @return [Boolean]
+      def self.matches?(sample:, content_type:)
+        return content_type.start_with?('text/') unless content_type.nil?
+
+        !FileType.binary?(sample)
+      end
+
+      # @param io [IO, StringIO] IO over the text content.
+      # @return [String] the content, tagged UTF-8. Deliberately NOT
+      #   derived from {.extract_lines} — a passthrough must stay
+      #   verbatim (trailing newline, CRLF line endings), which a
+      #   join of chomped lines would silently normalize away.
+      def self.extract(io)
+        io.read.force_encoding(Encoding::UTF_8)
+      end
+
+      # The lazy line stream for {Extractor.extract_paged}: the IO is
+      # read line-by-line, so a window over the head of a gigabyte
+      # log never loads the rest. Consuming the whole stream is a
+      # cheap sequential read — which is why the paging window counts
+      # this stream's tail for an exact +total_lines+ (see
+      # {Extractor.extract_paged}).
+      #
+      # @param io [IO, StringIO] IO over the text content; must
+      #   remain open while the enumerator is consumed.
+      # @return [Enumerator::Lazy<String>] chomped lines, tagged
+      #   UTF-8.
+      def self.extract_lines(io)
+        io.each_line.lazy.map { |raw| raw.chomp.force_encoding(Encoding::UTF_8) }
+      end
+    end
+  end
+end