pikuri-core 0.0.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +17 -30
  3. data/lib/pikuri/agent/chat_transport.rb +168 -107
  4. data/lib/pikuri/agent/configurator.rb +194 -173
  5. data/lib/pikuri/agent/control/cancellable.rb +100 -70
  6. data/lib/pikuri/agent/control/handoff.rb +75 -0
  7. data/lib/pikuri/agent/control/interloper.rb +107 -118
  8. data/lib/pikuri/agent/control.rb +13 -35
  9. data/lib/pikuri/agent/event.rb +144 -197
  10. data/lib/pikuri/agent/extension.rb +83 -95
  11. data/lib/pikuri/agent/history.rb +247 -0
  12. data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
  13. data/lib/pikuri/agent/listener/terminal.rb +169 -136
  14. data/lib/pikuri/agent/listener/token_log.rb +84 -157
  15. data/lib/pikuri/agent/listener.rb +24 -44
  16. data/lib/pikuri/agent/listener_list.rb +32 -47
  17. data/lib/pikuri/agent/request_instrumenter.rb +102 -0
  18. data/lib/pikuri/agent.rb +1095 -577
  19. data/lib/pikuri/attachment.rb +65 -0
  20. data/lib/pikuri/bundler_env.rb +68 -0
  21. data/lib/pikuri/extractor/passthrough.rb +24 -31
  22. data/lib/pikuri/extractor.rb +125 -155
  23. data/lib/pikuri/file_type.rb +64 -136
  24. data/lib/pikuri/finalizers.rb +32 -47
  25. data/lib/pikuri/paths.rb +104 -13
  26. data/lib/pikuri/sanitizer.rb +46 -67
  27. data/lib/pikuri/subprocess.rb +86 -118
  28. data/lib/pikuri/testing.rb +369 -0
  29. data/lib/pikuri/tool/calculator.rb +168 -74
  30. data/lib/pikuri/tool/execute_context.rb +42 -0
  31. data/lib/pikuri/tool/parameters.rb +31 -29
  32. data/lib/pikuri/tool/result.rb +164 -0
  33. data/lib/pikuri/tool/trifecta_legs.rb +248 -0
  34. data/lib/pikuri/tool.rb +416 -67
  35. data/lib/pikuri/trifecta/contribution.rb +43 -0
  36. data/lib/pikuri/trifecta/node.rb +47 -0
  37. data/lib/pikuri/trifecta/report.rb +228 -0
  38. data/lib/pikuri/trifecta.rb +126 -0
  39. data/lib/pikuri/version.rb +1 -1
  40. data/lib/pikuri-core.rb +70 -92
  41. data/prompts/agent-loop.txt +5 -0
  42. data/prompts/pikuri-chat.txt +3 -12
  43. metadata +26 -72
  44. data/lib/pikuri/agent/context_window_detector.rb +0 -222
  45. data/lib/pikuri/agent/control/step_limit.rb +0 -111
  46. data/lib/pikuri/agent/extension_context.rb +0 -147
  47. data/lib/pikuri/agent/listener/rate_limited.rb +0 -172
  48. data/lib/pikuri/agent/synthesizer.rb +0 -171
  49. data/lib/pikuri/extractor/html.rb +0 -303
  50. data/lib/pikuri/tool/fetch.rb +0 -172
  51. data/lib/pikuri/tool/scraper.rb +0 -186
  52. data/lib/pikuri/tool/search/brave.rb +0 -198
  53. data/lib/pikuri/tool/search/duckduckgo.rb +0 -207
  54. data/lib/pikuri/tool/search/engines.rb +0 -186
  55. data/lib/pikuri/tool/search/exa.rb +0 -229
  56. data/lib/pikuri/tool/search/rate_limiter.rb +0 -92
  57. data/lib/pikuri/tool/search/result.rb +0 -29
  58. data/lib/pikuri/tool/web_scrape.rb +0 -121
  59. data/lib/pikuri/tool/web_search.rb +0 -57
  60. data/lib/pikuri/url_cache.rb +0 -112
@@ -1,172 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Pikuri
4
- class Agent
5
- module Listener
6
- # Decorator that coalesces high-frequency streaming deltas
7
- # ({Event::AssistantDelta}, {Event::ThinkingDelta}) into at most
8
- # +fps+ events per second before handing them off to an inner
9
- # listener. Non-delta events flow through unmodified, with any
10
- # pending coalesced content either emitted ahead of them
11
- # (default) or dropped silently — see
12
- # +flush_pending_on_non_delta:+.
13
- #
14
- # Useful for renderers whose per-delta repaint cost is higher
15
- # than the wire-arrival rate of provider chunks: a fast provider
16
- # can emit dozens of deltas per second, and a Markdown-rendering
17
- # UI doesn't need every one of them on-screen. +fps: 15+ is a
18
- # reasonable starting point for a TUI.
19
- #
20
- # == Tick alignment
21
- #
22
- # The first flush is anchored to a monotonic-clock reference
23
- # taken at construction, not to the arrival of the first delta.
24
- # This keeps stream-startup latency from leaking into the
25
- # cadence: with +fps: 1+, if the first delta arrives 0.9s after
26
- # construction, it flushes immediately (the construction-time
27
- # tick has already passed) and the next tick fires 0.1s later
28
- # — preserving the configured rate. After a flush, the schedule
29
- # advances by exactly +1/fps+ per missed tick; a long-idle
30
- # stream doesn't backlog ticks against itself.
31
- #
32
- # == Per-stream buffering
33
- #
34
- # Assistant and thinking deltas are buffered independently. A
35
- # tick flushes whichever buffers are non-empty as separate
36
- # coalesced delta events; providers practically don't interleave
37
- # the two modalities within a single tick, but the
38
- # implementation doesn't rely on that.
39
- #
40
- # == Non-delta handling
41
- #
42
- # +flush_pending_on_non_delta: true+ (default) emits any pending
43
- # coalesced content as a delta event ahead of the non-delta
44
- # event so the inner listener sees the complete stream — the
45
- # lossless choice. +false+ drops the pending buffers silently:
46
- # appropriate for inner listeners that re-render the
47
- # authoritative final content on a bookend ({Event::Assistant} /
48
- # {Event::Thinking}), where rendering the trailing 0–66 ms of
49
- # deltas would be wasted CPU before the redraw.
50
- #
51
- # == Threading
52
- #
53
- # No threads, no timers. The decorator advances its tick state
54
- # only when an event arrives, so it's safe to install on any
55
- # listener regardless of which thread the agent emits on, as
56
- # long as delivery is sequential (which the {ListenerList}
57
- # contract guarantees).
58
- class RateLimited < Base
59
- # @param inner [Listener::Base] the listener to forward
60
- # (possibly coalesced) events to.
61
- # @param fps [Integer, Float] frames-per-second cap on the
62
- # coalesced delta stream. Must be positive.
63
- # @param flush_pending_on_non_delta [Boolean] +true+
64
- # (default) emits any pending coalesced content as a delta
65
- # before forwarding the non-delta event; +false+ drops the
66
- # pending buffers silently.
67
- # @param clock [Proc] zero-arg returning monotonic
68
- # seconds-since-some-epoch as a +Float+. Injection seam for
69
- # deterministic specs; production uses
70
- # +Process.clock_gettime(Process::CLOCK_MONOTONIC)+.
71
- # @raise [ArgumentError] if +fps+ is not positive.
72
- def initialize(inner, fps:, flush_pending_on_non_delta: true,
73
- clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
74
- super()
75
- raise ArgumentError, "fps must be positive, got #{fps.inspect}" unless fps.positive?
76
-
77
- @inner = inner
78
- @fps = fps
79
- @period = 1.0 / fps
80
- @flush_on_non_delta = flush_pending_on_non_delta
81
- @clock = clock
82
- @next_tick_at = clock.call
83
- @assistant_buffer = String.new
84
- @thinking_buffer = String.new
85
- end
86
-
87
- # @param event [Agent::Event] one of the {Agent::Event}
88
- # variants; see {Agent::Event} for the full list
89
- # @return [void]
90
- def on_event(event)
91
- case event
92
- when Event::AssistantDelta
93
- @assistant_buffer << event.content
94
- tick
95
- when Event::ThinkingDelta
96
- @thinking_buffer << event.content
97
- tick
98
- else
99
- if @flush_on_non_delta
100
- flush_thinking
101
- flush_assistant
102
- else
103
- @assistant_buffer.clear
104
- @thinking_buffer.clear
105
- end
106
- @inner.on_event(event)
107
- end
108
- end
109
-
110
- # Sub-agent variant: wraps the inner listener's sub-agent
111
- # variant (or the inner listener itself when it doesn't define
112
- # +for_sub_agent+) in a fresh +RateLimited+ with the same
113
- # knobs. Returns +nil+ when the inner's +for_sub_agent+
114
- # returned +nil+ — there's nothing to wrap.
115
- #
116
- # @param params [Hash{Symbol => Object}] forwarded to the
117
- # inner listener's +for_sub_agent+, same protocol as
118
- # {ListenerList#for_sub_agent}
119
- # @return [RateLimited, nil]
120
- def for_sub_agent(**params)
121
- inner_sub = @inner.respond_to?(:for_sub_agent) ? @inner.for_sub_agent(**params) : @inner
122
- return nil if inner_sub.nil?
123
-
124
- self.class.new(inner_sub, fps: @fps,
125
- flush_pending_on_non_delta: @flush_on_non_delta,
126
- clock: @clock)
127
- end
128
-
129
- # @return [String] short label for {Agent#to_s}; embeds the
130
- # inner listener's label so the chain reads end-to-end in
131
- # {ListenerList#to_s}
132
- def to_s
133
- policy = @flush_on_non_delta ? 'flush' : 'drop'
134
- "RateLimited(#{@fps}fps, #{policy}, #{@inner})"
135
- end
136
-
137
- private
138
-
139
- # Advance to the next tick strictly after +now+, in one shot.
140
- # Handles the case where deltas paused for several periods —
141
- # a long-idle stream doesn't backlog ticks against the next
142
- # stream's first delta.
143
- # @return [void]
144
- def tick
145
- now = @clock.call
146
- return if now < @next_tick_at
147
-
148
- flush_thinking
149
- flush_assistant
150
- missed = ((now - @next_tick_at) / @period).floor + 1
151
- @next_tick_at += missed * @period
152
- end
153
-
154
- # @return [void]
155
- def flush_assistant
156
- return if @assistant_buffer.empty?
157
-
158
- @inner.on_event(Event::AssistantDelta.new(content: @assistant_buffer.dup))
159
- @assistant_buffer.clear
160
- end
161
-
162
- # @return [void]
163
- def flush_thinking
164
- return if @thinking_buffer.empty?
165
-
166
- @inner.on_event(Event::ThinkingDelta.new(content: @thinking_buffer.dup))
167
- @thinking_buffer.clear
168
- end
169
- end
170
- end
171
- end
172
- end
@@ -1,171 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Pikuri
4
- class Agent
5
- # Prompt builder for the step-exhaustion rescue. When an
6
- # +Agent+'s {Control::StepLimit} trips with the +:synthesize+
7
- # policy, +Agent#run_loop+ runs this module's prompt on a
8
- # nested tools-free agent so the run still produces something
9
- # useful — an assistant turn that answers the user's question
10
- # from whatever evidence the failed agent collected before
11
- # running out of budget.
12
- #
13
- # == Why this exists
14
- #
15
- # Without a rescue, a step-exhausted run just raises a stack
16
- # trace past +bin/pikuri-chat+ and the user gets nothing
17
- # despite the agent having gathered useful information in the
18
- # first N-1 steps. The observed failure mode is the "wait,
19
- # but what about X?" death-loop: the agent collects sound
20
- # evidence in the first few rounds, then spends the rest of
21
- # the budget second-guessing. By the time the cap trips, the
22
- # answer is largely in the messages — it just needs a
23
- # tools-free pass to synthesize.
24
- #
25
- # Salvage is the wrong move for some agents, which is why the
26
- # policy lives on {Control::StepLimit} and defaults to
27
- # +:raise+ — a coding agent's half-finished work can't be
28
- # completed by a tools-free pass, only described. See
29
- # {Control::StepLimit}'s class header.
30
- #
31
- # == Seam discipline
32
- #
33
- # This module is pure prompt construction — no chat handling,
34
- # no +RubyLLM.chat+ call, no event wiring. The execution side
35
- # (constructing the nested agent, sharing the parent's
36
- # listener stream and cancellable, capturing the answer) is
37
- # +Agent#run_synthesizer+'s job: the synth is a regular
38
- # tools-free +Agent+, the same construction shape the +agent+
39
- # tool from +pikuri-subagents+ uses for sub-agents. The only
40
- # +RubyLLM::*+ surface read here is the value-type
41
- # +RubyLLM::Message+ / +ToolCall+ passthrough (per the
42
- # value-type rule in CLAUDE.md).
43
- module Synthesizer
44
- # The synthesizer's system prompt. Strict and short: use
45
- # the evidence, don't apologize, admit gaps when present.
46
- SYSTEM_PROMPT = <<~PROMPT
47
- You are given evidence another agent collected before running out of steps. Answer the user's question using only this evidence. You have no tools. If the evidence is insufficient, state plainly what's missing and what partial answer you can give. Do not apologize or comment on the previous agent.
48
- PROMPT
49
-
50
- # Render the user's question plus an "Evidence gathered"
51
- # section built from +parent_messages+ as a single prompt
52
- # string. Pure function — no I/O, safe to test directly
53
- # with fixture messages.
54
- #
55
- # @param parent_messages [Array<RubyLLM::Message>]
56
- # @param user_message [String]
57
- # @return [String]
58
- def self.build_prompt(parent_messages:, user_message:)
59
- transcript = format_evidence(parent_messages)
60
- "Question: #{user_message}\n\nEvidence gathered:\n#{transcript}"
61
- end
62
-
63
- # Walk the parent's message history and produce a paired
64
- # "Tool call:" / "Tool result:" log, preserving order. Tool
65
- # calls that have no matching +:tool+ message are dropped —
66
- # the call that tripped the step limit never executed, so
67
- # including it would mislead the synth into citing
68
- # nonexistent results. Non-empty assistant text content is
69
- # preserved as a "Note:" line, since the parent may have
70
- # summarized progress between tool calls.
71
- #
72
- # @param messages [Array<RubyLLM::Message>]
73
- # @return [String]
74
- def self.format_evidence(messages)
75
- results_by_id = messages
76
- .select { |m| m.role == :tool }
77
- .to_h { |m| [m.tool_call_id, m.content] }
78
-
79
- lines = []
80
- messages.each do |msg|
81
- next unless msg.role == :assistant
82
-
83
- text = msg.content
84
- lines << "Note: #{text}" if text.is_a?(String) && !text.empty?
85
-
86
- msg.tool_calls&.each_value do |tc|
87
- result = results_by_id[tc.id]
88
- next unless result
89
-
90
- args = tc.arguments.map { |k, v| "#{k}=#{v.inspect}" }.join(', ')
91
- lines << "Tool call: #{tc.name}(#{args})"
92
- lines << "Tool result: #{result}"
93
- lines << ''
94
- end
95
- end
96
- lines.join("\n").rstrip
97
- end
98
- private_class_method :format_evidence
99
-
100
- # The +:synthesize+ arm of the step-exhaustion policy (see the
101
- # class header). Runs the {Synthesizer} prompt over the
102
- # exhausted chat's history on a nested tools-free +Agent+ —
103
- # the same construction shape the +agent+ tool from
104
- # +pikuri-subagents+ uses for sub-agents, so the synth gets
105
- # listener propagation, transport / context-window-cap /
106
- # streaming inheritance, and teardown via +close+ for free.
107
- # The synth's answer is returned.
108
- #
109
- # @param ctx [ExtensionContext]
110
- # @param chat_messages [Array<RubyLLM::Message>] the
111
- # exhausted chat's full message history, the evidence
112
- # {.build_prompt} renders
113
- # @param user_message [String] the user's original question
114
- # from the turn that exhausted
115
- # @raise [Control::Cancellable::Cancelled] when a cancel
116
- # landed between the budget tripping and this rescue —
117
- # cancellation wins over salvage
118
- # @return [String] the synth answer
119
- def self.run_synthesizer(ctx, chat_messages, user_message)
120
- # Check the cancel flag *before* constructing the synth: the
121
- # nested run_loop resets the shared cancellable at its turn
122
- # boundary, which would erase a cancel requested in this
123
- # window. The raise propagates without a parent-side
124
- # {Event::Cancelled} — a cancel *during* synthesis emits it
125
- # from the synth's own rescue (on the derived listener list)
126
- # instead, so either way the stream sees at most one.
127
- ctx.agent.cancellable&.check!
128
-
129
- ctx.emit_event(Event::FallbackNotice.new(
130
- reason: "agent exhausted #{ctx.agent.step_limit.max} steps; " \
131
- 'synthesizing answer from gathered evidence'
132
- ))
133
-
134
- # Synth runs under this agent's identity but with a
135
- # different system prompt, so it gets a distinct
136
- # +_synthesizer+ suffix on the id — same +_+ separator the
137
- # sub-agent generator uses, so main becomes +"synthesizer"+
138
- # and a sub-agent +"researcher 0"+ becomes
139
- # +"researcher 0_synthesizer"+. Any +TokenLog+ in the list
140
- # tags the synth's prompt under that bracket so it's
141
- # obvious from the log which turns were the rescue rather
142
- # than the original loop.
143
- synth_id = ctx.agent.id.empty? ? 'synthesizer' : "#{ctx.agent.id}_synthesizer"
144
- synth = Agent.new(
145
- # Carry the parent's resolved cap on the transport so the synth
146
- # reuses it without a re-probe — the cap rides {ChatTransport}
147
- # now, not an +Agent.new(context_window:)+ kwarg.
148
- transport: ctx.agent.transport.with(context_window: ctx.agent.context_window_cap),
149
- system_prompt: Synthesizer::SYSTEM_PROMPT,
150
- # Defensive budget with the default :raise policy: the
151
- # synth has no tools so it should never tick, but a buggy
152
- # provider that somehow returns a tool call must not loop
153
- # forever — and a synth that needs its own synth is a bug,
154
- # not a rescue.
155
- step_limit: Control::StepLimit.new(max: 1),
156
- cancellable: ctx.agent.cancellable,
157
- id: synth_id,
158
- streaming: ctx.agent.streaming
159
- ) { |c| c.add_listeners(ctx.sub_agent_listeners(id: synth_id)) }
160
- begin
161
- synth.run_loop(user_message: Synthesizer.build_prompt(
162
- parent_messages: chat_messages, user_message: user_message
163
- ))
164
- synth.last_assistant_content
165
- ensure
166
- synth.close
167
- end
168
- end
169
- end
170
- end
171
- end
@@ -1,303 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'json'
4
- require 'nokogiri'
5
- require 'readability'
6
- require 'reverse_markdown'
7
-
8
- module Pikuri
9
- module Extractor
10
- # HTML → Markdown extractor.
11
- #
12
- # Matched by content-type only (+text/html+ /
13
- # +application/xhtml+xml+) — deliberately no byte sniff. The web
14
- # path always has the header; for local files a sniff would route
15
- # +Workspace::Read+ of an +.html+ source file through readability
16
- # extraction, when a developer reading an HTML file wants the
17
- # source. Local HTML stays on the {Passthrough} arm until a
18
- # consumer genuinely needs otherwise.
19
- #
20
- # Always renders both views of the page when available:
21
- #
22
- # 1. JSON-LD section. Any +<script type="application/ld+json">+ node
23
- # whose +@type+ matches a substantive schema.org content type
24
- # (Product, Article, Recipe, ...) is rendered as a header — title,
25
- # metadata bullets (brand, SKU, price, rating, author, published),
26
- # and the +articleBody+/+description+ copy when present.
27
- # 2. Readability section. The page is run through +Readability+ +
28
- # +reverse_markdown+, with a +<main>+/+<article>+ fallback for
29
- # pages whose content sits mostly outside +<p>+ tags.
30
- #
31
- # Concatenated with a horizontal rule, so the LLM gets both the
32
- # structured metadata and the rendered body and can pick whichever
33
- # is more useful for the task. Trades some duplication (when a
34
- # publisher embeds the article body in JSON-LD AND in HTML) for
35
- # fewer type-based heuristics on which branch should win — the
36
- # earlier "is this Article's +description+ a teaser or the real
37
- # body?" carve-out is no longer needed because both end up in
38
- # the output regardless.
39
- module HTML
40
- # @return [Array<String>] content-types this extractor claims.
41
- CONTENT_TYPES = %w[text/html application/xhtml+xml].freeze
42
-
43
- # @return [Array<String>] schema.org +@type+ values that we treat
44
- # as "the primary entity of this page" when picking a JSON-LD
45
- # node to render. Order does not matter — the first matching
46
- # node wins. Skips noise nodes (Organization, BreadcrumbList,
47
- # WebSite, ...) that ship on most pages but carry no page
48
- # content.
49
- INTERESTING_TYPES = %w[
50
- Product Article NewsArticle BlogPosting Recipe Event Book Movie
51
- ].freeze
52
-
53
- # @return [Array<String>] HTML tags preserved by the readability
54
- # pass. Anything outside this list is stripped before Markdown
55
- # conversion.
56
- READABILITY_TAGS = %w[
57
- h1 h2 h3 h4 h5 h6 p div span ul ol li blockquote pre code a img
58
- strong em b i br hr table thead tbody tr td th
59
- ].freeze
60
-
61
- # @return [Array<String>] HTML attributes preserved by the
62
- # readability pass; everything else (class, id, style, data-*)
63
- # is dropped before Markdown conversion
64
- READABILITY_ATTRS = %w[href src alt title].freeze
65
-
66
- # @return [Float] minimum +<main>+/+<article>+ to Readability
67
- # text-length ratio that triggers the semantic-container
68
- # fallback in {.readability_to_markdown}. Picked low enough to
69
- # catch the failure mode (Readability collapsing a page that
70
- # uses divs/lists instead of +<p>+ — e.g. +vaadin.com/company+,
71
- # ~5x) but high enough that pages where both produce
72
- # comparable output keep Readability's noise filtering.
73
- MAIN_FALLBACK_RATIO = 2.0
74
-
75
- # @return [Integer] minimum text length the
76
- # +<main>+/+<article>+ container must hold before the fallback
77
- # in {.readability_to_markdown} can fire. Below this, the
78
- # ratio comparison is dominated by noise and we'd swap on
79
- # tiny pages where Readability is doing the right thing.
80
- MAIN_FALLBACK_MIN_CHARS = 500
81
-
82
- # @return [Symbol] {Page#kind} tag.
83
- def self.kind
84
- :html
85
- end
86
-
87
- # @param sample [String] leading bytes of the content (unused —
88
- # see the no-sniff rationale in the module doc).
89
- # @param content_type [String, nil] normalized content-type.
90
- # @return [Boolean]
91
- def self.matches?(sample:, content_type:)
92
- CONTENT_TYPES.include?(content_type)
93
- end
94
-
95
- # Render the HTML document behind +io+ as Markdown by emitting
96
- # both the JSON-LD section (when an interesting node is present)
97
- # and the readability / +<main>+ section, joined by a horizontal
98
- # rule. Either section may be missing — pages with no JSON-LD
99
- # return only the readability output, and a malformed page with
100
- # no extractable body returns only the JSON-LD render.
101
- #
102
- # @param io [IO, StringIO] IO over the HTML document.
103
- # @return [String] Markdown representation
104
- def self.extract(io)
105
- html = io.read
106
- sections = [jsonld_section(html), readability_to_markdown(html)]
107
- sections.reject! { |s| s.nil? || s.strip.empty? }
108
- sections.join("\n\n---\n\n")
109
- end
110
-
111
- # Pick the first JSON-LD node whose +@type+ matches one of
112
- # {INTERESTING_TYPES} and render it as Markdown. Returns +nil+
113
- # when no such node exists, in which case {.extract} emits only
114
- # the readability section.
115
- #
116
- # No content-field gating: a node carrying just +name+/+author+/
117
- # +datePublished+ still renders (as a metadata-only header),
118
- # because the readability pass independently produces the page
119
- # body. That is the trade-off that lets us drop the type-based
120
- # "is this teaser or article copy?" heuristics — duplication is
121
- # acceptable when both views are available, and the LLM can
122
- # pick whichever it needs.
123
- #
124
- # @param html [String] HTML document body
125
- # @return [String, nil] Markdown render of the picked JSON-LD
126
- # node, or +nil+ when nothing matched
127
- def self.jsonld_section(html)
128
- node = parse_jsonld(html).find do |n|
129
- Array(n['@type']).any? { |t| INTERESTING_TYPES.include?(t) }
130
- end
131
- node ? jsonld_to_markdown(node) : nil
132
- end
133
-
134
- # Collect every JSON-LD payload embedded in +html+, flattening
135
- # +@graph+ wrappers so callers see one flat array of schema.org
136
- # nodes. Malformed JSON blocks are silently skipped — sites
137
- # frequently ship broken JSON-LD and we only need at least one
138
- # parseable block.
139
- #
140
- # @param html [String] HTML document body
141
- # @return [Array<Hash>] parsed JSON-LD nodes; possibly empty
142
- def self.parse_jsonld(html)
143
- doc = Nokogiri::HTML(html)
144
- blobs = doc.css('script[type="application/ld+json"]').map(&:text)
145
-
146
- blobs.flat_map do |raw|
147
- parsed = begin
148
- JSON.parse(raw)
149
- rescue JSON::ParserError
150
- nil
151
- end
152
- next [] unless parsed
153
-
154
- nodes = parsed.is_a?(Array) ? parsed : [parsed]
155
- nodes.flat_map { |n| n['@graph'].is_a?(Array) ? n['@graph'] : [n] }
156
- end
157
- end
158
-
159
- # Render a single JSON-LD +node+ as Markdown: a top-level title
160
- # from +name+/+headline+, a bullet list of common useful fields
161
- # (brand, SKU, price, rating, author, published date, ...), the
162
- # body copy, and the lead image.
163
- #
164
- # When the node carries +articleBody+ (the full publisher-supplied
165
- # article text), that wins over +description+ — the description
166
- # is typically a lede teaser and would just repeat the article's
167
- # opening lines.
168
- #
169
- # @param node [Hash] JSON-LD node, typically picked by
170
- # {.jsonld_section}
171
- # @return [String] Markdown representation
172
- def self.jsonld_to_markdown(node)
173
- out = +''
174
- name = node['name'] || node['headline']
175
- out << "# #{name}\n\n" if name
176
-
177
- offer = first_obj(node['offers'])
178
- rating = first_obj(node['aggregateRating'])
179
- brand = first_obj_or_string(node['brand'])
180
- author = first_obj_or_string(node['author'])
181
-
182
- brand_name = brand.is_a?(Hash) ? brand['name'] : brand
183
- author_name = author.is_a?(Hash) ? author['name'] : author
184
-
185
- fields = {
186
- 'Brand' => brand_name,
187
- 'SKU' => node['sku'],
188
- 'GTIN' => node['gtin13'] || node['gtin'],
189
- 'Price' => [offer['price'], offer['priceCurrency']].compact.join(' '),
190
- 'Availability' => offer['availability'],
191
- 'Rating' => rating['ratingValue'],
192
- 'Reviews' => rating['reviewCount'],
193
- 'Author' => author_name,
194
- 'Published' => node['datePublished']
195
- }.reject { |_, v| v.nil? || v.to_s.strip.empty? }
196
-
197
- unless fields.empty?
198
- fields.each { |k, v| out << "- **#{k}:** #{v}\n" }
199
- out << "\n"
200
- end
201
-
202
- if (body = node['articleBody'] || node['description'])
203
- out << "#{body}\n\n"
204
- end
205
-
206
- if (img = node['image'])
207
- img = img.first if img.is_a?(Array)
208
- img = img['url'] if img.is_a?(Hash)
209
- out << "![image](#{img})\n\n" if img
210
- end
211
-
212
- out
213
- end
214
-
215
- # Run +Readability+ over +html+ to isolate the main content node,
216
- # then convert that to Markdown via +reverse_markdown+. The page
217
- # +<title>+ is rendered as a top-level heading.
218
- #
219
- # When the page uses semantic HTML5 (+<main>+ or +<article>+) but
220
- # leaves most of its content outside +<p>+ tags — divs, lists,
221
- # spans — Readability's paragraph-density scoring collapses the
222
- # extraction to a sliver of the page. In that case we render the
223
- # +<main>+/+<article>+ container directly. The fallback only
224
- # fires when the container holds substantially more text than
225
- # Readability picked up (see {MAIN_FALLBACK_RATIO} /
226
- # {MAIN_FALLBACK_MIN_CHARS}); on pages where both agree we keep
227
- # Readability so its noise filtering still strips nav/ads/etc.
228
- #
229
- # @param html [String] HTML document body
230
- # @return [String] Markdown representation
231
- def self.readability_to_markdown(html)
232
- rdoc = Readability::Document.new(
233
- html,
234
- tags: READABILITY_TAGS,
235
- attributes: READABILITY_ATTRS,
236
- remove_empty_nodes: true
237
- )
238
- readability_html = rdoc.content
239
- title = rdoc.title
240
-
241
- body_html = main_fallback_html(html, readability_html) || readability_html
242
- body = ReverseMarkdown.convert(body_html, unknown_tags: :bypass, github_flavored: true)
243
-
244
- out = +''
245
- out << "# #{title.strip}\n\n" if title && !title.strip.empty?
246
- out << body
247
- out
248
- end
249
-
250
- # If +html+ has a +<main>+ or +<article>+ element holding
251
- # substantially more text than Readability extracted, return that
252
- # container's HTML so the caller can render it instead. Returns
253
- # +nil+ when the fallback should not fire — when there is no
254
- # semantic container, when it's too small to be meaningful, or
255
- # when Readability's output is already comparable.
256
- #
257
- # @param html [String] full HTML document body, used to locate
258
- # the +<main>+/+<article>+ container
259
- # @param readability_html [String] HTML produced by
260
- # +Readability::Document#content+, used as the comparison
261
- # baseline
262
- # @return [String, nil] container HTML when the fallback should
263
- # fire, +nil+ otherwise
264
- def self.main_fallback_html(html, readability_html)
265
- doc = Nokogiri::HTML(html)
266
- container = doc.at_css('main') || doc.at_css('article')
267
- return nil unless container
268
-
269
- container_text_len = container.text.gsub(/\s+/, ' ').strip.length
270
- return nil if container_text_len < MAIN_FALLBACK_MIN_CHARS
271
-
272
- readability_text_len = Nokogiri::HTML(readability_html).text.gsub(/\s+/, ' ').strip.length
273
- return nil if container_text_len < MAIN_FALLBACK_RATIO * readability_text_len
274
-
275
- container.to_html
276
- end
277
- private_class_method :main_fallback_html
278
-
279
- # JSON-LD fields can be a string, hash, or array of either.
280
- # Normalize to a single hash (the first one if it's a list) so
281
- # callers can +.dig+ safely.
282
- #
283
- # @param value [Object] raw JSON-LD field value
284
- # @return [Hash] empty hash when +value+ does not contain a hash
285
- def self.first_obj(value)
286
- value = value.first if value.is_a?(Array)
287
- value.is_a?(Hash) ? value : {}
288
- end
289
- private_class_method :first_obj
290
-
291
- # Same idea as {.first_obj} but preserves a bare string (e.g.
292
- # +brand: "Apple"+) instead of replacing it with +{}+.
293
- #
294
- # @param value [Object] raw JSON-LD field value
295
- # @return [String, Hash, nil]
296
- def self.first_obj_or_string(value)
297
- value = value.first if value.is_a?(Array)
298
- value
299
- end
300
- private_class_method :first_obj_or_string
301
- end
302
- end
303
- end