pikuri-core 0.0.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +17 -30
- data/lib/pikuri/agent/chat_transport.rb +168 -107
- data/lib/pikuri/agent/configurator.rb +194 -173
- data/lib/pikuri/agent/control/cancellable.rb +100 -70
- data/lib/pikuri/agent/control/handoff.rb +75 -0
- data/lib/pikuri/agent/control/interloper.rb +107 -118
- data/lib/pikuri/agent/control.rb +13 -35
- data/lib/pikuri/agent/event.rb +144 -197
- data/lib/pikuri/agent/extension.rb +83 -95
- data/lib/pikuri/agent/history.rb +247 -0
- data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
- data/lib/pikuri/agent/listener/terminal.rb +169 -136
- data/lib/pikuri/agent/listener/token_log.rb +84 -157
- data/lib/pikuri/agent/listener.rb +24 -44
- data/lib/pikuri/agent/listener_list.rb +32 -47
- data/lib/pikuri/agent/request_instrumenter.rb +102 -0
- data/lib/pikuri/agent.rb +1095 -577
- data/lib/pikuri/attachment.rb +65 -0
- data/lib/pikuri/bundler_env.rb +68 -0
- data/lib/pikuri/extractor/passthrough.rb +24 -31
- data/lib/pikuri/extractor.rb +125 -155
- data/lib/pikuri/file_type.rb +64 -136
- data/lib/pikuri/finalizers.rb +32 -47
- data/lib/pikuri/paths.rb +104 -13
- data/lib/pikuri/sanitizer.rb +46 -67
- data/lib/pikuri/subprocess.rb +86 -118
- data/lib/pikuri/testing.rb +369 -0
- data/lib/pikuri/tool/calculator.rb +168 -74
- data/lib/pikuri/tool/execute_context.rb +42 -0
- data/lib/pikuri/tool/parameters.rb +31 -29
- data/lib/pikuri/tool/result.rb +164 -0
- data/lib/pikuri/tool/trifecta_legs.rb +248 -0
- data/lib/pikuri/tool.rb +416 -67
- data/lib/pikuri/trifecta/contribution.rb +43 -0
- data/lib/pikuri/trifecta/node.rb +47 -0
- data/lib/pikuri/trifecta/report.rb +228 -0
- data/lib/pikuri/trifecta.rb +126 -0
- data/lib/pikuri/version.rb +1 -1
- data/lib/pikuri-core.rb +70 -92
- data/prompts/agent-loop.txt +5 -0
- data/prompts/pikuri-chat.txt +3 -12
- metadata +26 -72
- data/lib/pikuri/agent/context_window_detector.rb +0 -222
- data/lib/pikuri/agent/control/step_limit.rb +0 -111
- data/lib/pikuri/agent/extension_context.rb +0 -147
- data/lib/pikuri/agent/listener/rate_limited.rb +0 -172
- data/lib/pikuri/agent/synthesizer.rb +0 -171
- data/lib/pikuri/extractor/html.rb +0 -303
- data/lib/pikuri/tool/fetch.rb +0 -172
- data/lib/pikuri/tool/scraper.rb +0 -186
- data/lib/pikuri/tool/search/brave.rb +0 -198
- data/lib/pikuri/tool/search/duckduckgo.rb +0 -207
- data/lib/pikuri/tool/search/engines.rb +0 -186
- data/lib/pikuri/tool/search/exa.rb +0 -229
- data/lib/pikuri/tool/search/rate_limiter.rb +0 -92
- data/lib/pikuri/tool/search/result.rb +0 -29
- data/lib/pikuri/tool/web_scrape.rb +0 -121
- data/lib/pikuri/tool/web_search.rb +0 -57
- data/lib/pikuri/url_cache.rb +0 -112
|
@@ -1,172 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
class Agent
|
|
5
|
-
module Listener
|
|
6
|
-
# Decorator that coalesces high-frequency streaming deltas
|
|
7
|
-
# ({Event::AssistantDelta}, {Event::ThinkingDelta}) into at most
|
|
8
|
-
# +fps+ events per second before handing them off to an inner
|
|
9
|
-
# listener. Non-delta events flow through unmodified, with any
|
|
10
|
-
# pending coalesced content either emitted ahead of them
|
|
11
|
-
# (default) or dropped silently — see
|
|
12
|
-
# +flush_pending_on_non_delta:+.
|
|
13
|
-
#
|
|
14
|
-
# Useful for renderers whose per-delta repaint cost is higher
|
|
15
|
-
# than the wire-arrival rate of provider chunks: a fast provider
|
|
16
|
-
# can emit dozens of deltas per second, and a Markdown-rendering
|
|
17
|
-
# UI doesn't need every one of them on-screen. +fps: 15+ is a
|
|
18
|
-
# reasonable starting point for a TUI.
|
|
19
|
-
#
|
|
20
|
-
# == Tick alignment
|
|
21
|
-
#
|
|
22
|
-
# The first flush is anchored to a monotonic-clock reference
|
|
23
|
-
# taken at construction, not to the arrival of the first delta.
|
|
24
|
-
# This keeps stream-startup latency from leaking into the
|
|
25
|
-
# cadence: with +fps: 1+, if the first delta arrives 0.9s after
|
|
26
|
-
# construction, it flushes immediately (the construction-time
|
|
27
|
-
# tick has already passed) and the next tick fires 0.1s later
|
|
28
|
-
# — preserving the configured rate. After a flush, the schedule
|
|
29
|
-
# advances by exactly +1/fps+ per missed tick; a long-idle
|
|
30
|
-
# stream doesn't backlog ticks against itself.
|
|
31
|
-
#
|
|
32
|
-
# == Per-stream buffering
|
|
33
|
-
#
|
|
34
|
-
# Assistant and thinking deltas are buffered independently. A
|
|
35
|
-
# tick flushes whichever buffers are non-empty as separate
|
|
36
|
-
# coalesced delta events; providers practically don't interleave
|
|
37
|
-
# the two modalities within a single tick, but the
|
|
38
|
-
# implementation doesn't rely on that.
|
|
39
|
-
#
|
|
40
|
-
# == Non-delta handling
|
|
41
|
-
#
|
|
42
|
-
# +flush_pending_on_non_delta: true+ (default) emits any pending
|
|
43
|
-
# coalesced content as a delta event ahead of the non-delta
|
|
44
|
-
# event so the inner listener sees the complete stream — the
|
|
45
|
-
# lossless choice. +false+ drops the pending buffers silently:
|
|
46
|
-
# appropriate for inner listeners that re-render the
|
|
47
|
-
# authoritative final content on a bookend ({Event::Assistant} /
|
|
48
|
-
# {Event::Thinking}), where rendering the trailing 0–66 ms of
|
|
49
|
-
# deltas would be wasted CPU before the redraw.
|
|
50
|
-
#
|
|
51
|
-
# == Threading
|
|
52
|
-
#
|
|
53
|
-
# No threads, no timers. The decorator advances its tick state
|
|
54
|
-
# only when an event arrives, so it's safe to install on any
|
|
55
|
-
# listener regardless of which thread the agent emits on, as
|
|
56
|
-
# long as delivery is sequential (which the {ListenerList}
|
|
57
|
-
# contract guarantees).
|
|
58
|
-
class RateLimited < Base
|
|
59
|
-
# @param inner [Listener::Base] the listener to forward
|
|
60
|
-
# (possibly coalesced) events to.
|
|
61
|
-
# @param fps [Integer, Float] frames-per-second cap on the
|
|
62
|
-
# coalesced delta stream. Must be positive.
|
|
63
|
-
# @param flush_pending_on_non_delta [Boolean] +true+
|
|
64
|
-
# (default) emits any pending coalesced content as a delta
|
|
65
|
-
# before forwarding the non-delta event; +false+ drops the
|
|
66
|
-
# pending buffers silently.
|
|
67
|
-
# @param clock [Proc] zero-arg returning monotonic
|
|
68
|
-
# seconds-since-some-epoch as a +Float+. Injection seam for
|
|
69
|
-
# deterministic specs; production uses
|
|
70
|
-
# +Process.clock_gettime(Process::CLOCK_MONOTONIC)+.
|
|
71
|
-
# @raise [ArgumentError] if +fps+ is not positive.
|
|
72
|
-
def initialize(inner, fps:, flush_pending_on_non_delta: true,
|
|
73
|
-
clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
|
|
74
|
-
super()
|
|
75
|
-
raise ArgumentError, "fps must be positive, got #{fps.inspect}" unless fps.positive?
|
|
76
|
-
|
|
77
|
-
@inner = inner
|
|
78
|
-
@fps = fps
|
|
79
|
-
@period = 1.0 / fps
|
|
80
|
-
@flush_on_non_delta = flush_pending_on_non_delta
|
|
81
|
-
@clock = clock
|
|
82
|
-
@next_tick_at = clock.call
|
|
83
|
-
@assistant_buffer = String.new
|
|
84
|
-
@thinking_buffer = String.new
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
# @param event [Agent::Event] one of the {Agent::Event}
|
|
88
|
-
# variants; see {Agent::Event} for the full list
|
|
89
|
-
# @return [void]
|
|
90
|
-
def on_event(event)
|
|
91
|
-
case event
|
|
92
|
-
when Event::AssistantDelta
|
|
93
|
-
@assistant_buffer << event.content
|
|
94
|
-
tick
|
|
95
|
-
when Event::ThinkingDelta
|
|
96
|
-
@thinking_buffer << event.content
|
|
97
|
-
tick
|
|
98
|
-
else
|
|
99
|
-
if @flush_on_non_delta
|
|
100
|
-
flush_thinking
|
|
101
|
-
flush_assistant
|
|
102
|
-
else
|
|
103
|
-
@assistant_buffer.clear
|
|
104
|
-
@thinking_buffer.clear
|
|
105
|
-
end
|
|
106
|
-
@inner.on_event(event)
|
|
107
|
-
end
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
# Sub-agent variant: wraps the inner listener's sub-agent
|
|
111
|
-
# variant (or the inner listener itself when it doesn't define
|
|
112
|
-
# +for_sub_agent+) in a fresh +RateLimited+ with the same
|
|
113
|
-
# knobs. Returns +nil+ when the inner's +for_sub_agent+
|
|
114
|
-
# returned +nil+ — there's nothing to wrap.
|
|
115
|
-
#
|
|
116
|
-
# @param params [Hash{Symbol => Object}] forwarded to the
|
|
117
|
-
# inner listener's +for_sub_agent+, same protocol as
|
|
118
|
-
# {ListenerList#for_sub_agent}
|
|
119
|
-
# @return [RateLimited, nil]
|
|
120
|
-
def for_sub_agent(**params)
|
|
121
|
-
inner_sub = @inner.respond_to?(:for_sub_agent) ? @inner.for_sub_agent(**params) : @inner
|
|
122
|
-
return nil if inner_sub.nil?
|
|
123
|
-
|
|
124
|
-
self.class.new(inner_sub, fps: @fps,
|
|
125
|
-
flush_pending_on_non_delta: @flush_on_non_delta,
|
|
126
|
-
clock: @clock)
|
|
127
|
-
end
|
|
128
|
-
|
|
129
|
-
# @return [String] short label for {Agent#to_s}; embeds the
|
|
130
|
-
# inner listener's label so the chain reads end-to-end in
|
|
131
|
-
# {ListenerList#to_s}
|
|
132
|
-
def to_s
|
|
133
|
-
policy = @flush_on_non_delta ? 'flush' : 'drop'
|
|
134
|
-
"RateLimited(#{@fps}fps, #{policy}, #{@inner})"
|
|
135
|
-
end
|
|
136
|
-
|
|
137
|
-
private
|
|
138
|
-
|
|
139
|
-
# Advance to the next tick strictly after +now+, in one shot.
|
|
140
|
-
# Handles the case where deltas paused for several periods —
|
|
141
|
-
# a long-idle stream doesn't backlog ticks against the next
|
|
142
|
-
# stream's first delta.
|
|
143
|
-
# @return [void]
|
|
144
|
-
def tick
|
|
145
|
-
now = @clock.call
|
|
146
|
-
return if now < @next_tick_at
|
|
147
|
-
|
|
148
|
-
flush_thinking
|
|
149
|
-
flush_assistant
|
|
150
|
-
missed = ((now - @next_tick_at) / @period).floor + 1
|
|
151
|
-
@next_tick_at += missed * @period
|
|
152
|
-
end
|
|
153
|
-
|
|
154
|
-
# @return [void]
|
|
155
|
-
def flush_assistant
|
|
156
|
-
return if @assistant_buffer.empty?
|
|
157
|
-
|
|
158
|
-
@inner.on_event(Event::AssistantDelta.new(content: @assistant_buffer.dup))
|
|
159
|
-
@assistant_buffer.clear
|
|
160
|
-
end
|
|
161
|
-
|
|
162
|
-
# @return [void]
|
|
163
|
-
def flush_thinking
|
|
164
|
-
return if @thinking_buffer.empty?
|
|
165
|
-
|
|
166
|
-
@inner.on_event(Event::ThinkingDelta.new(content: @thinking_buffer.dup))
|
|
167
|
-
@thinking_buffer.clear
|
|
168
|
-
end
|
|
169
|
-
end
|
|
170
|
-
end
|
|
171
|
-
end
|
|
172
|
-
end
|
|
@@ -1,171 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
class Agent
|
|
5
|
-
# Prompt builder for the step-exhaustion rescue. When an
|
|
6
|
-
# +Agent+'s {Control::StepLimit} trips with the +:synthesize+
|
|
7
|
-
# policy, +Agent#run_loop+ runs this module's prompt on a
|
|
8
|
-
# nested tools-free agent so the run still produces something
|
|
9
|
-
# useful — an assistant turn that answers the user's question
|
|
10
|
-
# from whatever evidence the failed agent collected before
|
|
11
|
-
# running out of budget.
|
|
12
|
-
#
|
|
13
|
-
# == Why this exists
|
|
14
|
-
#
|
|
15
|
-
# Without a rescue, a step-exhausted run just raises a stack
|
|
16
|
-
# trace past +bin/pikuri-chat+ and the user gets nothing
|
|
17
|
-
# despite the agent having gathered useful information in the
|
|
18
|
-
# first N-1 steps. The observed failure mode is the "wait,
|
|
19
|
-
# but what about X?" death-loop: the agent collects sound
|
|
20
|
-
# evidence in the first few rounds, then spends the rest of
|
|
21
|
-
# the budget second-guessing. By the time the cap trips, the
|
|
22
|
-
# answer is largely in the messages — it just needs a
|
|
23
|
-
# tools-free pass to synthesize.
|
|
24
|
-
#
|
|
25
|
-
# Salvage is the wrong move for some agents, which is why the
|
|
26
|
-
# policy lives on {Control::StepLimit} and defaults to
|
|
27
|
-
# +:raise+ — a coding agent's half-finished work can't be
|
|
28
|
-
# completed by a tools-free pass, only described. See
|
|
29
|
-
# {Control::StepLimit}'s class header.
|
|
30
|
-
#
|
|
31
|
-
# == Seam discipline
|
|
32
|
-
#
|
|
33
|
-
# This module is pure prompt construction — no chat handling,
|
|
34
|
-
# no +RubyLLM.chat+ call, no event wiring. The execution side
|
|
35
|
-
# (constructing the nested agent, sharing the parent's
|
|
36
|
-
# listener stream and cancellable, capturing the answer) is
|
|
37
|
-
# +Agent#run_synthesizer+'s job: the synth is a regular
|
|
38
|
-
# tools-free +Agent+, the same construction shape the +agent+
|
|
39
|
-
# tool from +pikuri-subagents+ uses for sub-agents. The only
|
|
40
|
-
# +RubyLLM::*+ surface read here is the value-type
|
|
41
|
-
# +RubyLLM::Message+ / +ToolCall+ passthrough (per the
|
|
42
|
-
# value-type rule in CLAUDE.md).
|
|
43
|
-
module Synthesizer
|
|
44
|
-
# The synthesizer's system prompt. Strict and short: use
|
|
45
|
-
# the evidence, don't apologize, admit gaps when present.
|
|
46
|
-
SYSTEM_PROMPT = <<~PROMPT
|
|
47
|
-
You are given evidence another agent collected before running out of steps. Answer the user's question using only this evidence. You have no tools. If the evidence is insufficient, state plainly what's missing and what partial answer you can give. Do not apologize or comment on the previous agent.
|
|
48
|
-
PROMPT
|
|
49
|
-
|
|
50
|
-
# Render the user's question plus an "Evidence gathered"
|
|
51
|
-
# section built from +parent_messages+ as a single prompt
|
|
52
|
-
# string. Pure function — no I/O, safe to test directly
|
|
53
|
-
# with fixture messages.
|
|
54
|
-
#
|
|
55
|
-
# @param parent_messages [Array<RubyLLM::Message>]
|
|
56
|
-
# @param user_message [String]
|
|
57
|
-
# @return [String]
|
|
58
|
-
def self.build_prompt(parent_messages:, user_message:)
|
|
59
|
-
transcript = format_evidence(parent_messages)
|
|
60
|
-
"Question: #{user_message}\n\nEvidence gathered:\n#{transcript}"
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
# Walk the parent's message history and produce a paired
|
|
64
|
-
# "Tool call:" / "Tool result:" log, preserving order. Tool
|
|
65
|
-
# calls that have no matching +:tool+ message are dropped —
|
|
66
|
-
# the call that tripped the step limit never executed, so
|
|
67
|
-
# including it would mislead the synth into citing
|
|
68
|
-
# nonexistent results. Non-empty assistant text content is
|
|
69
|
-
# preserved as a "Note:" line, since the parent may have
|
|
70
|
-
# summarized progress between tool calls.
|
|
71
|
-
#
|
|
72
|
-
# @param messages [Array<RubyLLM::Message>]
|
|
73
|
-
# @return [String]
|
|
74
|
-
def self.format_evidence(messages)
|
|
75
|
-
results_by_id = messages
|
|
76
|
-
.select { |m| m.role == :tool }
|
|
77
|
-
.to_h { |m| [m.tool_call_id, m.content] }
|
|
78
|
-
|
|
79
|
-
lines = []
|
|
80
|
-
messages.each do |msg|
|
|
81
|
-
next unless msg.role == :assistant
|
|
82
|
-
|
|
83
|
-
text = msg.content
|
|
84
|
-
lines << "Note: #{text}" if text.is_a?(String) && !text.empty?
|
|
85
|
-
|
|
86
|
-
msg.tool_calls&.each_value do |tc|
|
|
87
|
-
result = results_by_id[tc.id]
|
|
88
|
-
next unless result
|
|
89
|
-
|
|
90
|
-
args = tc.arguments.map { |k, v| "#{k}=#{v.inspect}" }.join(', ')
|
|
91
|
-
lines << "Tool call: #{tc.name}(#{args})"
|
|
92
|
-
lines << "Tool result: #{result}"
|
|
93
|
-
lines << ''
|
|
94
|
-
end
|
|
95
|
-
end
|
|
96
|
-
lines.join("\n").rstrip
|
|
97
|
-
end
|
|
98
|
-
private_class_method :format_evidence
|
|
99
|
-
|
|
100
|
-
# The +:synthesize+ arm of the step-exhaustion policy (see the
|
|
101
|
-
# class header). Runs the {Synthesizer} prompt over the
|
|
102
|
-
# exhausted chat's history on a nested tools-free +Agent+ —
|
|
103
|
-
# the same construction shape the +agent+ tool from
|
|
104
|
-
# +pikuri-subagents+ uses for sub-agents, so the synth gets
|
|
105
|
-
# listener propagation, transport / context-window-cap /
|
|
106
|
-
# streaming inheritance, and teardown via +close+ for free.
|
|
107
|
-
# The synth's answer is returned.
|
|
108
|
-
#
|
|
109
|
-
# @param ctx [ExtensionContext]
|
|
110
|
-
# @param chat_messages [Array<RubyLLM::Message>] the
|
|
111
|
-
# exhausted chat's full message history, the evidence
|
|
112
|
-
# {.build_prompt} renders
|
|
113
|
-
# @param user_message [String] the user's original question
|
|
114
|
-
# from the turn that exhausted
|
|
115
|
-
# @raise [Control::Cancellable::Cancelled] when a cancel
|
|
116
|
-
# landed between the budget tripping and this rescue —
|
|
117
|
-
# cancellation wins over salvage
|
|
118
|
-
# @return [String] the synth answer
|
|
119
|
-
def self.run_synthesizer(ctx, chat_messages, user_message)
|
|
120
|
-
# Check the cancel flag *before* constructing the synth: the
|
|
121
|
-
# nested run_loop resets the shared cancellable at its turn
|
|
122
|
-
# boundary, which would erase a cancel requested in this
|
|
123
|
-
# window. The raise propagates without a parent-side
|
|
124
|
-
# {Event::Cancelled} — a cancel *during* synthesis emits it
|
|
125
|
-
# from the synth's own rescue (on the derived listener list)
|
|
126
|
-
# instead, so either way the stream sees at most one.
|
|
127
|
-
ctx.agent.cancellable&.check!
|
|
128
|
-
|
|
129
|
-
ctx.emit_event(Event::FallbackNotice.new(
|
|
130
|
-
reason: "agent exhausted #{ctx.agent.step_limit.max} steps; " \
|
|
131
|
-
'synthesizing answer from gathered evidence'
|
|
132
|
-
))
|
|
133
|
-
|
|
134
|
-
# Synth runs under this agent's identity but with a
|
|
135
|
-
# different system prompt, so it gets a distinct
|
|
136
|
-
# +_synthesizer+ suffix on the id — same +_+ separator the
|
|
137
|
-
# sub-agent generator uses, so main becomes +"synthesizer"+
|
|
138
|
-
# and a sub-agent +"researcher 0"+ becomes
|
|
139
|
-
# +"researcher 0_synthesizer"+. Any +TokenLog+ in the list
|
|
140
|
-
# tags the synth's prompt under that bracket so it's
|
|
141
|
-
# obvious from the log which turns were the rescue rather
|
|
142
|
-
# than the original loop.
|
|
143
|
-
synth_id = ctx.agent.id.empty? ? 'synthesizer' : "#{ctx.agent.id}_synthesizer"
|
|
144
|
-
synth = Agent.new(
|
|
145
|
-
# Carry the parent's resolved cap on the transport so the synth
|
|
146
|
-
# reuses it without a re-probe — the cap rides {ChatTransport}
|
|
147
|
-
# now, not an +Agent.new(context_window:)+ kwarg.
|
|
148
|
-
transport: ctx.agent.transport.with(context_window: ctx.agent.context_window_cap),
|
|
149
|
-
system_prompt: Synthesizer::SYSTEM_PROMPT,
|
|
150
|
-
# Defensive budget with the default :raise policy: the
|
|
151
|
-
# synth has no tools so it should never tick, but a buggy
|
|
152
|
-
# provider that somehow returns a tool call must not loop
|
|
153
|
-
# forever — and a synth that needs its own synth is a bug,
|
|
154
|
-
# not a rescue.
|
|
155
|
-
step_limit: Control::StepLimit.new(max: 1),
|
|
156
|
-
cancellable: ctx.agent.cancellable,
|
|
157
|
-
id: synth_id,
|
|
158
|
-
streaming: ctx.agent.streaming
|
|
159
|
-
) { |c| c.add_listeners(ctx.sub_agent_listeners(id: synth_id)) }
|
|
160
|
-
begin
|
|
161
|
-
synth.run_loop(user_message: Synthesizer.build_prompt(
|
|
162
|
-
parent_messages: chat_messages, user_message: user_message
|
|
163
|
-
))
|
|
164
|
-
synth.last_assistant_content
|
|
165
|
-
ensure
|
|
166
|
-
synth.close
|
|
167
|
-
end
|
|
168
|
-
end
|
|
169
|
-
end
|
|
170
|
-
end
|
|
171
|
-
end
|
|
@@ -1,303 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'json'
|
|
4
|
-
require 'nokogiri'
|
|
5
|
-
require 'readability'
|
|
6
|
-
require 'reverse_markdown'
|
|
7
|
-
|
|
8
|
-
module Pikuri
|
|
9
|
-
module Extractor
|
|
10
|
-
# HTML → Markdown extractor.
|
|
11
|
-
#
|
|
12
|
-
# Matched by content-type only (+text/html+ /
|
|
13
|
-
# +application/xhtml+xml+) — deliberately no byte sniff. The web
|
|
14
|
-
# path always has the header; for local files a sniff would route
|
|
15
|
-
# +Workspace::Read+ of an +.html+ source file through readability
|
|
16
|
-
# extraction, when a developer reading an HTML file wants the
|
|
17
|
-
# source. Local HTML stays on the {Passthrough} arm until a
|
|
18
|
-
# consumer genuinely needs otherwise.
|
|
19
|
-
#
|
|
20
|
-
# Always renders both views of the page when available:
|
|
21
|
-
#
|
|
22
|
-
# 1. JSON-LD section. Any +<script type="application/ld+json">+ node
|
|
23
|
-
# whose +@type+ matches a substantive schema.org content type
|
|
24
|
-
# (Product, Article, Recipe, ...) is rendered as a header — title,
|
|
25
|
-
# metadata bullets (brand, SKU, price, rating, author, published),
|
|
26
|
-
# and the +articleBody+/+description+ copy when present.
|
|
27
|
-
# 2. Readability section. The page is run through +Readability+ +
|
|
28
|
-
# +reverse_markdown+, with a +<main>+/+<article>+ fallback for
|
|
29
|
-
# pages whose content sits mostly outside +<p>+ tags.
|
|
30
|
-
#
|
|
31
|
-
# Concatenated with a horizontal rule, so the LLM gets both the
|
|
32
|
-
# structured metadata and the rendered body and can pick whichever
|
|
33
|
-
# is more useful for the task. Trades some duplication (when a
|
|
34
|
-
# publisher embeds the article body in JSON-LD AND in HTML) for
|
|
35
|
-
# fewer type-based heuristics on which branch should win — the
|
|
36
|
-
# earlier "is this Article's +description+ a teaser or the real
|
|
37
|
-
# body?" carve-out is no longer needed because both end up in
|
|
38
|
-
# the output regardless.
|
|
39
|
-
module HTML
|
|
40
|
-
# @return [Array<String>] content-types this extractor claims.
|
|
41
|
-
CONTENT_TYPES = %w[text/html application/xhtml+xml].freeze
|
|
42
|
-
|
|
43
|
-
# @return [Array<String>] schema.org +@type+ values that we treat
|
|
44
|
-
# as "the primary entity of this page" when picking a JSON-LD
|
|
45
|
-
# node to render. Order does not matter — the first matching
|
|
46
|
-
# node wins. Skips noise nodes (Organization, BreadcrumbList,
|
|
47
|
-
# WebSite, ...) that ship on most pages but carry no page
|
|
48
|
-
# content.
|
|
49
|
-
INTERESTING_TYPES = %w[
|
|
50
|
-
Product Article NewsArticle BlogPosting Recipe Event Book Movie
|
|
51
|
-
].freeze
|
|
52
|
-
|
|
53
|
-
# @return [Array<String>] HTML tags preserved by the readability
|
|
54
|
-
# pass. Anything outside this list is stripped before Markdown
|
|
55
|
-
# conversion.
|
|
56
|
-
READABILITY_TAGS = %w[
|
|
57
|
-
h1 h2 h3 h4 h5 h6 p div span ul ol li blockquote pre code a img
|
|
58
|
-
strong em b i br hr table thead tbody tr td th
|
|
59
|
-
].freeze
|
|
60
|
-
|
|
61
|
-
# @return [Array<String>] HTML attributes preserved by the
|
|
62
|
-
# readability pass; everything else (class, id, style, data-*)
|
|
63
|
-
# is dropped before Markdown conversion
|
|
64
|
-
READABILITY_ATTRS = %w[href src alt title].freeze
|
|
65
|
-
|
|
66
|
-
# @return [Float] minimum +<main>+/+<article>+ to Readability
|
|
67
|
-
# text-length ratio that triggers the semantic-container
|
|
68
|
-
# fallback in {.readability_to_markdown}. Picked low enough to
|
|
69
|
-
# catch the failure mode (Readability collapsing a page that
|
|
70
|
-
# uses divs/lists instead of +<p>+ — e.g. +vaadin.com/company+,
|
|
71
|
-
# ~5x) but high enough that pages where both produce
|
|
72
|
-
# comparable output keep Readability's noise filtering.
|
|
73
|
-
MAIN_FALLBACK_RATIO = 2.0
|
|
74
|
-
|
|
75
|
-
# @return [Integer] minimum text length the
|
|
76
|
-
# +<main>+/+<article>+ container must hold before the fallback
|
|
77
|
-
# in {.readability_to_markdown} can fire. Below this, the
|
|
78
|
-
# ratio comparison is dominated by noise and we'd swap on
|
|
79
|
-
# tiny pages where Readability is doing the right thing.
|
|
80
|
-
MAIN_FALLBACK_MIN_CHARS = 500
|
|
81
|
-
|
|
82
|
-
# @return [Symbol] {Page#kind} tag.
|
|
83
|
-
def self.kind
|
|
84
|
-
:html
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
# @param sample [String] leading bytes of the content (unused —
|
|
88
|
-
# see the no-sniff rationale in the module doc).
|
|
89
|
-
# @param content_type [String, nil] normalized content-type.
|
|
90
|
-
# @return [Boolean]
|
|
91
|
-
def self.matches?(sample:, content_type:)
|
|
92
|
-
CONTENT_TYPES.include?(content_type)
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
# Render the HTML document behind +io+ as Markdown by emitting
|
|
96
|
-
# both the JSON-LD section (when an interesting node is present)
|
|
97
|
-
# and the readability / +<main>+ section, joined by a horizontal
|
|
98
|
-
# rule. Either section may be missing — pages with no JSON-LD
|
|
99
|
-
# return only the readability output, and a malformed page with
|
|
100
|
-
# no extractable body returns only the JSON-LD render.
|
|
101
|
-
#
|
|
102
|
-
# @param io [IO, StringIO] IO over the HTML document.
|
|
103
|
-
# @return [String] Markdown representation
|
|
104
|
-
def self.extract(io)
|
|
105
|
-
html = io.read
|
|
106
|
-
sections = [jsonld_section(html), readability_to_markdown(html)]
|
|
107
|
-
sections.reject! { |s| s.nil? || s.strip.empty? }
|
|
108
|
-
sections.join("\n\n---\n\n")
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
# Pick the first JSON-LD node whose +@type+ matches one of
|
|
112
|
-
# {INTERESTING_TYPES} and render it as Markdown. Returns +nil+
|
|
113
|
-
# when no such node exists, in which case {.extract} emits only
|
|
114
|
-
# the readability section.
|
|
115
|
-
#
|
|
116
|
-
# No content-field gating: a node carrying just +name+/+author+/
|
|
117
|
-
# +datePublished+ still renders (as a metadata-only header),
|
|
118
|
-
# because the readability pass independently produces the page
|
|
119
|
-
# body. That is the trade-off that lets us drop the type-based
|
|
120
|
-
# "is this teaser or article copy?" heuristics — duplication is
|
|
121
|
-
# acceptable when both views are available, and the LLM can
|
|
122
|
-
# pick whichever it needs.
|
|
123
|
-
#
|
|
124
|
-
# @param html [String] HTML document body
|
|
125
|
-
# @return [String, nil] Markdown render of the picked JSON-LD
|
|
126
|
-
# node, or +nil+ when nothing matched
|
|
127
|
-
def self.jsonld_section(html)
|
|
128
|
-
node = parse_jsonld(html).find do |n|
|
|
129
|
-
Array(n['@type']).any? { |t| INTERESTING_TYPES.include?(t) }
|
|
130
|
-
end
|
|
131
|
-
node ? jsonld_to_markdown(node) : nil
|
|
132
|
-
end
|
|
133
|
-
|
|
134
|
-
# Collect every JSON-LD payload embedded in +html+, flattening
|
|
135
|
-
# +@graph+ wrappers so callers see one flat array of schema.org
|
|
136
|
-
# nodes. Malformed JSON blocks are silently skipped — sites
|
|
137
|
-
# frequently ship broken JSON-LD and we only need at least one
|
|
138
|
-
# parseable block.
|
|
139
|
-
#
|
|
140
|
-
# @param html [String] HTML document body
|
|
141
|
-
# @return [Array<Hash>] parsed JSON-LD nodes; possibly empty
|
|
142
|
-
def self.parse_jsonld(html)
|
|
143
|
-
doc = Nokogiri::HTML(html)
|
|
144
|
-
blobs = doc.css('script[type="application/ld+json"]').map(&:text)
|
|
145
|
-
|
|
146
|
-
blobs.flat_map do |raw|
|
|
147
|
-
parsed = begin
|
|
148
|
-
JSON.parse(raw)
|
|
149
|
-
rescue JSON::ParserError
|
|
150
|
-
nil
|
|
151
|
-
end
|
|
152
|
-
next [] unless parsed
|
|
153
|
-
|
|
154
|
-
nodes = parsed.is_a?(Array) ? parsed : [parsed]
|
|
155
|
-
nodes.flat_map { |n| n['@graph'].is_a?(Array) ? n['@graph'] : [n] }
|
|
156
|
-
end
|
|
157
|
-
end
|
|
158
|
-
|
|
159
|
-
# Render a single JSON-LD +node+ as Markdown: a top-level title
|
|
160
|
-
# from +name+/+headline+, a bullet list of common useful fields
|
|
161
|
-
# (brand, SKU, price, rating, author, published date, ...), the
|
|
162
|
-
# body copy, and the lead image.
|
|
163
|
-
#
|
|
164
|
-
# When the node carries +articleBody+ (the full publisher-supplied
|
|
165
|
-
# article text), that wins over +description+ — the description
|
|
166
|
-
# is typically a lede teaser and would just repeat the article's
|
|
167
|
-
# opening lines.
|
|
168
|
-
#
|
|
169
|
-
# @param node [Hash] JSON-LD node, typically picked by
|
|
170
|
-
# {.jsonld_section}
|
|
171
|
-
# @return [String] Markdown representation
|
|
172
|
-
def self.jsonld_to_markdown(node)
|
|
173
|
-
out = +''
|
|
174
|
-
name = node['name'] || node['headline']
|
|
175
|
-
out << "# #{name}\n\n" if name
|
|
176
|
-
|
|
177
|
-
offer = first_obj(node['offers'])
|
|
178
|
-
rating = first_obj(node['aggregateRating'])
|
|
179
|
-
brand = first_obj_or_string(node['brand'])
|
|
180
|
-
author = first_obj_or_string(node['author'])
|
|
181
|
-
|
|
182
|
-
brand_name = brand.is_a?(Hash) ? brand['name'] : brand
|
|
183
|
-
author_name = author.is_a?(Hash) ? author['name'] : author
|
|
184
|
-
|
|
185
|
-
fields = {
|
|
186
|
-
'Brand' => brand_name,
|
|
187
|
-
'SKU' => node['sku'],
|
|
188
|
-
'GTIN' => node['gtin13'] || node['gtin'],
|
|
189
|
-
'Price' => [offer['price'], offer['priceCurrency']].compact.join(' '),
|
|
190
|
-
'Availability' => offer['availability'],
|
|
191
|
-
'Rating' => rating['ratingValue'],
|
|
192
|
-
'Reviews' => rating['reviewCount'],
|
|
193
|
-
'Author' => author_name,
|
|
194
|
-
'Published' => node['datePublished']
|
|
195
|
-
}.reject { |_, v| v.nil? || v.to_s.strip.empty? }
|
|
196
|
-
|
|
197
|
-
unless fields.empty?
|
|
198
|
-
fields.each { |k, v| out << "- **#{k}:** #{v}\n" }
|
|
199
|
-
out << "\n"
|
|
200
|
-
end
|
|
201
|
-
|
|
202
|
-
if (body = node['articleBody'] || node['description'])
|
|
203
|
-
out << "#{body}\n\n"
|
|
204
|
-
end
|
|
205
|
-
|
|
206
|
-
if (img = node['image'])
|
|
207
|
-
img = img.first if img.is_a?(Array)
|
|
208
|
-
img = img['url'] if img.is_a?(Hash)
|
|
209
|
-
out << "\n\n" if img
|
|
210
|
-
end
|
|
211
|
-
|
|
212
|
-
out
|
|
213
|
-
end
|
|
214
|
-
|
|
215
|
-
# Run +Readability+ over +html+ to isolate the main content node,
|
|
216
|
-
# then convert that to Markdown via +reverse_markdown+. The page
|
|
217
|
-
# +<title>+ is rendered as a top-level heading.
|
|
218
|
-
#
|
|
219
|
-
# When the page uses semantic HTML5 (+<main>+ or +<article>+) but
|
|
220
|
-
# leaves most of its content outside +<p>+ tags — divs, lists,
|
|
221
|
-
# spans — Readability's paragraph-density scoring collapses the
|
|
222
|
-
# extraction to a sliver of the page. In that case we render the
|
|
223
|
-
# +<main>+/+<article>+ container directly. The fallback only
|
|
224
|
-
# fires when the container holds substantially more text than
|
|
225
|
-
# Readability picked up (see {MAIN_FALLBACK_RATIO} /
|
|
226
|
-
# {MAIN_FALLBACK_MIN_CHARS}); on pages where both agree we keep
|
|
227
|
-
# Readability so its noise filtering still strips nav/ads/etc.
|
|
228
|
-
#
|
|
229
|
-
# @param html [String] HTML document body
|
|
230
|
-
# @return [String] Markdown representation
|
|
231
|
-
def self.readability_to_markdown(html)
|
|
232
|
-
rdoc = Readability::Document.new(
|
|
233
|
-
html,
|
|
234
|
-
tags: READABILITY_TAGS,
|
|
235
|
-
attributes: READABILITY_ATTRS,
|
|
236
|
-
remove_empty_nodes: true
|
|
237
|
-
)
|
|
238
|
-
readability_html = rdoc.content
|
|
239
|
-
title = rdoc.title
|
|
240
|
-
|
|
241
|
-
body_html = main_fallback_html(html, readability_html) || readability_html
|
|
242
|
-
body = ReverseMarkdown.convert(body_html, unknown_tags: :bypass, github_flavored: true)
|
|
243
|
-
|
|
244
|
-
out = +''
|
|
245
|
-
out << "# #{title.strip}\n\n" if title && !title.strip.empty?
|
|
246
|
-
out << body
|
|
247
|
-
out
|
|
248
|
-
end
|
|
249
|
-
|
|
250
|
-
# If +html+ has a +<main>+ or +<article>+ element holding
|
|
251
|
-
# substantially more text than Readability extracted, return that
|
|
252
|
-
# container's HTML so the caller can render it instead. Returns
|
|
253
|
-
# +nil+ when the fallback should not fire — when there is no
|
|
254
|
-
# semantic container, when it's too small to be meaningful, or
|
|
255
|
-
# when Readability's output is already comparable.
|
|
256
|
-
#
|
|
257
|
-
# @param html [String] full HTML document body, used to locate
|
|
258
|
-
# the +<main>+/+<article>+ container
|
|
259
|
-
# @param readability_html [String] HTML produced by
|
|
260
|
-
# +Readability::Document#content+, used as the comparison
|
|
261
|
-
# baseline
|
|
262
|
-
# @return [String, nil] container HTML when the fallback should
|
|
263
|
-
# fire, +nil+ otherwise
|
|
264
|
-
def self.main_fallback_html(html, readability_html)
|
|
265
|
-
doc = Nokogiri::HTML(html)
|
|
266
|
-
container = doc.at_css('main') || doc.at_css('article')
|
|
267
|
-
return nil unless container
|
|
268
|
-
|
|
269
|
-
container_text_len = container.text.gsub(/\s+/, ' ').strip.length
|
|
270
|
-
return nil if container_text_len < MAIN_FALLBACK_MIN_CHARS
|
|
271
|
-
|
|
272
|
-
readability_text_len = Nokogiri::HTML(readability_html).text.gsub(/\s+/, ' ').strip.length
|
|
273
|
-
return nil if container_text_len < MAIN_FALLBACK_RATIO * readability_text_len
|
|
274
|
-
|
|
275
|
-
container.to_html
|
|
276
|
-
end
|
|
277
|
-
private_class_method :main_fallback_html
|
|
278
|
-
|
|
279
|
-
# JSON-LD fields can be a string, hash, or array of either.
|
|
280
|
-
# Normalize to a single hash (the first one if it's a list) so
|
|
281
|
-
# callers can +.dig+ safely.
|
|
282
|
-
#
|
|
283
|
-
# @param value [Object] raw JSON-LD field value
|
|
284
|
-
# @return [Hash] empty hash when +value+ does not contain a hash
|
|
285
|
-
def self.first_obj(value)
|
|
286
|
-
value = value.first if value.is_a?(Array)
|
|
287
|
-
value.is_a?(Hash) ? value : {}
|
|
288
|
-
end
|
|
289
|
-
private_class_method :first_obj
|
|
290
|
-
|
|
291
|
-
# Same idea as {.first_obj} but preserves a bare string (e.g.
|
|
292
|
-
# +brand: "Apple"+) instead of replacing it with +{}+.
|
|
293
|
-
#
|
|
294
|
-
# @param value [Object] raw JSON-LD field value
|
|
295
|
-
# @return [String, Hash, nil]
|
|
296
|
-
def self.first_obj_or_string(value)
|
|
297
|
-
value = value.first if value.is_a?(Array)
|
|
298
|
-
value
|
|
299
|
-
end
|
|
300
|
-
private_class_method :first_obj_or_string
|
|
301
|
-
end
|
|
302
|
-
end
|
|
303
|
-
end
|