pikuri-core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +16 -29
- data/lib/pikuri/agent/chat_transport.rb +114 -33
- data/lib/pikuri/agent/configurator.rb +191 -110
- data/lib/pikuri/agent/control/cancellable.rb +25 -16
- data/lib/pikuri/agent/control/handoff.rb +75 -0
- data/lib/pikuri/agent/control/interloper.rb +87 -120
- data/lib/pikuri/agent/control.rb +9 -11
- data/lib/pikuri/agent/event.rb +73 -61
- data/lib/pikuri/agent/extension.rb +45 -81
- data/lib/pikuri/agent/history.rb +91 -497
- data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
- data/lib/pikuri/agent/listener/terminal.rb +115 -108
- data/lib/pikuri/agent/listener/token_log.rb +21 -55
- data/lib/pikuri/agent/listener.rb +4 -4
- data/lib/pikuri/agent/listener_list.rb +19 -13
- data/lib/pikuri/agent/request_instrumenter.rb +102 -0
- data/lib/pikuri/agent.rb +624 -448
- data/lib/pikuri/attachment.rb +65 -0
- data/lib/pikuri/extractor/passthrough.rb +5 -2
- data/lib/pikuri/extractor.rb +43 -12
- data/lib/pikuri/file_type.rb +4 -4
- data/lib/pikuri/finalizers.rb +2 -2
- data/lib/pikuri/paths.rb +1 -1
- data/lib/pikuri/sanitizer.rb +4 -3
- data/lib/pikuri/subprocess.rb +17 -5
- data/lib/pikuri/testing.rb +122 -49
- data/lib/pikuri/tool/calculator.rb +134 -30
- data/lib/pikuri/tool/execute_context.rb +2 -2
- data/lib/pikuri/tool/parameters.rb +10 -0
- data/lib/pikuri/tool/result.rb +164 -0
- data/lib/pikuri/tool/trifecta_legs.rb +52 -21
- data/lib/pikuri/tool.rb +298 -24
- data/lib/pikuri/trifecta/report.rb +28 -30
- data/lib/pikuri/trifecta.rb +1 -2
- data/lib/pikuri/version.rb +1 -1
- data/lib/pikuri-core.rb +5 -11
- metadata +16 -73
- data/lib/pikuri/agent/context_window_detector.rb +0 -181
- data/lib/pikuri/agent/control/step_limit.rb +0 -95
- data/lib/pikuri/agent/extension_context.rb +0 -120
- data/lib/pikuri/agent/listener/rate_limited.rb +0 -146
- data/lib/pikuri/agent/synthesizer.rb +0 -129
- data/lib/pikuri/extractor/html.rb +0 -256
- data/lib/pikuri/ruby_llm_patches.rb +0 -106
- data/lib/pikuri/tool/fetch.rb +0 -146
- data/lib/pikuri/tool/scraper.rb +0 -144
- data/lib/pikuri/tool/search/brave.rb +0 -170
- data/lib/pikuri/tool/search/duckduckgo.rb +0 -175
- data/lib/pikuri/tool/search/engines.rb +0 -319
- data/lib/pikuri/tool/search/exa.rb +0 -195
- data/lib/pikuri/tool/search/rate_limiter.rb +0 -115
- data/lib/pikuri/tool/search/result.rb +0 -24
- data/lib/pikuri/tool/web_scrape.rb +0 -105
- data/lib/pikuri/tool/web_search.rb +0 -133
- data/lib/pikuri/url_cache.rb +0 -96
|
@@ -1,146 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
class Agent
|
|
5
|
-
module Listener
|
|
6
|
-
# Decorator that coalesces high-frequency streaming deltas
|
|
7
|
-
# ({Event::AssistantDelta}, {Event::ThinkingDelta}) into at most +fps+
|
|
8
|
-
# events/sec before forwarding to an inner listener. Non-delta events flow
|
|
9
|
-
# through unmodified, with any pending coalesced content emitted ahead of
|
|
10
|
-
# them (default) or dropped — see +flush_pending_on_non_delta:+.
|
|
11
|
-
#
|
|
12
|
-
# For renderers whose per-delta repaint costs more than provider chunks
|
|
13
|
-
# arrive: a fast provider emits dozens of deltas/sec, and a
|
|
14
|
-
# Markdown-rendering UI doesn't need every one. +fps: 15+ suits a TUI.
|
|
15
|
-
#
|
|
16
|
-
# == Tick alignment
|
|
17
|
-
#
|
|
18
|
-
# The first flush is anchored to a monotonic-clock reference taken at
|
|
19
|
-
# construction, not the first delta's arrival — so stream-startup latency
|
|
20
|
-
# doesn't leak into the cadence. After a flush the schedule advances by
|
|
21
|
-
# exactly +1/fps+ per missed tick, so a long-idle stream doesn't backlog
|
|
22
|
-
# ticks against itself.
|
|
23
|
-
#
|
|
24
|
-
# == Per-stream buffering
|
|
25
|
-
#
|
|
26
|
-
# Assistant and thinking deltas buffer independently; a tick flushes
|
|
27
|
-
# whichever are non-empty as separate events.
|
|
28
|
-
#
|
|
29
|
-
# == Non-delta handling
|
|
30
|
-
#
|
|
31
|
-
# +flush_pending_on_non_delta: true+ (default) emits pending content ahead
|
|
32
|
-
# of the non-delta event — lossless. +false+ drops it silently:
|
|
33
|
-
# appropriate for inner listeners that re-render the authoritative final
|
|
34
|
-
# content on a bookend ({Event::Assistant}/{Event::Thinking}), where the
|
|
35
|
-
# trailing 0–66 ms of deltas would be wasted CPU before the redraw.
|
|
36
|
-
#
|
|
37
|
-
# No threads or timers: tick state advances only when an event arrives, so
|
|
38
|
-
# it's safe on any thread as long as delivery is sequential (the
|
|
39
|
-
# {ListenerList} contract).
|
|
40
|
-
class RateLimited < Base
|
|
41
|
-
# @param inner [Listener::Base] the listener to forward (coalesced) events to.
|
|
42
|
-
# @param fps [Integer, Float] frames-per-second cap on the coalesced
|
|
43
|
-
# delta stream. Must be positive.
|
|
44
|
-
# @param flush_pending_on_non_delta [Boolean] +true+ (default) emits
|
|
45
|
-
# pending content as a delta before a non-delta event; +false+ drops it.
|
|
46
|
-
# @param clock [Proc] zero-arg → monotonic seconds as a +Float+;
|
|
47
|
-
# injection seam for specs (production uses
|
|
48
|
-
# +Process.clock_gettime(Process::CLOCK_MONOTONIC)+).
|
|
49
|
-
# @raise [ArgumentError] if +fps+ is not positive.
|
|
50
|
-
def initialize(inner, fps:, flush_pending_on_non_delta: true,
|
|
51
|
-
clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
|
|
52
|
-
super()
|
|
53
|
-
raise ArgumentError, "fps must be positive, got #{fps.inspect}" unless fps.positive?
|
|
54
|
-
|
|
55
|
-
@inner = inner
|
|
56
|
-
@fps = fps
|
|
57
|
-
@period = 1.0 / fps
|
|
58
|
-
@flush_on_non_delta = flush_pending_on_non_delta
|
|
59
|
-
@clock = clock
|
|
60
|
-
@next_tick_at = clock.call
|
|
61
|
-
@assistant_buffer = String.new
|
|
62
|
-
@thinking_buffer = String.new
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
# @param event [Agent::Event] one of the {Agent::Event}
|
|
66
|
-
# variants; see {Agent::Event} for the full list
|
|
67
|
-
# @return [void]
|
|
68
|
-
def on_event(event)
|
|
69
|
-
case event
|
|
70
|
-
when Event::AssistantDelta
|
|
71
|
-
@assistant_buffer << event.content
|
|
72
|
-
tick
|
|
73
|
-
when Event::ThinkingDelta
|
|
74
|
-
@thinking_buffer << event.content
|
|
75
|
-
tick
|
|
76
|
-
else
|
|
77
|
-
if @flush_on_non_delta
|
|
78
|
-
flush_thinking
|
|
79
|
-
flush_assistant
|
|
80
|
-
else
|
|
81
|
-
@assistant_buffer.clear
|
|
82
|
-
@thinking_buffer.clear
|
|
83
|
-
end
|
|
84
|
-
@inner.on_event(event)
|
|
85
|
-
end
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
# Sub-agent variant: wraps the inner listener's sub-agent variant (or
|
|
89
|
-
# the inner itself when it has no +for_sub_agent+) in a fresh
|
|
90
|
-
# +RateLimited+ with the same knobs. +nil+ when the inner's returned
|
|
91
|
-
# +nil+.
|
|
92
|
-
#
|
|
93
|
-
# @param params [Hash{Symbol => Object}] forwarded to the inner's
|
|
94
|
-
# +for_sub_agent+ (same protocol as {ListenerList#for_sub_agent})
|
|
95
|
-
# @return [RateLimited, nil]
|
|
96
|
-
def for_sub_agent(**params)
|
|
97
|
-
inner_sub = @inner.respond_to?(:for_sub_agent) ? @inner.for_sub_agent(**params) : @inner
|
|
98
|
-
return nil if inner_sub.nil?
|
|
99
|
-
|
|
100
|
-
self.class.new(inner_sub, fps: @fps,
|
|
101
|
-
flush_pending_on_non_delta: @flush_on_non_delta,
|
|
102
|
-
clock: @clock)
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
# @return [String] short label for {Agent#to_s}, embedding the inner
|
|
106
|
-
# listener's so the chain reads end-to-end.
|
|
107
|
-
def to_s
|
|
108
|
-
policy = @flush_on_non_delta ? 'flush' : 'drop'
|
|
109
|
-
"RateLimited(#{@fps}fps, #{policy}, #{@inner})"
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
private
|
|
113
|
-
|
|
114
|
-
# Advance to the next tick strictly after +now+, in one shot — a stream
|
|
115
|
-
# idle for several periods doesn't backlog ticks against the next
|
|
116
|
-
# stream's first delta.
|
|
117
|
-
# @return [void]
|
|
118
|
-
def tick
|
|
119
|
-
now = @clock.call
|
|
120
|
-
return if now < @next_tick_at
|
|
121
|
-
|
|
122
|
-
flush_thinking
|
|
123
|
-
flush_assistant
|
|
124
|
-
missed = ((now - @next_tick_at) / @period).floor + 1
|
|
125
|
-
@next_tick_at += missed * @period
|
|
126
|
-
end
|
|
127
|
-
|
|
128
|
-
# @return [void]
|
|
129
|
-
def flush_assistant
|
|
130
|
-
return if @assistant_buffer.empty?
|
|
131
|
-
|
|
132
|
-
@inner.on_event(Event::AssistantDelta.new(content: @assistant_buffer.dup))
|
|
133
|
-
@assistant_buffer.clear
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
# @return [void]
|
|
137
|
-
def flush_thinking
|
|
138
|
-
return if @thinking_buffer.empty?
|
|
139
|
-
|
|
140
|
-
@inner.on_event(Event::ThinkingDelta.new(content: @thinking_buffer.dup))
|
|
141
|
-
@thinking_buffer.clear
|
|
142
|
-
end
|
|
143
|
-
end
|
|
144
|
-
end
|
|
145
|
-
end
|
|
146
|
-
end
|
|
@@ -1,129 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
class Agent
|
|
5
|
-
# Prompt builder for the step-exhaustion rescue. When an +Agent+'s
|
|
6
|
-
# {Control::StepLimit} trips with the +:synthesize+ policy, +Agent#run_loop+
|
|
7
|
-
# runs this prompt on a nested tools-free agent so the run still produces an
|
|
8
|
-
# answer from whatever evidence the failed agent gathered before running out
|
|
9
|
-
# of budget.
|
|
10
|
-
#
|
|
11
|
-
# The failure mode it salvages is the "wait, but what about X?" death-loop:
|
|
12
|
-
# the agent collects sound evidence in the first few rounds, then burns the
|
|
13
|
-
# rest of the budget second-guessing — by the cap, the answer is largely in
|
|
14
|
-
# the messages and just needs a tools-free pass to synthesize. Salvage is
|
|
15
|
-
# wrong for some agents (a coding agent's half-finished work can only be
|
|
16
|
-
# described, not completed), which is why the policy lives on
|
|
17
|
-
# {Control::StepLimit} and defaults to +:raise+.
|
|
18
|
-
#
|
|
19
|
-
# Pure prompt construction — no chat handling or +RubyLLM.chat+ call.
|
|
20
|
-
# {.run_synthesizer} owns the execution (a regular tools-free +Agent+, the
|
|
21
|
-
# same shape sub-agents use). The only +RubyLLM::*+ surface read here is the
|
|
22
|
-
# +RubyLLM::Message+/+ToolCall+ value-type passthrough.
|
|
23
|
-
module Synthesizer
|
|
24
|
-
# The synthesizer's system prompt. Strict and short: use
|
|
25
|
-
# the evidence, don't apologize, admit gaps when present.
|
|
26
|
-
SYSTEM_PROMPT = <<~PROMPT
|
|
27
|
-
You are given evidence another agent collected before running out of steps. Answer the user's question using only this evidence. You have no tools. If the evidence is insufficient, state plainly what's missing and what partial answer you can give. Do not apologize or comment on the previous agent.
|
|
28
|
-
PROMPT
|
|
29
|
-
|
|
30
|
-
# Render the question plus an "Evidence gathered" section from
|
|
31
|
-
# +parent_messages+. Pure — no I/O.
|
|
32
|
-
#
|
|
33
|
-
# @param parent_messages [Array<RubyLLM::Message>]
|
|
34
|
-
# @param user_message [String]
|
|
35
|
-
# @return [String]
|
|
36
|
-
def self.build_prompt(parent_messages:, user_message:)
|
|
37
|
-
transcript = format_evidence(parent_messages)
|
|
38
|
-
"Question: #{user_message}\n\nEvidence gathered:\n#{transcript}"
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
# Walk the parent's history into a paired "Tool call:" / "Tool result:"
|
|
42
|
-
# log, in order. Tool calls with no matching +:tool+ message are dropped —
|
|
43
|
-
# the call that tripped the limit never executed, so citing its
|
|
44
|
-
# nonexistent result would mislead the synth. Non-empty assistant text
|
|
45
|
-
# becomes a "Note:" line.
|
|
46
|
-
#
|
|
47
|
-
# @param messages [Array<RubyLLM::Message>]
|
|
48
|
-
# @return [String]
|
|
49
|
-
def self.format_evidence(messages)
|
|
50
|
-
results_by_id = messages
|
|
51
|
-
.select { |m| m.role == :tool }
|
|
52
|
-
.to_h { |m| [m.tool_call_id, m.content] }
|
|
53
|
-
|
|
54
|
-
lines = []
|
|
55
|
-
messages.each do |msg|
|
|
56
|
-
next unless msg.role == :assistant
|
|
57
|
-
|
|
58
|
-
text = msg.content
|
|
59
|
-
lines << "Note: #{text}" if text.is_a?(String) && !text.empty?
|
|
60
|
-
|
|
61
|
-
msg.tool_calls&.each_value do |tc|
|
|
62
|
-
result = results_by_id[tc.id]
|
|
63
|
-
next unless result
|
|
64
|
-
|
|
65
|
-
args = tc.arguments.map { |k, v| "#{k}=#{v.inspect}" }.join(', ')
|
|
66
|
-
lines << "Tool call: #{tc.name}(#{args})"
|
|
67
|
-
lines << "Tool result: #{result}"
|
|
68
|
-
lines << ''
|
|
69
|
-
end
|
|
70
|
-
end
|
|
71
|
-
lines.join("\n").rstrip
|
|
72
|
-
end
|
|
73
|
-
private_class_method :format_evidence
|
|
74
|
-
|
|
75
|
-
# The +:synthesize+ arm of the step-exhaustion policy. Runs the
|
|
76
|
-
# {Synthesizer} prompt over the exhausted chat's history on a nested
|
|
77
|
-
# tools-free +Agent+ (the sub-agent construction shape, so it inherits
|
|
78
|
-
# listener propagation, transport/cap/streaming, and +close+ teardown).
|
|
79
|
-
#
|
|
80
|
-
# @param ctx [ExtensionContext]
|
|
81
|
-
# @param chat_messages [Array<RubyLLM::Message>] the exhausted chat's
|
|
82
|
-
# history, the evidence {.build_prompt} renders
|
|
83
|
-
# @param user_message [String] the user's original question
|
|
84
|
-
# @raise [Control::Cancellable::Cancelled] when a cancel landed between
|
|
85
|
-
# the budget tripping and this rescue — cancellation wins over salvage
|
|
86
|
-
# @return [String] the synth answer
|
|
87
|
-
def self.run_synthesizer(ctx, chat_messages, user_message)
|
|
88
|
-
# Check the cancel flag *before* constructing the synth: the nested
|
|
89
|
-
# run_loop resets the shared cancellable at its turn boundary, which
|
|
90
|
-
# would erase a cancel requested in this window. The raise propagates
|
|
91
|
-
# without a parent-side {Event::Cancelled} — a cancel *during* synthesis
|
|
92
|
-
# emits it from the synth's own rescue instead, so the stream sees at
|
|
93
|
-
# most one.
|
|
94
|
-
ctx.agent.cancellable&.check!
|
|
95
|
-
|
|
96
|
-
ctx.emit_event(Event::FallbackNotice.new(
|
|
97
|
-
reason: "agent exhausted #{ctx.agent.step_limit.max} steps; " \
|
|
98
|
-
'synthesizing answer from gathered evidence'
|
|
99
|
-
))
|
|
100
|
-
|
|
101
|
-
# Synth runs under this agent's identity with a distinct +_synthesizer+
|
|
102
|
-
# id suffix (same +_+ separator the sub-agent generator uses), so a
|
|
103
|
-
# +TokenLog+ tags its turns as the rescue, not the loop.
|
|
104
|
-
synth_id = ctx.agent.id.empty? ? 'synthesizer' : "#{ctx.agent.id}_synthesizer"
|
|
105
|
-
synth = Agent.new(
|
|
106
|
-
# Carry the parent's resolved cap on the transport so the synth reuses
|
|
107
|
-
# it without a re-probe (the cap rides {ChatTransport}).
|
|
108
|
-
transport: ctx.agent.transport.with(context_window: ctx.agent.context_window_cap),
|
|
109
|
-
system_prompt: Synthesizer::SYSTEM_PROMPT,
|
|
110
|
-
# Defensive :raise budget: the synth has no tools so should never
|
|
111
|
-
# tick, but a buggy provider returning a tool call must not loop — a
|
|
112
|
-
# synth that needs its own synth is a bug, not a rescue.
|
|
113
|
-
step_limit: Control::StepLimit.new(max: 1),
|
|
114
|
-
cancellable: ctx.agent.cancellable,
|
|
115
|
-
id: synth_id,
|
|
116
|
-
streaming: ctx.agent.streaming
|
|
117
|
-
) { |c| c.add_listeners(ctx.sub_agent_listeners(id: synth_id)) }
|
|
118
|
-
begin
|
|
119
|
-
synth.run_loop(user_message: Synthesizer.build_prompt(
|
|
120
|
-
parent_messages: chat_messages, user_message: user_message
|
|
121
|
-
))
|
|
122
|
-
synth.last_assistant_content
|
|
123
|
-
ensure
|
|
124
|
-
synth.close
|
|
125
|
-
end
|
|
126
|
-
end
|
|
127
|
-
end
|
|
128
|
-
end
|
|
129
|
-
end
|
|
@@ -1,256 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'json'
|
|
4
|
-
require 'nokogiri'
|
|
5
|
-
require 'readability'
|
|
6
|
-
require 'reverse_markdown'
|
|
7
|
-
|
|
8
|
-
module Pikuri
|
|
9
|
-
module Extractor
|
|
10
|
-
# HTML → Markdown extractor.
|
|
11
|
-
#
|
|
12
|
-
# Matched by content-type only (+text/html+ / +application/xhtml+xml+) —
|
|
13
|
-
# no byte sniff: the web path always has the header, and sniffing would
|
|
14
|
-
# route a +Workspace::Read+ of a local +.html+ *source* file through
|
|
15
|
-
# readability when the developer wants the source. Local HTML stays on the
|
|
16
|
-
# {Passthrough} arm.
|
|
17
|
-
#
|
|
18
|
-
# Renders both views when available, concatenated with a horizontal rule,
|
|
19
|
-
# so the LLM gets structured metadata and rendered body and picks whichever
|
|
20
|
-
# fits:
|
|
21
|
-
#
|
|
22
|
-
# 1. JSON-LD — the first +<script type="application/ld+json">+ node whose
|
|
23
|
-
# +@type+ is a substantive schema.org type ({INTERESTING_TYPES}),
|
|
24
|
-
# rendered as title + metadata bullets + body copy.
|
|
25
|
-
# 2. Readability — +Readability+ + +reverse_markdown+, with a
|
|
26
|
-
# +<main>+/+<article>+ fallback for pages whose content sits outside
|
|
27
|
-
# +<p>+ tags.
|
|
28
|
-
#
|
|
29
|
-
# Emitting both trades some duplication (a body embedded in JSON-LD *and*
|
|
30
|
-
# HTML) for dropping type-based "teaser or real body?" heuristics.
|
|
31
|
-
module HTML
|
|
32
|
-
# @return [Array<String>] content-types this extractor claims.
|
|
33
|
-
CONTENT_TYPES = %w[text/html application/xhtml+xml].freeze
|
|
34
|
-
|
|
35
|
-
# @return [Array<String>] schema.org +@type+ values treated as the page's
|
|
36
|
-
# primary entity when picking a JSON-LD node (first match wins). Skips
|
|
37
|
-
# noise nodes (Organization, BreadcrumbList, WebSite) that carry no
|
|
38
|
-
# content.
|
|
39
|
-
INTERESTING_TYPES = %w[
|
|
40
|
-
Product Article NewsArticle BlogPosting Recipe Event Book Movie
|
|
41
|
-
].freeze
|
|
42
|
-
|
|
43
|
-
# @return [Array<String>] HTML tags preserved by the readability
|
|
44
|
-
# pass. Anything outside this list is stripped before Markdown
|
|
45
|
-
# conversion.
|
|
46
|
-
READABILITY_TAGS = %w[
|
|
47
|
-
h1 h2 h3 h4 h5 h6 p div span ul ol li blockquote pre code a img
|
|
48
|
-
strong em b i br hr table thead tbody tr td th
|
|
49
|
-
].freeze
|
|
50
|
-
|
|
51
|
-
# @return [Array<String>] HTML attributes preserved by the
|
|
52
|
-
# readability pass; everything else (class, id, style, data-*)
|
|
53
|
-
# is dropped before Markdown conversion
|
|
54
|
-
READABILITY_ATTRS = %w[href src alt title].freeze
|
|
55
|
-
|
|
56
|
-
# @return [Float] minimum +<main>+/+<article>+-to-Readability text-length
|
|
57
|
-
# ratio that triggers the semantic-container fallback in
|
|
58
|
-
# {.readability_to_markdown}. Low enough to catch Readability collapsing
|
|
59
|
-
# a div/list-based page (vaadin.com/company, ~5x), high enough to keep
|
|
60
|
-
# its noise filtering where both outputs are comparable.
|
|
61
|
-
MAIN_FALLBACK_RATIO = 2.0
|
|
62
|
-
|
|
63
|
-
# @return [Integer] minimum text length the +<main>+/+<article>+ container
|
|
64
|
-
# must hold before the {.readability_to_markdown} fallback can fire —
|
|
65
|
-
# below it the ratio is noise-dominated and would swap on tiny pages.
|
|
66
|
-
MAIN_FALLBACK_MIN_CHARS = 500
|
|
67
|
-
|
|
68
|
-
# @return [Symbol] {Page#kind} tag.
|
|
69
|
-
def self.kind
|
|
70
|
-
:html
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
# @param sample [String] leading bytes of the content (unused —
|
|
74
|
-
# see the no-sniff rationale in the module doc).
|
|
75
|
-
# @param content_type [String, nil] normalized content-type.
|
|
76
|
-
# @return [Boolean]
|
|
77
|
-
def self.matches?(sample:, content_type:)
|
|
78
|
-
CONTENT_TYPES.include?(content_type)
|
|
79
|
-
end
|
|
80
|
-
|
|
81
|
-
# Render the HTML behind +io+ as Markdown: the JSON-LD section (when an
|
|
82
|
-
# interesting node is present) and the readability/+<main>+ section,
|
|
83
|
-
# joined by a horizontal rule. Either may be missing.
|
|
84
|
-
#
|
|
85
|
-
# @param io [IO, StringIO] IO over the HTML document.
|
|
86
|
-
# @return [String] Markdown representation
|
|
87
|
-
def self.extract(io)
|
|
88
|
-
html = io.read
|
|
89
|
-
sections = [jsonld_section(html), readability_to_markdown(html)]
|
|
90
|
-
sections.reject! { |s| s.nil? || s.strip.empty? }
|
|
91
|
-
sections.join("\n\n---\n\n")
|
|
92
|
-
end
|
|
93
|
-
|
|
94
|
-
# Pick the first JSON-LD node whose +@type+ is in {INTERESTING_TYPES} and
|
|
95
|
-
# render it. +nil+ when none matches. No content-field gating — a
|
|
96
|
-
# metadata-only node (name/author/date) still renders, since the
|
|
97
|
-
# readability pass independently produces the body.
|
|
98
|
-
#
|
|
99
|
-
# @param html [String] HTML document body
|
|
100
|
-
# @return [String, nil] Markdown render, or +nil+ when nothing matched
|
|
101
|
-
def self.jsonld_section(html)
|
|
102
|
-
node = parse_jsonld(html).find do |n|
|
|
103
|
-
Array(n['@type']).any? { |t| INTERESTING_TYPES.include?(t) }
|
|
104
|
-
end
|
|
105
|
-
node ? jsonld_to_markdown(node) : nil
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
# Collect every JSON-LD payload in +html+, flattening +@graph+ wrappers
|
|
109
|
-
# into one flat array. Malformed JSON blocks are skipped (sites ship
|
|
110
|
-
# broken JSON-LD; one parseable block suffices).
|
|
111
|
-
#
|
|
112
|
-
# @param html [String] HTML document body
|
|
113
|
-
# @return [Array<Hash>] parsed nodes; possibly empty
|
|
114
|
-
def self.parse_jsonld(html)
|
|
115
|
-
doc = Nokogiri::HTML(html)
|
|
116
|
-
blobs = doc.css('script[type="application/ld+json"]').map(&:text)
|
|
117
|
-
|
|
118
|
-
blobs.flat_map do |raw|
|
|
119
|
-
parsed = begin
|
|
120
|
-
JSON.parse(raw)
|
|
121
|
-
rescue JSON::ParserError
|
|
122
|
-
nil
|
|
123
|
-
end
|
|
124
|
-
next [] unless parsed
|
|
125
|
-
|
|
126
|
-
nodes = parsed.is_a?(Array) ? parsed : [parsed]
|
|
127
|
-
nodes.flat_map { |n| n['@graph'].is_a?(Array) ? n['@graph'] : [n] }
|
|
128
|
-
end
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
# Render a JSON-LD +node+ as Markdown: title from +name+/+headline+, a
|
|
132
|
-
# bullet list of common fields (brand, SKU, price, rating, author, date),
|
|
133
|
-
# body copy, and the lead image. +articleBody+ wins over +description+
|
|
134
|
-
# (the description is usually a lede teaser).
|
|
135
|
-
#
|
|
136
|
-
# @param node [Hash] JSON-LD node
|
|
137
|
-
# @return [String] Markdown representation
|
|
138
|
-
def self.jsonld_to_markdown(node)
|
|
139
|
-
out = +''
|
|
140
|
-
name = node['name'] || node['headline']
|
|
141
|
-
out << "# #{name}\n\n" if name
|
|
142
|
-
|
|
143
|
-
offer = first_obj(node['offers'])
|
|
144
|
-
rating = first_obj(node['aggregateRating'])
|
|
145
|
-
brand = first_obj_or_string(node['brand'])
|
|
146
|
-
author = first_obj_or_string(node['author'])
|
|
147
|
-
|
|
148
|
-
brand_name = brand.is_a?(Hash) ? brand['name'] : brand
|
|
149
|
-
author_name = author.is_a?(Hash) ? author['name'] : author
|
|
150
|
-
|
|
151
|
-
fields = {
|
|
152
|
-
'Brand' => brand_name,
|
|
153
|
-
'SKU' => node['sku'],
|
|
154
|
-
'GTIN' => node['gtin13'] || node['gtin'],
|
|
155
|
-
'Price' => [offer['price'], offer['priceCurrency']].compact.join(' '),
|
|
156
|
-
'Availability' => offer['availability'],
|
|
157
|
-
'Rating' => rating['ratingValue'],
|
|
158
|
-
'Reviews' => rating['reviewCount'],
|
|
159
|
-
'Author' => author_name,
|
|
160
|
-
'Published' => node['datePublished']
|
|
161
|
-
}.reject { |_, v| v.nil? || v.to_s.strip.empty? }
|
|
162
|
-
|
|
163
|
-
unless fields.empty?
|
|
164
|
-
fields.each { |k, v| out << "- **#{k}:** #{v}\n" }
|
|
165
|
-
out << "\n"
|
|
166
|
-
end
|
|
167
|
-
|
|
168
|
-
if (body = node['articleBody'] || node['description'])
|
|
169
|
-
out << "#{body}\n\n"
|
|
170
|
-
end
|
|
171
|
-
|
|
172
|
-
if (img = node['image'])
|
|
173
|
-
img = img.first if img.is_a?(Array)
|
|
174
|
-
img = img['url'] if img.is_a?(Hash)
|
|
175
|
-
out << "\n\n" if img
|
|
176
|
-
end
|
|
177
|
-
|
|
178
|
-
out
|
|
179
|
-
end
|
|
180
|
-
|
|
181
|
-
# Run +Readability+ over +html+, convert to Markdown via
|
|
182
|
-
# +reverse_markdown+, +<title>+ as a top-level heading. When the page
|
|
183
|
-
# uses HTML5 semantics (+<main>+/+<article>+) but leaves content outside
|
|
184
|
-
# +<p>+ tags, Readability's paragraph-density scoring collapses the
|
|
185
|
-
# extraction; then the container is rendered directly. The fallback fires
|
|
186
|
-
# only when the container holds substantially more text (see
|
|
187
|
-
# {MAIN_FALLBACK_RATIO} / {MAIN_FALLBACK_MIN_CHARS}).
|
|
188
|
-
#
|
|
189
|
-
# @param html [String] HTML document body
|
|
190
|
-
# @return [String] Markdown representation
|
|
191
|
-
def self.readability_to_markdown(html)
|
|
192
|
-
rdoc = Readability::Document.new(
|
|
193
|
-
html,
|
|
194
|
-
tags: READABILITY_TAGS,
|
|
195
|
-
attributes: READABILITY_ATTRS,
|
|
196
|
-
remove_empty_nodes: true
|
|
197
|
-
)
|
|
198
|
-
readability_html = rdoc.content
|
|
199
|
-
title = rdoc.title
|
|
200
|
-
|
|
201
|
-
body_html = main_fallback_html(html, readability_html) || readability_html
|
|
202
|
-
body = ReverseMarkdown.convert(body_html, unknown_tags: :bypass, github_flavored: true)
|
|
203
|
-
|
|
204
|
-
out = +''
|
|
205
|
-
out << "# #{title.strip}\n\n" if title && !title.strip.empty?
|
|
206
|
-
out << body
|
|
207
|
-
out
|
|
208
|
-
end
|
|
209
|
-
|
|
210
|
-
# Return the +<main>+/+<article>+ container's HTML when it holds
|
|
211
|
-
# substantially more text than Readability extracted, else +nil+ (no
|
|
212
|
-
# container, too small, or Readability comparable).
|
|
213
|
-
#
|
|
214
|
-
# @param html [String] full HTML document body
|
|
215
|
-
# @param readability_html [String] +Readability::Document#content+ output,
|
|
216
|
-
# the comparison baseline
|
|
217
|
-
# @return [String, nil] container HTML when the fallback should fire
|
|
218
|
-
def self.main_fallback_html(html, readability_html)
|
|
219
|
-
doc = Nokogiri::HTML(html)
|
|
220
|
-
container = doc.at_css('main') || doc.at_css('article')
|
|
221
|
-
return nil unless container
|
|
222
|
-
|
|
223
|
-
container_text_len = container.text.gsub(/\s+/, ' ').strip.length
|
|
224
|
-
return nil if container_text_len < MAIN_FALLBACK_MIN_CHARS
|
|
225
|
-
|
|
226
|
-
readability_text_len = Nokogiri::HTML(readability_html).text.gsub(/\s+/, ' ').strip.length
|
|
227
|
-
return nil if container_text_len < MAIN_FALLBACK_RATIO * readability_text_len
|
|
228
|
-
|
|
229
|
-
container.to_html
|
|
230
|
-
end
|
|
231
|
-
private_class_method :main_fallback_html
|
|
232
|
-
|
|
233
|
-
# Normalize a JSON-LD field (string, hash, or array of either) to a single
|
|
234
|
-
# hash (first if a list) so callers can +.dig+ safely.
|
|
235
|
-
#
|
|
236
|
-
# @param value [Object] raw JSON-LD field value
|
|
237
|
-
# @return [Hash] empty hash when +value+ contains no hash
|
|
238
|
-
def self.first_obj(value)
|
|
239
|
-
value = value.first if value.is_a?(Array)
|
|
240
|
-
value.is_a?(Hash) ? value : {}
|
|
241
|
-
end
|
|
242
|
-
private_class_method :first_obj
|
|
243
|
-
|
|
244
|
-
# Like {.first_obj} but preserves a bare string (e.g. +brand: "Apple"+)
|
|
245
|
-
# instead of replacing it with +{}+.
|
|
246
|
-
#
|
|
247
|
-
# @param value [Object] raw JSON-LD field value
|
|
248
|
-
# @return [String, Hash, nil]
|
|
249
|
-
def self.first_obj_or_string(value)
|
|
250
|
-
value = value.first if value.is_a?(Array)
|
|
251
|
-
value
|
|
252
|
-
end
|
|
253
|
-
private_class_method :first_obj_or_string
|
|
254
|
-
end
|
|
255
|
-
end
|
|
256
|
-
end
|
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'ruby_llm'
|
|
4
|
-
|
|
5
|
-
module Pikuri
|
|
6
|
-
# Local corrections to ruby_llm, applied at load time, carried until the fix
|
|
7
|
-
# ships upstream. Each patch pins us to a ruby_llm internal's private shape, so
|
|
8
|
-
# it's a liability: small, self-contained, tied to a tracked upstream issue so
|
|
9
|
-
# we know when to delete it. Reopening a third-party class is last resort —
|
|
10
|
-
# here only because the defect is on the live chat path and the release cadence
|
|
11
|
-
# isn't ours. Patching rather than dropping the dependency is the standing
|
|
12
|
-
# policy — `DECISIONS.md` +D_keep_ruby_llm+.
|
|
13
|
-
#
|
|
14
|
-
# == The streaming error-parse crash (ruby_llm #837)
|
|
15
|
-
#
|
|
16
|
-
# {https://github.com/crmne/ruby_llm/issues/837}
|
|
17
|
-
#
|
|
18
|
-
# On a *streaming* turn, a non-200 response whose body parses to a non-Hash
|
|
19
|
-
# JSON value crashes ruby_llm 1.16.0 with +TypeError: String does not have
|
|
20
|
-
# #dig+ at +providers/openai/streaming.rb:43+, instead of surfacing the
|
|
21
|
-
# endpoint's error. Two bodies trigger it, both because the method assumes a
|
|
22
|
-
# nested object to +.dig+:
|
|
23
|
-
#
|
|
24
|
-
# 1. A bare JSON string. The guard +return unless error_data['error']+ is
|
|
25
|
-
# meant as key-presence, but on a +String+ that's substring matching, so a
|
|
26
|
-
# body containing "error" slips through into +.dig+.
|
|
27
|
-
# 2. A +{"error": "<message>"}+ body where +error+ is a string value
|
|
28
|
-
# (xAI/Grok), not an object: the guard passes, then
|
|
29
|
-
# +error_data.dig('error', 'type')+ calls +String#dig+ — same +TypeError+.
|
|
30
|
-
#
|
|
31
|
-
# The non-streaming path already handles both (cases on Hash/Array/else,
|
|
32
|
-
# short-circuits a string +error+). This mirrors that: never +.dig+ a non-Hash,
|
|
33
|
-
# hand a non-typed error up as the message with a +nil+ status so ruby_llm's
|
|
34
|
-
# +build_stream_error_response+ falls back to the real HTTP status.
|
|
35
|
-
#
|
|
36
|
-
# Redefined whole (not +prepend+ + +super+) because it's a small self-contained
|
|
37
|
-
# +module_function+ with no state — faithful reproduction plus the guard is
|
|
38
|
-
# clearer than wrapping the buggy original. Re-running +module_function+
|
|
39
|
-
# replaces both the module method and the private copy the provider gets via
|
|
40
|
-
# +include+.
|
|
41
|
-
#
|
|
42
|
-
# == When to delete this file
|
|
43
|
-
#
|
|
44
|
-
# ruby_llm main already carries an equivalent fix, in a module renamed to
|
|
45
|
-
# +RubyLLM::Protocols::ChatCompletions::Streaming+ — so the next release past
|
|
46
|
-
# 1.16.0 obsoletes the patch *and* moves its target. The load-time guard below
|
|
47
|
-
# turns that into a boot failure naming this file; delete it and its spec then.
|
|
48
|
-
module RubyLlmPatches
|
|
49
|
-
module_function
|
|
50
|
-
|
|
51
|
-
# @return [Boolean] +true+ once the patches below have been applied; a
|
|
52
|
-
# literal, kept honest by the load-time guard that raises if the target
|
|
53
|
-
# module has moved out from under it
|
|
54
|
-
def applied? = true
|
|
55
|
-
end
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
# Reopening a module that has moved silently defines a fresh empty one: the
|
|
59
|
-
# patch would stop applying, the bug would return, and nothing — not even
|
|
60
|
-
# {Pikuri::RubyLlmPatches.applied?} — would notice. Fail at boot instead.
|
|
61
|
-
unless defined?(RubyLLM::Providers::OpenAI::Streaming) &&
|
|
62
|
-
RubyLLM::Providers::OpenAI::Streaming.respond_to?(:parse_streaming_error)
|
|
63
|
-
raise "RubyLLM::Providers::OpenAI::Streaming.parse_streaming_error is gone from ruby_llm " \
|
|
64
|
-
"#{RubyLLM::VERSION} — the streaming error-parse patch no longer applies. It is fixed " \
|
|
65
|
-
'upstream: delete pikuri-core/lib/pikuri/ruby_llm_patches.rb and its spec (ruby_llm #837).'
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
module RubyLLM
|
|
69
|
-
module Providers
|
|
70
|
-
class OpenAI
|
|
71
|
-
module Streaming
|
|
72
|
-
# Parse a streaming error body into a +[status, message]+ pair,
|
|
73
|
-
# tolerating a non-Hash parsed body. See
|
|
74
|
-
# {Pikuri::RubyLlmPatches} for why this override exists and when
|
|
75
|
-
# to remove it (ruby_llm #837).
|
|
76
|
-
#
|
|
77
|
-
# @param data [String] the JSON error body as received
|
|
78
|
-
# @return [Array(Integer, nil), String] a +[status, message]+
|
|
79
|
-
# pair; +status+ is +nil+ when the body carries no error type,
|
|
80
|
-
# deferring classification to the response's HTTP status
|
|
81
|
-
def parse_streaming_error(data)
|
|
82
|
-
error_data = JSON.parse(data)
|
|
83
|
-
# Non-Hash body (a bare JSON string/number/array): no structured
|
|
84
|
-
# error to read — surface it as the message, defer status.
|
|
85
|
-
return [nil, error_data.to_s] unless error_data.is_a?(Hash)
|
|
86
|
-
|
|
87
|
-
error = error_data['error']
|
|
88
|
-
return [nil, error_data.to_s] unless error
|
|
89
|
-
# A string error value (xAI/Grok's {"error": "..."}) has no type
|
|
90
|
-
# field — hand it up as-is, defer status to the HTTP response.
|
|
91
|
-
return [nil, error] if error.is_a?(String)
|
|
92
|
-
|
|
93
|
-
case error['type']
|
|
94
|
-
when 'server_error'
|
|
95
|
-
[500, error['message']]
|
|
96
|
-
when 'rate_limit_exceeded', 'insufficient_quota'
|
|
97
|
-
[429, error['message']]
|
|
98
|
-
else
|
|
99
|
-
[400, error['message']]
|
|
100
|
-
end
|
|
101
|
-
end
|
|
102
|
-
module_function :parse_streaming_error
|
|
103
|
-
end
|
|
104
|
-
end
|
|
105
|
-
end
|
|
106
|
-
end
|