pikuri-core 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +16 -29
  3. data/lib/pikuri/agent/chat_transport.rb +114 -33
  4. data/lib/pikuri/agent/configurator.rb +191 -110
  5. data/lib/pikuri/agent/control/cancellable.rb +25 -16
  6. data/lib/pikuri/agent/control/handoff.rb +75 -0
  7. data/lib/pikuri/agent/control/interloper.rb +87 -120
  8. data/lib/pikuri/agent/control.rb +9 -11
  9. data/lib/pikuri/agent/event.rb +73 -61
  10. data/lib/pikuri/agent/extension.rb +45 -81
  11. data/lib/pikuri/agent/history.rb +91 -497
  12. data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
  13. data/lib/pikuri/agent/listener/terminal.rb +115 -108
  14. data/lib/pikuri/agent/listener/token_log.rb +21 -55
  15. data/lib/pikuri/agent/listener.rb +4 -4
  16. data/lib/pikuri/agent/listener_list.rb +19 -13
  17. data/lib/pikuri/agent/request_instrumenter.rb +102 -0
  18. data/lib/pikuri/agent.rb +624 -448
  19. data/lib/pikuri/attachment.rb +65 -0
  20. data/lib/pikuri/extractor/passthrough.rb +5 -2
  21. data/lib/pikuri/extractor.rb +43 -12
  22. data/lib/pikuri/file_type.rb +4 -4
  23. data/lib/pikuri/finalizers.rb +2 -2
  24. data/lib/pikuri/paths.rb +1 -1
  25. data/lib/pikuri/sanitizer.rb +4 -3
  26. data/lib/pikuri/subprocess.rb +17 -5
  27. data/lib/pikuri/testing.rb +122 -49
  28. data/lib/pikuri/tool/calculator.rb +134 -30
  29. data/lib/pikuri/tool/execute_context.rb +2 -2
  30. data/lib/pikuri/tool/parameters.rb +10 -0
  31. data/lib/pikuri/tool/result.rb +164 -0
  32. data/lib/pikuri/tool/trifecta_legs.rb +52 -21
  33. data/lib/pikuri/tool.rb +298 -24
  34. data/lib/pikuri/trifecta/report.rb +28 -30
  35. data/lib/pikuri/trifecta.rb +1 -2
  36. data/lib/pikuri/version.rb +1 -1
  37. data/lib/pikuri-core.rb +5 -11
  38. metadata +16 -73
  39. data/lib/pikuri/agent/context_window_detector.rb +0 -181
  40. data/lib/pikuri/agent/control/step_limit.rb +0 -95
  41. data/lib/pikuri/agent/extension_context.rb +0 -120
  42. data/lib/pikuri/agent/listener/rate_limited.rb +0 -146
  43. data/lib/pikuri/agent/synthesizer.rb +0 -129
  44. data/lib/pikuri/extractor/html.rb +0 -256
  45. data/lib/pikuri/ruby_llm_patches.rb +0 -106
  46. data/lib/pikuri/tool/fetch.rb +0 -146
  47. data/lib/pikuri/tool/scraper.rb +0 -144
  48. data/lib/pikuri/tool/search/brave.rb +0 -170
  49. data/lib/pikuri/tool/search/duckduckgo.rb +0 -175
  50. data/lib/pikuri/tool/search/engines.rb +0 -319
  51. data/lib/pikuri/tool/search/exa.rb +0 -195
  52. data/lib/pikuri/tool/search/rate_limiter.rb +0 -115
  53. data/lib/pikuri/tool/search/result.rb +0 -24
  54. data/lib/pikuri/tool/web_scrape.rb +0 -105
  55. data/lib/pikuri/tool/web_search.rb +0 -133
  56. data/lib/pikuri/url_cache.rb +0 -96
@@ -1,146 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Pikuri
4
- class Tool
5
- # Truncation policy and Tool spec for the +fetch+ tool. HTTP work lives in
6
- # {Tool::Scraper.fetch}; this wrapper accepts only textual content-types,
7
- # applies a character cap, and exposes the result in OpenAI tool-call shape.
8
- # Sister of {Tool::WebScrape} but with no extraction pass — bodies are
9
- # returned verbatim, for raw textual data (JSON APIs, CSV, +robots.txt+,
10
- # sitemaps, source files) where a rendering pass would corrupt the payload.
11
- module Fetch
12
- # @return [Integer] default character cap on the body from {.fetch}.
13
- # Smaller than {Tool::WebScrape::DEFAULT_MAX_CHARS} because fetch's
14
- # content is bimodal — most JSON/XML/CSV is tiny, and large dumps are
15
- # better re-requested deliberately than padded into every default.
16
- DEFAULT_MAX_CHARS = 5_000
17
-
18
- # @return [Integer] hard ceiling on {.fetch}'s +max_chars+; matches
19
- # {Tool::WebScrape::MAX_MAX_CHARS}.
20
- MAX_MAX_CHARS = 100_000
21
-
22
- # Application content-types that are textual in practice, so safe to
23
- # return verbatim despite failing the +text/*+ check. Anything outside
24
- # +text/*+ and this list is refused.
25
- # @return [Array<String>]
26
- TEXTUAL_APPLICATION_TYPES = %w[
27
- application/json
28
- application/xml
29
- application/javascript
30
- application/xhtml+xml
31
- application/rss+xml
32
- application/atom+xml
33
- ].freeze
34
-
35
- # On-disk cache for {.fetch}, in its own subdir so a +fetch+ and a
36
- # +web_scrape+ on the same URL don't collide (raw body vs extracted
37
- # Markdown). A method so specs can swap {UrlCache::NULL}.
38
- #
39
- # @return [UrlCache, #fetch]
40
- CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/fetch")
41
- # Accessor for {CACHE}; specs swap in {UrlCache::NULL} or an isolated cache.
42
- #
43
- # @return [UrlCache, #fetch]
44
- def self.cache
45
- CACHE
46
- end
47
-
48
- # Download +url+ via {Tool::Scraper.fetch} and return the body verbatim,
49
- # if the content-type is textual (any +text/*+ plus
50
- # {TEXTUAL_APPLICATION_TYPES}); anything else (PDF, image, binary) yields
51
- # an +"Error: ..."+ string. Cached on disk keyed by URL; +max_chars+ is
52
- # *not* in the key (values share one entry, truncation runs after the
53
- # lookup). Cached only on success — {Scraper::FetchError} is caught
54
- # outside +cache.fetch+, so failures are never persisted; other
55
- # exceptions bubble up.
56
- #
57
- # @param url [String] absolute HTTP(S) URL
58
- # @param max_chars [Integer] body cap, clamped to +[1, {MAX_MAX_CHARS}]+;
59
- # default {DEFAULT_MAX_CHARS}. Over the cap, output is cut with a marker.
60
- # @return [String] body, possibly truncated, or +"Error: ..."+ on a
61
- # recoverable failure
62
- def self.fetch(url, max_chars: DEFAULT_MAX_CHARS)
63
- max_chars = max_chars.clamp(1, MAX_MAX_CHARS)
64
- body = cache.fetch(url) { download(url) }
65
- truncate(body, max_chars)
66
- rescue Scraper::FetchError => e
67
- "Error: #{e.message}"
68
- end
69
-
70
- # GET +url+ and verify the content-type is textual. Always hits the
71
- # network (caller caches/truncates).
72
- #
73
- # @param url [String]
74
- # @return [String] response body
75
- # @raise [Scraper::FetchError] on HTTP non-2xx, network failure, redirect
76
- # exhaustion, missing +Location+, or a non-textual content-type
77
- def self.download(url)
78
- fetched = Scraper.fetch(url)
79
- return fetched.body if textual?(fetched.content_type)
80
-
81
- raise Scraper::FetchError,
82
- "refused to fetch #{url}: content-type #{fetched.content_type.inspect} " \
83
- 'is not textual (use web_scrape for rendered pages)'
84
- end
85
-
86
- # @param content_type [String] normalized content-type from {Scraper.fetch}
87
- # @return [Boolean] true when +text/*+ or in {TEXTUAL_APPLICATION_TYPES}
88
- def self.textual?(content_type)
89
- content_type.start_with?('text/') ||
90
- TEXTUAL_APPLICATION_TYPES.include?(content_type)
91
- end
92
-
93
- # Cut +body+ to at most +max_chars+, appending a marker with the original
94
- # length when truncation happens. Same shape as {Tool::WebScrape.truncate}.
95
- #
96
- # @param body [String] full response body
97
- # @param max_chars [Integer] cap; assumed already clamped
98
- # @return [String]
99
- def self.truncate(body, max_chars)
100
- return body if body.length <= max_chars
101
-
102
- "#{body[0, max_chars]}\n\n" \
103
- "... [truncated at #{max_chars} of #{body.length} chars; " \
104
- 'call again with a larger `max_chars` to see more]'
105
- end
106
- end
107
-
108
- # Verbatim URL download tool — a thin wrapper over {Tool::Fetch.fetch} in
109
- # OpenAI tool-call shape. Use for raw textual payloads (JSON, CSV,
110
- # +robots.txt+, source files); use {Tool::WEB_SCRAPE} for rendered pages.
111
- #
112
- # Sharing: +P_shared_benign+ — no per-agent state, but {Fetch::CACHE} is
113
- # process-wide (and on disk, so cross-process too): agent B's +fetch+ can
114
- # be answered from agent A's download within the TTL. See
115
- # {Pikuri::UrlCache} for the race and why it's harmless.
116
- #
117
- # @return [Tool]
118
- FETCH = new(
119
- name: 'fetch',
120
- description: <<~DESC,
121
- Downloads the given URL and returns its body verbatim.
122
-
123
- Usage:
124
- - Use for raw textual payloads: JSON APIs, CSV files, robots.txt, sitemaps, source files — anywhere a rendering pass would corrupt the data.
125
- - For rendered HTML pages, use web_scrape — it extracts readable content; fetch returns the raw HTML bytes unchanged.
126
- - Accepts text/* and common textual application/* types (JSON, XML, JS, XHTML, RSS, Atom). Refuses PDFs, images, and other binaries.
127
- DESC
128
- parameters: Parameters.build { |p|
129
- p.required_string :url,
130
- 'Absolute URL to download, including the scheme, ' \
131
- 'e.g. "https://example.com/data.json".'
132
- p.optional_integer :max_chars,
133
- 'Maximum number of characters of the body to ' \
134
- 'return. Defaults to 5000; hard-capped at ' \
135
- '100000. When the body is longer than this, ' \
136
- 'output is cut and a marker reports the full ' \
137
- 'length.'
138
- },
139
- execute: ->(url:, max_chars: Fetch::DEFAULT_MAX_CHARS) {
140
- Fetch.fetch(url, max_chars: max_chars)
141
- },
142
- # As {WEB_SCRAPE}: attacker-choosable host, attacker-authored response.
143
- trifecta_legs: Pikuri::Tool::TrifectaLegs::ASSUMED
144
- )
145
- end
146
- end
@@ -1,144 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'faraday'
4
- require 'stringio'
5
- require 'uri'
6
-
7
- module Pikuri
8
- class Tool
9
- # HTTP side of the web tools ({Tool::WEB_SCRAPE} and {Tool::FETCH}): GET the
10
- # URL with a real-browser User-Agent, follow redirects, and hand the body to
11
- # {Pikuri::Extractor.extract} with the response +Content-Type+ as the hint.
12
- # HTML/XHTML renders via {Extractor::HTML}, other +text/*+ passes through
13
- # verbatim, plug-in extractors extend the set (with pikuri-pdf,
14
- # +application/pdf+ extracts — by header or +%PDF-+ magic, so a lying header
15
- # still works); remaining types raise {FetchError} so the LLM observes the
16
- # failure rather than an empty rendering.
17
- #
18
- # Split into a thin HTTP fetch ({.fetch}) and the extraction wrapper
19
- # ({.visit}) so tests drive each in isolation and {Tool::Fetch} can reuse the
20
- # HTTP half. Nothing here knows about the LLM — the wrapping tools own
21
- # caching, truncation, and turning the result into an observation.
22
- module Scraper
23
- # Raised when a URL can't be rendered to Markdown — HTTP non-2xx, network
24
- # failure, redirect loop, missing +Location+, unsupported content-type, or
25
- # a parse failure. The web tools rescue this into an +"Error: ..."+
26
- # observation; anything else bubbles up so genuine bugs stay visible.
27
- class FetchError < StandardError; end
28
-
29
- # @return [String] User-Agent sent with each request; many sites reject a
30
- # missing or obviously-bot UA.
31
- USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
32
- '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
33
- # @return [String] +Accept+ header, so content-negotiating servers hand
34
- # back something usable: HTML first, then +application/pdf+, then +text/*+.
35
- ACCEPT = 'text/html,application/xhtml+xml,application/pdf,text/*;q=0.8'
36
- # @return [Integer] maximum HTTP redirects to follow before giving up.
37
- MAX_REDIRECTS = 5
38
- # @return [Integer] connect timeout (s) for the Faraday request.
39
- OPEN_TIMEOUT = 10
40
- # @return [Integer] read timeout (s) for the Faraday request.
41
- READ_TIMEOUT = 20
42
-
43
- # @return [Integer] max chars of an error response body in a {FetchError}
44
- # message — enough to show what came back (a Cloudflare/WAF page) without
45
- # flooding the observation.
46
- ERROR_BODY_EXCERPT = 200
47
-
48
- # Result of a successful {Scraper.fetch}: the response body, the
49
- # normalized content-type (lower-cased, with any +; charset=...+
50
- # parameters stripped), and the final URL after redirects.
51
- Fetched = Data.define(:body, :content_type, :url)
52
-
53
- # Fetch +url+ and render its main content as Markdown. No caching — every
54
- # call hits the network (wrap it to memoize; {Tool::WebScrape.visit}
55
- # does). The output is +strip+'d (leading/trailing blank lines are common
56
- # from PDF page-feeds and trailing newlines) but interior whitespace is
57
- # preserved (paragraph breaks and code indentation are load-bearing).
58
- #
59
- # @param url [String] absolute HTTP(S) URL
60
- # @return [String] Markdown, whitespace-trimmed, otherwise uncapped —
61
- # caller size-limits before feeding the LLM
62
- # @raise [FetchError] on HTTP non-2xx, network failure, redirect loop, a
63
- # 3xx without +Location+, an unrecognized response, or extraction failure
64
- def self.visit(url)
65
- extract(fetch(url)).strip
66
- end
67
-
68
- # Render a {Fetched} response as Markdown via {Pikuri::Extractor.extract},
69
- # re-raising both extraction failure modes as {FetchError}. The
70
- # content-type is passed verbatim (including +""+ for a missing header,
71
- # which matches no text arm — a body without transport metadata is
72
- # refused, not sniffed; only a strong magic sniff like +%PDF-+ overrides).
73
- #
74
- # @param fetched [Fetched]
75
- # @return [String] Markdown from the matched extractor
76
- # @raise [FetchError] when no extractor matches, or extraction fails
77
- def self.extract(fetched)
78
- Pikuri::Extractor.extract(StringIO.new(fetched.body), content_type: fetched.content_type)
79
- rescue Pikuri::Extractor::Unsupported
80
- raise FetchError, "unsupported content-type #{fetched.content_type.inspect} for #{fetched.url}"
81
- rescue Pikuri::Extractor::Error => e
82
- raise FetchError, e.message
83
- end
84
-
85
- # Download +url+, manually following up to {MAX_REDIRECTS} redirects. All
86
- # recoverable failures (HTTP 4xx/5xx, +Faraday::Error+, exhausted redirect
87
- # budget, 3xx without +Location+) surface as {FetchError}. Error bodies
88
- # are trimmed to {ERROR_BODY_EXCERPT} so a challenge page doesn't dump
89
- # kilobytes into the observation.
90
- #
91
- # @param url [String] absolute HTTP(S) URL
92
- # @param limit [Integer] redirects remaining; recurses with +limit - 1+ on 3xx
93
- # @return [Fetched] body, normalized content-type, final URL
94
- # @raise [FetchError] on non-2xx/3xx, network error, redirect-loop
95
- # exhaustion, or 3xx without +Location+
96
- def self.fetch(url, limit: MAX_REDIRECTS)
97
- raise FetchError, "too many redirects fetching #{url}" if limit.zero?
98
-
99
- response = begin
100
- Faraday.new(request: { open_timeout: OPEN_TIMEOUT, timeout: READ_TIMEOUT }).get(url) do |req|
101
- req.headers['User-Agent'] = USER_AGENT
102
- req.headers['Accept'] = ACCEPT
103
- end
104
- rescue Faraday::Error => e
105
- raise FetchError, "#{e.class.name.split('::').last} fetching #{url}: #{e.message}"
106
- end
107
-
108
- case response.status
109
- when 200..299
110
- Fetched.new(body: response.body, content_type: normalize_content_type(response.headers['content-type']), url: url)
111
- when 300..399
112
- location = response.headers['location']
113
- raise FetchError, "HTTP #{response.status} from #{url} with no Location header" if location.nil? || location.empty?
114
-
115
- fetch(URI.join(url, location).to_s, limit: limit - 1)
116
- else
117
- raise FetchError, "HTTP #{response.status} fetching #{url}: #{excerpt(response.body)}"
118
- end
119
- end
120
-
121
- # Lower-case +raw+ and strip any +; charset=...+ parameters so the
122
- # extractors can match on a canonical token.
123
- #
124
- # @param raw [String, nil] raw +Content-Type+ header value
125
- # @return [String] normalized content-type, or +""+ when the
126
- # header was missing
127
- def self.normalize_content_type(raw)
128
- raw.to_s.split(';').first.to_s.strip.downcase
129
- end
130
- private_class_method :normalize_content_type
131
-
132
- # Whitespace-collapse +body+ and clip to {ERROR_BODY_EXCERPT}, keeping the
133
- # {FetchError} message a single readable line.
134
- #
135
- # @param body [String, nil]
136
- # @return [String]
137
- def self.excerpt(body)
138
- text = body.to_s.gsub(/\s+/, ' ').strip
139
- text.length > ERROR_BODY_EXCERPT ? "#{text[0, ERROR_BODY_EXCERPT]}..." : text
140
- end
141
- private_class_method :excerpt
142
- end
143
- end
144
- end
@@ -1,170 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'faraday'
4
- require 'json'
5
- require 'nokogiri'
6
-
7
- module Pikuri
8
- class Tool
9
- module Search
10
- # Performs a Brave Search via the official Web Search API, returning hits
11
- # as {Result} rows. Split into a thin HTTP fetch (#search) and a pure
12
- # parser (.parse) so tests exercise the parser against fixture JSON.
13
- # {Engines#search} owns the final Markdown rendering.
14
- #
15
- # Constructed with its API key (+Brave.new(api_key:)+); {Engines} builds
16
- # one only when a Brave key was configured. pikuri reads no key from the
17
- # environment (CLAUDE.md "Environment is not a secret store"). Key at
18
- # https://api-dashboard.search.brave.com — the free "Data for Search" tier
19
- # allows 1 query/sec, ~2k/month.
20
- #
21
- # == Privacy posture
22
- #
23
- # Brave's API notice retains Search Query Logs 90 days and collects no
24
- # identifiers linking a query to a person; it publicly commits not to train
25
- # on query data, and offers Zero Data Retention — but only on Enterprise,
26
- # not the free tier pikuri defaults to. Bottom line: the cleanest
27
- # API-level posture of pikuri's three providers (no training, no IP
28
- # linkage, capped 90-day retention), but a logged 90-day window on the
29
- # cheap tier, so not a ZDR substitute for genuinely sensitive queries.
30
- class Brave
31
- # @return [String] Web Search endpoint
32
- ENDPOINT = 'https://api.search.brave.com/res/v1/web/search'
33
- # @return [Integer] default number of results returned.
34
- DEFAULT_MAX_RESULTS = 10
35
- # @return [RateLimiter] free-tier Brave caps at 1 req/sec; the 5-minute
36
- # cooldown protects the monthly quota from doomed retries on a 429.
37
- LIMITER = RateLimiter.new(min_interval: 1.0, cooldown: 300.0)
38
-
39
- # @param api_key [String] Brave Search subscription token. Required
40
- # and non-blank: pikuri reads no key from the environment — the
41
- # host supplies it ({Engines} only constructs a Brave when a key
42
- # was configured).
43
- # @raise [ArgumentError] if +api_key+ is blank
44
- def initialize(api_key:)
45
- raise ArgumentError, 'Brave Search API key is blank' if api_key.to_s.strip.empty?
46
-
47
- @api_key = api_key
48
- end
49
-
50
- # @return [String] short provider label for {Engines}.
51
- def label
52
- 'Brave'
53
- end
54
-
55
- # Fetch results for +query+ as an +Array<Result>+. Throttled to 1/sec and
56
- # circuit-broken 5 min on rate-limit/quota (see {LIMITER}). The caller
57
- # ({Engines#search}) normalizes the query and wraps this in a cache.
58
- #
59
- # @param query [String] search query (already normalized)
60
- # @param max_results [Integer] max entries; Brave's +count+ (1..20)
61
- # @param cancellable [Pikuri::Agent::Control::Cancellable] makes
62
- # {LIMITER}'s pacing wait interruptible; see {RateLimiter#call}.
63
- # @return [Array<Result>] hits, possibly empty when Brave matched nothing
64
- # @raise [Engines::Unavailable] on HTTP 429 or 5xx (the cascade falls
65
- # back), or if {LIMITER} is in cooldown. Other non-2xx (401/403 bad
66
- # key) bubble up as +RuntimeError+.
67
- # @raise [RuntimeError] for non-rate-limit HTTP failures, or a
68
- # zero-result response that isn't a recognized empty-results payload.
69
- def search(query, max_results: DEFAULT_MAX_RESULTS,
70
- cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
71
- LIMITER.call(cancellable: cancellable) do
72
- response = Faraday.get(
73
- ENDPOINT,
74
- { q: query, count: max_results },
75
- { 'X-Subscription-Token' => @api_key, 'Accept' => 'application/json' }
76
- )
77
- unless response.success?
78
- if response.status == 429 || response.status >= 500
79
- raise Engines::Unavailable, "HTTP #{response.status}"
80
- end
81
-
82
- raise "Brave Search request failed: #{response.status} #{response.body}"
83
- end
84
-
85
- self.class.parse(response.body, max_results: max_results)
86
- end
87
- end
88
-
89
- # Parse a Brave JSON response into {Result} rows (+<strong>+ highlights
90
- # stripped). A genuine "no results" payload (recognized search shape,
91
- # empty +mixed.main/top/side+) returns +[]+ so {Engines.search} renders
92
- # its stub; any other zero-result shape raises.
93
- #
94
- # @param json [String] response body from {ENDPOINT}
95
- # @param max_results [Integer] max entries
96
- # @return [Array<Result>] hits, possibly empty on a recognized
97
- # empty-results payload
98
- # @raise [RuntimeError] on an unrecognized zero-result response
99
- def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
100
- data = JSON.parse(json)
101
- results = Array(data.dig('web', 'results')).take(max_results).filter_map do |r|
102
- href = r['url'].to_s
103
- next nil if href.empty?
104
-
105
- Result.new(
106
- url: href,
107
- title: strip_html(r['title']),
108
- body: strip_html(r['description'])
109
- )
110
- end
111
-
112
- if results.empty?
113
- return [] if genuine_no_results?(data)
114
-
115
- raise diagnose_empty(data, json)
116
- end
117
-
118
- results
119
- end
120
-
121
- # Strip HTML markup (notably the +<strong>+ highlights Brave wraps around
122
- # query terms) and collapse whitespace.
123
- #
124
- # @param html [String, nil]
125
- # @return [String] plain text; empty string for nil
126
- def self.strip_html(html)
127
- return '' if html.nil?
128
-
129
- Nokogiri::HTML.fragment(html).text.gsub(/\s+/, ' ').strip
130
- end
131
- private_class_method :strip_html
132
-
133
- # True when a zero-+web.results+ response is Brave's own "ran, matched
134
- # nothing" payload, not an error. Markers: the +type: "search"+ envelope
135
- # and an empty +mixed+ block (Brave fills +mixed.main/top/side+ with
136
- # matched verticals, so all three empty = succeeded-but-found-nothing).
137
- #
138
- # @param data [Hash, Object] parsed response
139
- # @return [Boolean]
140
- def self.genuine_no_results?(data)
141
- return false unless data.is_a?(Hash) && data['type'] == 'search'
142
-
143
- mixed = data['mixed']
144
- return false unless mixed.is_a?(Hash)
145
-
146
- %w[main top side].all? { |k| Array(mixed[k]).empty? }
147
- end
148
- private_class_method :genuine_no_results?
149
-
150
- # Build an error message for a zero-result response: quote Brave's
151
- # +error.detail+ if present, else a truncated raw body.
152
- #
153
- # @param data [Hash, Object] parsed response
154
- # @param raw [String] raw response body
155
- # @return [String] diagnostic to feed to +raise+
156
- def self.diagnose_empty(data, raw)
157
- if data.is_a?(Hash) && data['error'].is_a?(Hash)
158
- err = data['error']
159
- return "Brave Search returned an error: #{[err['code'], err['detail'] || err['meta']].compact.join(' — ')}"
160
- end
161
-
162
- snippet = raw.to_s[0, 800]
163
- snippet += '…' if raw.to_s.length > 800
164
- "Brave Search returned no results. Body: #{snippet}"
165
- end
166
- private_class_method :diagnose_empty
167
- end
168
- end
169
- end
170
- end
@@ -1,175 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'faraday'
4
- require 'nokogiri'
5
- require 'uri'
6
-
7
- module Pikuri
8
- class Tool
9
- module Search
10
- # Performs a DuckDuckGo search by scraping +html.duckduckgo.com+, returning
11
- # hits as {Result} rows. Split into a thin HTTP fetch (#search) and a pure
12
- # parser (.parse) so tests exercise the parser against fixture HTML.
13
- # {Engines#search} owns the final Markdown rendering. Constructed with no
14
- # arguments, sharing the uniform +#search+/+#label+ provider shape with the
15
- # keyed {Brave} / {Exa}.
16
- #
17
- # == Privacy posture
18
- #
19
- # DDG doesn't save your IP or tie a per-user profile to queries, and
20
- # proxies requests so downstream providers can't build a search history —
21
- # real, but DDG relays web results over Bing, so *query content* still
22
- # reaches Microsoft (who has no no-training pledge). Better than Exa for
23
- # sensitive queries, worse than Brave; for anything genuinely embarrassing,
24
- # don't search the web at all.
25
- class DuckDuckGo
26
- # @return [String] HTML search endpoint
27
- ENDPOINT = 'https://html.duckduckgo.com/html/'
28
- # @return [String] User-Agent sent with each request; DDG often rejects
29
- # a missing or obviously-bot UA.
30
- USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
31
- '(KHTML, like Gecko) Chrome/120.0 Safari/537.36'
32
- # @return [Integer] default number of results returned.
33
- DEFAULT_MAX_RESULTS = 10
34
- # @return [RateLimiter] paces calls (DDG bans IPs that hammer the HTML
35
- # endpoint) and circuit-breaks 5 min on {Engines::Unavailable}.
36
- LIMITER = RateLimiter.new(min_interval: 2.0, cooldown: 300.0)
37
-
38
- # @return [String] short provider label for {Engines}.
39
- def label
40
- 'DuckDuckGo'
41
- end
42
-
43
- # Fetch results for +query+ as an +Array<Result>+. Throttled to one every
44
- # 2s and circuit-broken 5 min after a soft-block (see {LIMITER}). The
45
- # caller ({Engines#search}) normalizes the query and wraps this in a cache.
46
- #
47
- # @param query [String] search query (already normalized)
48
- # @param max_results [Integer] max result entries
49
- # @param cancellable [Pikuri::Agent::Control::Cancellable] makes
50
- # {LIMITER}'s pacing wait interruptible; see {RateLimiter#call}.
51
- # @return [Array<Result>] hits, possibly empty when DDG matched nothing
52
- # @raise [Engines::Unavailable] on a soft-block (anomaly/CAPTCHA page) or
53
- # HTTP 429/5xx (the cascade falls back), or if {LIMITER} is in cooldown.
54
- # @raise [RuntimeError] on other HTTP failures or an unrecognized
55
- # empty-results layout. A genuine empty-results page is not an error;
56
- # see {.parse}.
57
- def search(query, max_results: DEFAULT_MAX_RESULTS,
58
- cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
59
- LIMITER.call(cancellable: cancellable) do
60
- response = Faraday.get(ENDPOINT, { q: query }, { 'User-Agent' => USER_AGENT })
61
- unless response.success?
62
- if response.status == 429 || response.status >= 500
63
- raise Engines::Unavailable, "HTTP #{response.status}"
64
- end
65
-
66
- raise "DuckDuckGo request failed: #{response.status} #{response.body}"
67
- end
68
-
69
- self.class.parse(response.body, max_results: max_results)
70
- end
71
- end
72
-
73
- # Parse a +html.duckduckgo.com+ result page into {Result} rows (+<b>+
74
- # highlights stripped). On zero result nodes: a genuine "no results" page
75
- # returns +[]+ (so {Engines.search} renders its stub); the anomaly/CAPTCHA
76
- # modal raises {Engines::Unavailable}; anything else raises {RuntimeError}
77
- # (likely a layout change) — so an IP soft-block surfaces rather than
78
- # masquerading as an empty search.
79
- #
80
- # @param html [String] HTML from html.duckduckgo.com
81
- # @param max_results [Integer] max result entries
82
- # @return [Array<Result>] hits, possibly empty on a genuine no-results page
83
- # @raise [Engines::Unavailable] on the anomaly/CAPTCHA modal (IP soft-block)
84
- # @raise [RuntimeError] on zero results with an unrecognized layout
85
- def self.parse(html, max_results: DEFAULT_MAX_RESULTS)
86
- doc = Nokogiri::HTML(html)
87
- results = doc.css('div.result.web-result').take(max_results).filter_map do |node|
88
- title_link = node.at_css('a.result__a')
89
- next nil if title_link.nil?
90
-
91
- snippet = node.at_css('a.result__snippet')
92
- Result.new(
93
- url: extract_url(title_link['href']),
94
- title: title_link.text.strip,
95
- body: snippet&.text&.strip.to_s
96
- )
97
- end
98
-
99
- if results.empty?
100
- return [] if genuine_no_results?(doc)
101
-
102
- message = diagnose_empty(doc)
103
- raise(anomaly_modal?(doc) ? Engines::Unavailable : RuntimeError, message)
104
- end
105
-
106
- results
107
- end
108
-
109
- # True when the page is DDG's anomaly/CAPTCHA modal (IP soft-blocked).
110
- # {.parse} uses it to pick {Engines::Unavailable} (recoverable) over
111
- # {RuntimeError} (unknown layout).
112
- #
113
- # @param doc [Nokogiri::HTML::Document]
114
- # @return [Boolean]
115
- def self.anomaly_modal?(doc)
116
- !!(doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description'))
117
- end
118
- private_class_method :anomaly_modal?
119
-
120
- # True when a zero-node page is DDG's own "no results found" page, not an
121
- # anomaly/CAPTCHA. Anomaly wins: if the modal divs are present it's never
122
- # a genuine empty result, even if the text mentions "No results".
123
- #
124
- # @param doc [Nokogiri::HTML::Document]
125
- # @return [Boolean]
126
- def self.genuine_no_results?(doc)
127
- return false if doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description')
128
- return true if doc.at_css('div.no-results')
129
-
130
- doc.text.include?('No results found')
131
- end
132
- private_class_method :genuine_no_results?
133
-
134
- # Build an error message for a zero-match page: quote the DDG anomaly
135
- # modal's title/description if present, else a truncated copy of the
136
- # body's visible text (+<script>/<style>/<noscript>+ stripped).
137
- #
138
- # @param doc [Nokogiri::HTML::Document]
139
- # @return [String] diagnostic to feed to +raise+
140
- def self.diagnose_empty(doc)
141
- title = doc.at_css('.anomaly-modal__title')&.text&.strip
142
- desc = doc.at_css('.anomaly-modal__description')&.text&.strip
143
- if title || desc
144
- return "DuckDuckGo anomaly check (likely IP soft-block): #{[title, desc].compact.reject(&:empty?).join(' — ')}"
145
- end
146
-
147
- doc.css('script, style, noscript').remove
148
- text = doc.text.gsub(/\s+/, ' ').strip
149
- snippet = text.empty? ? '<empty body>' : text[0, 1500]
150
- snippet += '…' if text.length > 1500
151
- "DuckDuckGo returned no results. Page text: #{snippet}"
152
- end
153
- private_class_method :diagnose_empty
154
-
155
- # Decode DDG's +//duckduckgo.com/l/?uddg=<encoded>+ redirect wrapper to
156
- # the real target URL.
157
- #
158
- # @param href [String, nil] href from the result page
159
- # @return [String] the decoded target, or +href+ unchanged when not a
160
- # recognized DDG redirect or unparseable
161
- def self.extract_url(href)
162
- return href if href.nil? || href.empty?
163
-
164
- uri = URI.parse(href.start_with?('//') ? "https:#{href}" : href)
165
- return href unless uri.host&.end_with?('duckduckgo.com') && uri.path == '/l/'
166
-
167
- params = URI.decode_www_form(uri.query.to_s).to_h
168
- params['uddg'] || href
169
- rescue URI::InvalidURIError
170
- href
171
- end
172
- end
173
- end
174
- end
175
- end