pikuri-core 0.0.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +17 -30
- data/lib/pikuri/agent/chat_transport.rb +168 -107
- data/lib/pikuri/agent/configurator.rb +194 -173
- data/lib/pikuri/agent/control/cancellable.rb +100 -70
- data/lib/pikuri/agent/control/handoff.rb +75 -0
- data/lib/pikuri/agent/control/interloper.rb +107 -118
- data/lib/pikuri/agent/control.rb +13 -35
- data/lib/pikuri/agent/event.rb +144 -197
- data/lib/pikuri/agent/extension.rb +83 -95
- data/lib/pikuri/agent/history.rb +247 -0
- data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
- data/lib/pikuri/agent/listener/terminal.rb +169 -136
- data/lib/pikuri/agent/listener/token_log.rb +84 -157
- data/lib/pikuri/agent/listener.rb +24 -44
- data/lib/pikuri/agent/listener_list.rb +32 -47
- data/lib/pikuri/agent/request_instrumenter.rb +102 -0
- data/lib/pikuri/agent.rb +1095 -577
- data/lib/pikuri/attachment.rb +65 -0
- data/lib/pikuri/bundler_env.rb +68 -0
- data/lib/pikuri/extractor/passthrough.rb +24 -31
- data/lib/pikuri/extractor.rb +125 -155
- data/lib/pikuri/file_type.rb +64 -136
- data/lib/pikuri/finalizers.rb +32 -47
- data/lib/pikuri/paths.rb +104 -13
- data/lib/pikuri/sanitizer.rb +46 -67
- data/lib/pikuri/subprocess.rb +86 -118
- data/lib/pikuri/testing.rb +369 -0
- data/lib/pikuri/tool/calculator.rb +168 -74
- data/lib/pikuri/tool/execute_context.rb +42 -0
- data/lib/pikuri/tool/parameters.rb +31 -29
- data/lib/pikuri/tool/result.rb +164 -0
- data/lib/pikuri/tool/trifecta_legs.rb +248 -0
- data/lib/pikuri/tool.rb +416 -67
- data/lib/pikuri/trifecta/contribution.rb +43 -0
- data/lib/pikuri/trifecta/node.rb +47 -0
- data/lib/pikuri/trifecta/report.rb +228 -0
- data/lib/pikuri/trifecta.rb +126 -0
- data/lib/pikuri/version.rb +1 -1
- data/lib/pikuri-core.rb +70 -92
- data/prompts/agent-loop.txt +5 -0
- data/prompts/pikuri-chat.txt +3 -12
- metadata +26 -72
- data/lib/pikuri/agent/context_window_detector.rb +0 -222
- data/lib/pikuri/agent/control/step_limit.rb +0 -111
- data/lib/pikuri/agent/extension_context.rb +0 -147
- data/lib/pikuri/agent/listener/rate_limited.rb +0 -172
- data/lib/pikuri/agent/synthesizer.rb +0 -171
- data/lib/pikuri/extractor/html.rb +0 -303
- data/lib/pikuri/tool/fetch.rb +0 -172
- data/lib/pikuri/tool/scraper.rb +0 -186
- data/lib/pikuri/tool/search/brave.rb +0 -198
- data/lib/pikuri/tool/search/duckduckgo.rb +0 -207
- data/lib/pikuri/tool/search/engines.rb +0 -186
- data/lib/pikuri/tool/search/exa.rb +0 -229
- data/lib/pikuri/tool/search/rate_limiter.rb +0 -92
- data/lib/pikuri/tool/search/result.rb +0 -29
- data/lib/pikuri/tool/web_scrape.rb +0 -121
- data/lib/pikuri/tool/web_search.rb +0 -57
- data/lib/pikuri/url_cache.rb +0 -112
|
@@ -1,92 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
# Loaded after {Tool} itself is defined; the +class Tool+ reopening below
|
|
5
|
-
# assumes that order.
|
|
6
|
-
class Tool
|
|
7
|
-
module Search
|
|
8
|
-
# Thread-safe pacing + circuit-breaker wrapper for a search provider.
|
|
9
|
-
#
|
|
10
|
-
# +#call { ... }+ enforces a minimum interval between consecutive
|
|
11
|
-
# invocations of the block (sleeping if the previous one was too
|
|
12
|
-
# recent), and watches for {Engines::Unavailable} raised by the
|
|
13
|
-
# block: when that happens, a cooldown deadline is recorded and
|
|
14
|
-
# further calls within the window raise {Engines::Unavailable}
|
|
15
|
-
# immediately without invoking the block. This stops a provider
|
|
16
|
-
# that has been rate-limited or bot-blocked from being hammered
|
|
17
|
-
# with retries.
|
|
18
|
-
#
|
|
19
|
-
# The mutex is held across the block, so concurrent callers
|
|
20
|
-
# serialize — matching the behavior {DuckDuckGo} has always
|
|
21
|
-
# required to keep its IP-spacing throttle correct under
|
|
22
|
-
# concurrent agents.
|
|
23
|
-
#
|
|
24
|
-
# Uses wall-clock {Time.now} rather than the monotonic clock; the
|
|
25
|
-
# intervals here are 1s–5min, well above any realistic NTP step,
|
|
26
|
-
# and {Time.now} keeps tests trivially fakeable with Timecop.
|
|
27
|
-
class RateLimiter
|
|
28
|
-
# @param min_interval [Float] minimum seconds between consecutive
|
|
29
|
-
# block invocations. {#call} sleeps if a previous call was more
|
|
30
|
-
# recent.
|
|
31
|
-
# @param cooldown [Float] seconds to refuse calls after the block
|
|
32
|
-
# raises {Engines::Unavailable}. Calls within this window raise
|
|
33
|
-
# {Engines::Unavailable} immediately without invoking the block.
|
|
34
|
-
def initialize(min_interval:, cooldown:)
|
|
35
|
-
@min_interval = min_interval
|
|
36
|
-
@cooldown = cooldown
|
|
37
|
-
@mutex = Mutex.new
|
|
38
|
-
@last_call_at = nil
|
|
39
|
-
@cooldown_until = nil
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
# Run the given block subject to throttle and cooldown rules.
|
|
43
|
-
#
|
|
44
|
-
# The block is invoked with the mutex held, so concurrent calls
|
|
45
|
-
# serialize: only one block runs at a time per limiter instance.
|
|
46
|
-
# If the block raises {Engines::Unavailable}, the cooldown is
|
|
47
|
-
# armed and the exception is re-raised. Any other exception
|
|
48
|
-
# bubbles up without arming cooldown — only "try again later"
|
|
49
|
-
# signals from the provider are treated as backoff triggers.
|
|
50
|
-
#
|
|
51
|
-
# @yieldreturn [Object] block's return value is passed through
|
|
52
|
-
# @return [Object] whatever the block returned
|
|
53
|
-
# @raise [Engines::Unavailable] either re-raised from the block,
|
|
54
|
-
# or raised directly when the limiter is currently in cooldown
|
|
55
|
-
def call
|
|
56
|
-
@mutex.synchronize do
|
|
57
|
-
now = Time.now
|
|
58
|
-
if @cooldown_until && now < @cooldown_until
|
|
59
|
-
remaining = (@cooldown_until - now).ceil
|
|
60
|
-
raise Engines::Unavailable, "rate-limiter cooldown active for another #{remaining}s"
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
if @last_call_at
|
|
64
|
-
elapsed = now - @last_call_at
|
|
65
|
-
sleep_for(@min_interval - elapsed) if elapsed < @min_interval
|
|
66
|
-
end
|
|
67
|
-
@last_call_at = Time.now
|
|
68
|
-
|
|
69
|
-
begin
|
|
70
|
-
yield
|
|
71
|
-
rescue Engines::Unavailable
|
|
72
|
-
@cooldown_until = Time.now + @cooldown
|
|
73
|
-
raise
|
|
74
|
-
end
|
|
75
|
-
end
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
# Sleep for +seconds+. Isolated as a private method so tests can
|
|
79
|
-
# override it on a single instance (typically to advance a frozen
|
|
80
|
-
# Timecop clock by the same amount) without monkey-patching the
|
|
81
|
-
# global +sleep+.
|
|
82
|
-
#
|
|
83
|
-
# @param seconds [Float] non-negative duration to sleep
|
|
84
|
-
# @return [void]
|
|
85
|
-
def sleep_for(seconds)
|
|
86
|
-
sleep(seconds)
|
|
87
|
-
end
|
|
88
|
-
private :sleep_for
|
|
89
|
-
end
|
|
90
|
-
end
|
|
91
|
-
end
|
|
92
|
-
end
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
class Tool
|
|
5
|
-
module Search
|
|
6
|
-
# A single search hit produced by a {Tool::Search} provider
|
|
7
|
-
# ({DuckDuckGo}, {Brave}, {Exa}). Providers return +Array<Result>+
|
|
8
|
-
# from +.parse+ / +.search+; {Engines.search} concatenates the
|
|
9
|
-
# rows into the smolagents-style Markdown the LLM sees.
|
|
10
|
-
#
|
|
11
|
-
# Splitting structure from rendering keeps the three providers
|
|
12
|
-
# interchangeable — they only have to agree on these three fields
|
|
13
|
-
# (provider-specific extras like relevance scores or published
|
|
14
|
-
# dates are discarded today, which is fine because no caller uses
|
|
15
|
-
# them).
|
|
16
|
-
#
|
|
17
|
-
# @!attribute [r] url
|
|
18
|
-
# @return [String] absolute URL of the hit
|
|
19
|
-
# @!attribute [r] title
|
|
20
|
-
# @return [String] plain-text title, with provider-specific
|
|
21
|
-
# highlight markup ({Brave}'s +<strong>+, {DuckDuckGo}'s
|
|
22
|
-
# +<b>+) already stripped
|
|
23
|
-
# @!attribute [r] body
|
|
24
|
-
# @return [String] plain-text snippet, possibly empty (e.g. an
|
|
25
|
-
# {Exa} navigational result with no highlights)
|
|
26
|
-
Result = Data.define(:url, :title, :body)
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
|
-
end
|
|
@@ -1,121 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
class Tool
|
|
5
|
-
# Truncation policy and Tool spec for the +web_scrape+ tool. The actual
|
|
6
|
-
# scraping lives in {Tool::Scraper}; this module is a thin
|
|
7
|
-
# wrapper that picks the scraper, applies a character cap so the LLM
|
|
8
|
-
# doesn't drown in long-form content, and exposes the result to the
|
|
9
|
-
# agent loop in OpenAI tool-call shape.
|
|
10
|
-
module WebScrape
|
|
11
|
-
# @return [Integer] default character cap on the Markdown returned
|
|
12
|
-
# by {.visit}. Sized to cover most post-readability article bodies
|
|
13
|
-
# in full on the first call, so the LLM doesn't have to re-request
|
|
14
|
-
# with a larger cap and pollute its context with the same prefix
|
|
15
|
-
# twice. ~5K tokens at the typical char/token ratio — light even
|
|
16
|
-
# for small local models. The genuinely long pages (long Wikipedia
|
|
17
|
-
# entries, multi-section docs) still get cut, and the truncation
|
|
18
|
-
# marker invites a deliberate larger {.visit} call when needed.
|
|
19
|
-
DEFAULT_MAX_CHARS = 20_000
|
|
20
|
-
|
|
21
|
-
# @return [Integer] hard ceiling on the +max_chars+ argument to
|
|
22
|
-
# {.visit}. Requests above this are clamped silently so the LLM
|
|
23
|
-
# cannot dump an arbitrarily large page into the conversation.
|
|
24
|
-
MAX_MAX_CHARS = 100_000
|
|
25
|
-
|
|
26
|
-
# On-disk cache used by {.visit} to memoize fetched pages. Defined
|
|
27
|
-
# as a method so specs can swap it for an isolated cache or
|
|
28
|
-
# {UrlCache::NULL} without touching the shared instance.
|
|
29
|
-
#
|
|
30
|
-
# @return [UrlCache, #fetch]
|
|
31
|
-
CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/web_scrape")
|
|
32
|
-
# Accessor for {CACHE}; specs override this to swap in
|
|
33
|
-
# {UrlCache::NULL} or an isolated cache.
|
|
34
|
-
#
|
|
35
|
-
# @return [UrlCache, #fetch]
|
|
36
|
-
def self.cache
|
|
37
|
-
CACHE
|
|
38
|
-
end
|
|
39
|
-
|
|
40
|
-
# Fetch +url+ via {Tool::Scraper} and truncate the rendered
|
|
41
|
-
# Markdown to +max_chars+ characters.
|
|
42
|
-
#
|
|
43
|
-
# The full extracted Markdown is cached on disk via {.cache}, keyed
|
|
44
|
-
# by URL, so repeat visits within the cache TTL skip the network
|
|
45
|
-
# and the extraction pass entirely. +max_chars+ is not part of the
|
|
46
|
-
# cache key — different values for the same URL share one entry,
|
|
47
|
-
# and truncation runs after the cache lookup.
|
|
48
|
-
#
|
|
49
|
-
# {Tool::Scraper::FetchError} (HTTP non-2xx, network failure,
|
|
50
|
-
# redirect-loop, missing +Location+ header) is caught and returned
|
|
51
|
-
# as +"Error: ..."+ in the calculator-style convention so the agent
|
|
52
|
-
# loop feeds the failure back to the model as the next observation
|
|
53
|
-
# instead of crashing — the LLM can then try a different URL or
|
|
54
|
-
# search again. The rescue lives outside {.cache}'s +fetch+ block,
|
|
55
|
-
# so failure strings are never persisted: a retry on the next call
|
|
56
|
-
# hits the network again. Other exceptions (parser bugs in our own
|
|
57
|
-
# code) bubble up unchanged.
|
|
58
|
-
#
|
|
59
|
-
# @param url [String] absolute HTTP(S) URL of the page to download
|
|
60
|
-
# @param max_chars [Integer] character cap on the returned Markdown.
|
|
61
|
-
# Clamped to +[1, {MAX_MAX_CHARS}]+; defaults to
|
|
62
|
-
# {DEFAULT_MAX_CHARS}. When the full page exceeds the cap, output
|
|
63
|
-
# is cut and a marker noting the original length is appended.
|
|
64
|
-
# @return [String] Markdown representation of the page, possibly
|
|
65
|
-
# truncated, or +"Error: ..."+ on a recoverable fetch failure
|
|
66
|
-
def self.visit(url, max_chars: DEFAULT_MAX_CHARS)
|
|
67
|
-
max_chars = max_chars.clamp(1, MAX_MAX_CHARS)
|
|
68
|
-
markdown = cache.fetch(url) { Scraper.visit(url) }
|
|
69
|
-
truncate(markdown, max_chars)
|
|
70
|
-
rescue Scraper::FetchError => e
|
|
71
|
-
"Error: #{e.message}"
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
# Cut +markdown+ to at most +max_chars+ characters, appending a
|
|
75
|
-
# marker describing the original length when truncation actually
|
|
76
|
-
# happens. Returns +markdown+ unchanged if it already fits.
|
|
77
|
-
#
|
|
78
|
-
# @param markdown [String] full Markdown text
|
|
79
|
-
# @param max_chars [Integer] character cap; assumed already clamped
|
|
80
|
-
# @return [String]
|
|
81
|
-
def self.truncate(markdown, max_chars)
|
|
82
|
-
return markdown if markdown.length <= max_chars
|
|
83
|
-
|
|
84
|
-
"#{markdown[0, max_chars]}\n\n" \
|
|
85
|
-
"... [truncated at #{max_chars} of #{markdown.length} chars; " \
|
|
86
|
-
'call again with a larger `max_chars` to see more]'
|
|
87
|
-
end
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
# Webpage download + Markdown conversion tool. Thin wrapper over
|
|
91
|
-
# {Tool::WebScrape.visit} that exposes it to the agent loop in OpenAI
|
|
92
|
-
# tool-call shape.
|
|
93
|
-
#
|
|
94
|
-
# @return [Tool]
|
|
95
|
-
WEB_SCRAPE = new(
|
|
96
|
-
name: 'web_scrape',
|
|
97
|
-
description: <<~DESC,
|
|
98
|
-
Scrapes the rendered webpage or text file at the given URL and returns its main content as Markdown.
|
|
99
|
-
|
|
100
|
-
Usage:
|
|
101
|
-
- Use for HTML pages where you want readable content — readability extraction strips nav, sidebars, and boilerplate.
|
|
102
|
-
- For raw textual payloads (JSON, CSV, robots.txt, source files), use fetch instead — it returns bytes verbatim, while web_scrape would corrupt them with a Markdown pass.
|
|
103
|
-
- A Single Page App may return very little or no content. Do NOT retry with a larger max_chars; try a different URL instead.
|
|
104
|
-
DESC
|
|
105
|
-
parameters: Parameters.build { |p|
|
|
106
|
-
p.required_string :url,
|
|
107
|
-
'Absolute URL of the webpage to scrape, including ' \
|
|
108
|
-
'the scheme, e.g. "https://example.com/article".'
|
|
109
|
-
p.optional_integer :max_chars,
|
|
110
|
-
'Maximum number of characters of Markdown to ' \
|
|
111
|
-
'return. Defaults to 20000; hard-capped at ' \
|
|
112
|
-
'100000. When the page is longer than this, ' \
|
|
113
|
-
'output is cut and a marker reports the full ' \
|
|
114
|
-
'length.'
|
|
115
|
-
},
|
|
116
|
-
execute: ->(url:, max_chars: WebScrape::DEFAULT_MAX_CHARS) {
|
|
117
|
-
WebScrape.visit(url, max_chars: max_chars)
|
|
118
|
-
}
|
|
119
|
-
)
|
|
120
|
-
end
|
|
121
|
-
end
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
class Tool
|
|
5
|
-
# Builder for the LLM-facing web-search tool. The search orchestration
|
|
6
|
-
# lives in {Tool::Search::Engines}; this module owns only the {.build}
|
|
7
|
-
# factory that wires a configured {Search::Engines} into a {Tool}.
|
|
8
|
-
#
|
|
9
|
-
# Unlike the stateless bundled tools (+CALCULATOR+ / +WEB_SCRAPE+ /
|
|
10
|
-
# +FETCH+, shared value constants), web_search is *host-configured*:
|
|
11
|
-
# the paid providers (Brave / Exa) join the cascade only when the host
|
|
12
|
-
# passes their API key, so the tool is built per-wiring — like
|
|
13
|
-
# {Code::Bash} — rather than handed out as a single shared constant.
|
|
14
|
-
# pikuri reads no key from the environment; the host supplies them (the
|
|
15
|
-
# bundled +bin/+ examples load a JSON config file by convention). See
|
|
16
|
-
# CLAUDE.md "Environment is not a secret store".
|
|
17
|
-
module WebSearch
|
|
18
|
-
# Description shown to the LLM, opencode-shape (summary + +Usage:+).
|
|
19
|
-
#
|
|
20
|
-
# @return [String]
|
|
21
|
-
DESCRIPTION = <<~DESC
|
|
22
|
-
Searches the web for a query and returns the top results as a Markdown list of titles, URLs, and short snippets.
|
|
23
|
-
|
|
24
|
-
Usage:
|
|
25
|
-
- Use this to find candidate URLs, then call web_scrape on the most promising one(s) for full content. Snippets alone rarely answer a question.
|
|
26
|
-
DESC
|
|
27
|
-
|
|
28
|
-
# Build the +web_search+ tool. It calls {Search::Engines#search},
|
|
29
|
-
# which cascades through DuckDuckGo (always) plus whichever keyed
|
|
30
|
-
# providers were configured, in random order, falling back on
|
|
31
|
-
# temporary-unavailability errors and rendering the winning
|
|
32
|
-
# provider's rows into a stable Markdown shape.
|
|
33
|
-
#
|
|
34
|
-
# @param brave_key [String, nil] Brave Search API key; when present
|
|
35
|
-
# (non-blank) Brave joins the cascade. Get one at
|
|
36
|
-
# https://api-dashboard.search.brave.com.
|
|
37
|
-
# @param exa_key [String, nil] Exa API key; when present (non-blank)
|
|
38
|
-
# Exa joins the cascade. Get one at https://exa.ai.
|
|
39
|
-
# @return [Tool] the +web_search+ tool in OpenAI tool-call shape
|
|
40
|
-
def self.build(brave_key: nil, exa_key: nil)
|
|
41
|
-
engines = Search::Engines.new(brave_key: brave_key, exa_key: exa_key)
|
|
42
|
-
Tool.new(
|
|
43
|
-
name: 'web_search',
|
|
44
|
-
description: DESCRIPTION,
|
|
45
|
-
parameters: Parameters.build { |p|
|
|
46
|
-
p.required_string :query,
|
|
47
|
-
'The search query, e.g. "BigDecimal precision Ruby".'
|
|
48
|
-
p.optional_integer :max_results,
|
|
49
|
-
'Maximum number of result entries to return. ' \
|
|
50
|
-
'Defaults to 10; most providers cap this at 20.'
|
|
51
|
-
},
|
|
52
|
-
execute: ->(query:, max_results: 10) { engines.search(query, max_results: max_results) }
|
|
53
|
-
)
|
|
54
|
-
end
|
|
55
|
-
end
|
|
56
|
-
end
|
|
57
|
-
end
|
data/lib/pikuri/url_cache.rb
DELETED
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'digest'
|
|
4
|
-
require 'fileutils'
|
|
5
|
-
|
|
6
|
-
module Pikuri
|
|
7
|
-
# On-disk cache for string-keyed text payloads. Used by the bundled tools
|
|
8
|
-
# to avoid re-fetching the same page or re-issuing the same web-search
|
|
9
|
-
# query within a TTL window: {Tool::WebScrape.visit} caches the rendered
|
|
10
|
-
# Markdown for a URL, and {Tool::Search::Engines.search} caches the
|
|
11
|
-
# rendered result list for a query (the query string itself acts as the
|
|
12
|
-
# key — keys are SHA-256 hashed, so any opaque string works).
|
|
13
|
-
#
|
|
14
|
-
# Each tool wires its own {UrlCache} instance against a dedicated
|
|
15
|
-
# subdirectory under {ROOT_DIR}, so a +web_search+ query string and a
|
|
16
|
-
# +web_scrape+ URL string can never collide on the same cache file. There
|
|
17
|
-
# is no global default singleton — pass a fresh instance to whichever
|
|
18
|
-
# code needs caching, or use {NULL} to disable caching entirely.
|
|
19
|
-
#
|
|
20
|
-
# One file per entry, named +<sha256>.txt+ under {#initialize}'s +dir+.
|
|
21
|
-
# Freshness is tracked via the file's mtime; there is no sidecar metadata.
|
|
22
|
-
# Stale entries are simply overwritten the next time {#fetch} is called
|
|
23
|
-
# with the same key. To clear the cache, +rm -rf+ the directory.
|
|
24
|
-
#
|
|
25
|
-
# Not thread-safe: if two callers race on the same cold key, both compute
|
|
26
|
-
# and both write the same file. That is the intended tradeoff to keep this
|
|
27
|
-
# under a few dozen lines — the worst-case cost is a duplicate fetch.
|
|
28
|
-
class UrlCache
|
|
29
|
-
# Root directory under which per-tool cache subdirectories live.
|
|
30
|
-
# Follows the XDG Base Directory spec: +$XDG_CACHE_HOME/pikuri/url_cache+
|
|
31
|
-
# if the env var is set to a non-empty value, else
|
|
32
|
-
# +~/.cache/pikuri/url_cache+. Each tool picks its own subdir
|
|
33
|
-
# (e.g. +"#{ROOT_DIR}/web_scrape"+) so keys from different tools cannot
|
|
34
|
-
# collide. The directory is created lazily on first cache write; pikuri
|
|
35
|
-
# does not pre-create it.
|
|
36
|
-
# @return [String]
|
|
37
|
-
ROOT_DIR = begin
|
|
38
|
-
xdg = ENV['XDG_CACHE_HOME']
|
|
39
|
-
cache_home = xdg && !xdg.empty? ? xdg : File.join(Dir.home, '.cache')
|
|
40
|
-
File.join(cache_home, 'pikuri', 'url_cache')
|
|
41
|
-
end.freeze
|
|
42
|
-
|
|
43
|
-
# Default freshness window: 2 hours, in seconds.
|
|
44
|
-
#
|
|
45
|
-
# Long enough to cover a single interactive session — revisiting
|
|
46
|
-
# a scraped page or re-running a similar search within the same
|
|
47
|
-
# working window hits the cache. Short enough that resuming the
|
|
48
|
-
# next day doesn't serve stale news, docs, or search results.
|
|
49
|
-
# Reference points: opencode keeps no cache, the +pi-web-fetch+
|
|
50
|
-
# community extension uses 15 minutes, +pi-web-search+ uses 5;
|
|
51
|
-
# 2 hours sits comfortably above the "single follow-up" window
|
|
52
|
-
# those numbers are aimed at without holding content across days.
|
|
53
|
-
# @return [Integer]
|
|
54
|
-
DEFAULT_TTL = 2 * 60 * 60
|
|
55
|
-
|
|
56
|
-
# @param ttl [Integer] freshness window in seconds; entries with an
|
|
57
|
-
# mtime older than this are treated as misses
|
|
58
|
-
# @param dir [String] directory under which cache files live; created
|
|
59
|
-
# lazily on first write
|
|
60
|
-
def initialize(ttl:, dir:)
|
|
61
|
-
@ttl = ttl
|
|
62
|
-
@dir = dir
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
# Return the cached payload for +url+ if a fresh entry exists, otherwise
|
|
66
|
-
# yield to compute it, persist the result, and return it.
|
|
67
|
-
#
|
|
68
|
-
# The block is only invoked on a miss. If the block raises, no file is
|
|
69
|
-
# written — errors are not cached.
|
|
70
|
-
#
|
|
71
|
-
# @param url [String] cache key; a URL or any opaque string identifier
|
|
72
|
-
# @yieldreturn [String] payload to store and return on a miss
|
|
73
|
-
# @return [String] cached or freshly-computed payload
|
|
74
|
-
def fetch(url)
|
|
75
|
-
path = path_for(url)
|
|
76
|
-
return File.read(path) if fresh?(path)
|
|
77
|
-
|
|
78
|
-
content = yield
|
|
79
|
-
FileUtils.mkdir_p(@dir)
|
|
80
|
-
File.write(path, content)
|
|
81
|
-
content
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
# @param path [String]
|
|
85
|
-
# @return [Boolean] true when +path+ exists and was written within the
|
|
86
|
-
# TTL window
|
|
87
|
-
def fresh?(path)
|
|
88
|
-
File.exist?(path) && Time.now - File.mtime(path) < @ttl
|
|
89
|
-
end
|
|
90
|
-
|
|
91
|
-
# @param url [String]
|
|
92
|
-
# @return [String] absolute path of the cache file for +url+
|
|
93
|
-
def path_for(url)
|
|
94
|
-
File.join(@dir, "#{Digest::SHA256.hexdigest(url)}.txt")
|
|
95
|
-
end
|
|
96
|
-
|
|
97
|
-
# Null cache: a drop-in replacement that always misses and never
|
|
98
|
-
# persists. Use this in tests (or anywhere else you want caching off)
|
|
99
|
-
# without giving up the {UrlCache#fetch} contract.
|
|
100
|
-
NULL = Object.new
|
|
101
|
-
# Singleton +fetch+ on {NULL} that mirrors {UrlCache#fetch}'s shape:
|
|
102
|
-
# ignores +_key+, always yields, never persists.
|
|
103
|
-
#
|
|
104
|
-
# @param _key [String] cache key; ignored
|
|
105
|
-
# @yieldreturn [String] freshly-computed payload
|
|
106
|
-
# @return [String] whatever the block returned
|
|
107
|
-
def NULL.fetch(_key)
|
|
108
|
-
yield
|
|
109
|
-
end
|
|
110
|
-
NULL.freeze
|
|
111
|
-
end
|
|
112
|
-
end
|