pikuri-core 0.0.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +17 -30
- data/lib/pikuri/agent/chat_transport.rb +168 -107
- data/lib/pikuri/agent/configurator.rb +194 -173
- data/lib/pikuri/agent/control/cancellable.rb +100 -70
- data/lib/pikuri/agent/control/handoff.rb +75 -0
- data/lib/pikuri/agent/control/interloper.rb +107 -118
- data/lib/pikuri/agent/control.rb +13 -35
- data/lib/pikuri/agent/event.rb +144 -197
- data/lib/pikuri/agent/extension.rb +83 -95
- data/lib/pikuri/agent/history.rb +247 -0
- data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
- data/lib/pikuri/agent/listener/terminal.rb +169 -136
- data/lib/pikuri/agent/listener/token_log.rb +84 -157
- data/lib/pikuri/agent/listener.rb +24 -44
- data/lib/pikuri/agent/listener_list.rb +32 -47
- data/lib/pikuri/agent/request_instrumenter.rb +102 -0
- data/lib/pikuri/agent.rb +1095 -577
- data/lib/pikuri/attachment.rb +65 -0
- data/lib/pikuri/bundler_env.rb +68 -0
- data/lib/pikuri/extractor/passthrough.rb +24 -31
- data/lib/pikuri/extractor.rb +125 -155
- data/lib/pikuri/file_type.rb +64 -136
- data/lib/pikuri/finalizers.rb +32 -47
- data/lib/pikuri/paths.rb +104 -13
- data/lib/pikuri/sanitizer.rb +46 -67
- data/lib/pikuri/subprocess.rb +86 -118
- data/lib/pikuri/testing.rb +369 -0
- data/lib/pikuri/tool/calculator.rb +168 -74
- data/lib/pikuri/tool/execute_context.rb +42 -0
- data/lib/pikuri/tool/parameters.rb +31 -29
- data/lib/pikuri/tool/result.rb +164 -0
- data/lib/pikuri/tool/trifecta_legs.rb +248 -0
- data/lib/pikuri/tool.rb +416 -67
- data/lib/pikuri/trifecta/contribution.rb +43 -0
- data/lib/pikuri/trifecta/node.rb +47 -0
- data/lib/pikuri/trifecta/report.rb +228 -0
- data/lib/pikuri/trifecta.rb +126 -0
- data/lib/pikuri/version.rb +1 -1
- data/lib/pikuri-core.rb +70 -92
- data/prompts/agent-loop.txt +5 -0
- data/prompts/pikuri-chat.txt +3 -12
- metadata +26 -72
- data/lib/pikuri/agent/context_window_detector.rb +0 -222
- data/lib/pikuri/agent/control/step_limit.rb +0 -111
- data/lib/pikuri/agent/extension_context.rb +0 -147
- data/lib/pikuri/agent/listener/rate_limited.rb +0 -172
- data/lib/pikuri/agent/synthesizer.rb +0 -171
- data/lib/pikuri/extractor/html.rb +0 -303
- data/lib/pikuri/tool/fetch.rb +0 -172
- data/lib/pikuri/tool/scraper.rb +0 -186
- data/lib/pikuri/tool/search/brave.rb +0 -198
- data/lib/pikuri/tool/search/duckduckgo.rb +0 -207
- data/lib/pikuri/tool/search/engines.rb +0 -186
- data/lib/pikuri/tool/search/exa.rb +0 -229
- data/lib/pikuri/tool/search/rate_limiter.rb +0 -92
- data/lib/pikuri/tool/search/result.rb +0 -29
- data/lib/pikuri/tool/web_scrape.rb +0 -121
- data/lib/pikuri/tool/web_search.rb +0 -57
- data/lib/pikuri/url_cache.rb +0 -112
data/lib/pikuri/file_type.rb
CHANGED
|
@@ -2,66 +2,37 @@
|
|
|
2
2
|
|
|
3
3
|
module Pikuri
|
|
4
4
|
# Magic-byte content sniffing, plus the path-aware front over the
|
|
5
|
-
# {Extractor} registry. Two responsibilities:
|
|
5
|
+
# {Extractor} registry. Two core responsibilities:
|
|
6
6
|
#
|
|
7
|
-
# * {.detect_mime} — recognise a file from its leading bytes. Returns
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
# {.detect_mime}: a file can be both recognised (e.g. PDF) *and*
|
|
14
|
-
# binary. {.detect_mime} tells you what the bytes are;
|
|
15
|
-
# {.binary?} tells you whether they're safe to render as text.
|
|
7
|
+
# * {.detect_mime} — recognise a file from its leading bytes. Returns a
|
|
8
|
+
# MIME String for formats pikuri handles specially (+application/pdf+,
|
|
9
|
+
# the four image formats), or +nil+ for "unrecognised".
|
|
10
|
+
# * {.binary?} — text-vs-binary heuristic, independent of {.detect_mime}
|
|
11
|
+
# (a file can be both recognised, e.g. PDF, *and* binary): what the
|
|
12
|
+
# bytes are vs. whether they're safe to render as text.
|
|
16
13
|
#
|
|
17
|
-
# On top
|
|
18
|
-
# {.
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
#
|
|
22
|
-
# hand the opened IO to {Extractor.extract} /
|
|
23
|
-
# {Extractor.extract_paged} — which format the bytes are and how
|
|
24
|
-
# they become text is entirely the registry's business, so a gem
|
|
25
|
-
# plugging a new extractor in extends these wrappers for free.
|
|
14
|
+
# On top sit two +Pathname+ conveniences, {.read_as_text} (whole document)
|
|
15
|
+
# and {.read_as_text_paged} (line-windowed). Both own the *path-level*
|
|
16
|
+
# refusals (missing file, directory, image) and the exception mapping,
|
|
17
|
+
# then hand the IO to {Extractor}; format-to-text is the registry's
|
|
18
|
+
# business, so a new extractor extends these wrappers for free.
|
|
26
19
|
#
|
|
27
|
-
# {.detect_mime}
|
|
28
|
-
#
|
|
29
|
-
# the module opens the file in binary mode and reads {SAMPLE_BYTES}
|
|
30
|
-
# for the sniff itself. The Pathname form is the convenience path;
|
|
31
|
-
# the bytes form is for callers that already have the sample or are
|
|
32
|
-
# calling both methods on the same file and want to avoid a second
|
|
33
|
-
# open.
|
|
34
|
-
#
|
|
35
|
-
# == Why a separate module
|
|
36
|
-
#
|
|
37
|
-
# Without this module, magic-byte tables and the binary heuristic
|
|
38
|
-
# ended up scattered through whichever tool needed them — first PDF
|
|
39
|
-
# in {Workspace::Read}, then images alongside it, then a copy of
|
|
40
|
-
# {.binary?} reached for by {Workspace::Edit}. Collecting the
|
|
41
|
-
# detection logic here lets {Read} focus on routing
|
|
42
|
-
# (mime-to-formatter), {Edit} drop its cross-tool reach, and new
|
|
43
|
-
# tools share one set of magic-byte truths.
|
|
20
|
+
# {.detect_mime}/{.binary?} take either a +String+ of bytes (caller
|
|
21
|
+
# sampled) or a +Pathname+ (opened binary, {SAMPLE_BYTES} read).
|
|
44
22
|
#
|
|
45
23
|
# == Deliberate non-goals
|
|
46
24
|
#
|
|
47
|
-
# * *Not a full MIME database.* The set grows when a
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
# extension-based behaviour can layer it themselves.
|
|
54
|
-
# * *No convenience predicates* like +image?+ / +pdf?+. Callers do
|
|
55
|
-
# +mime == 'application/pdf'+ or +mime&.start_with?('image/')+ —
|
|
56
|
-
# one extra character, zero added API surface.
|
|
25
|
+
# * *Not a full MIME database.* The set grows when a tool needs a format,
|
|
26
|
+
# not speculatively.
|
|
27
|
+
# * *No path/extension fallback.* Extensions lie (a renamed +.png+);
|
|
28
|
+
# magic-byte detection on content is the source of truth.
|
|
29
|
+
# * *No +image?+/+pdf?+ predicates.* Callers do +mime == 'application/pdf'+
|
|
30
|
+
# — one character, zero added surface.
|
|
57
31
|
module FileType
|
|
58
32
|
module_function
|
|
59
33
|
|
|
60
|
-
# @return [Integer]
|
|
61
|
-
#
|
|
62
|
-
# prefix pikuri sniffs today (the largest is WebP's 12-byte
|
|
63
|
-
# container header) with comfortable slack; small enough that
|
|
64
|
-
# reading it off any reasonable filesystem is effectively free.
|
|
34
|
+
# @return [Integer] bytes to sample for {.detect_mime}/{.binary?}. Covers
|
|
35
|
+
# every prefix sniffed today (largest: WebP's 12-byte header) with slack.
|
|
65
36
|
SAMPLE_BYTES = 4096
|
|
66
37
|
|
|
67
38
|
# @return [Float] fraction of the sample that may be non-printable
|
|
@@ -85,15 +56,13 @@ module Pikuri
|
|
|
85
56
|
# with this five-byte ASCII sequence per ISO 32000-1 §7.5.2.
|
|
86
57
|
PDF_MAGIC = '%PDF-'
|
|
87
58
|
|
|
88
|
-
# Recognise a file from its leading bytes. Returns the MIME
|
|
89
|
-
#
|
|
90
|
-
# "unrecognised" — callers interpret +nil+ themselves (text,
|
|
59
|
+
# Recognise a file from its leading bytes. Returns the MIME String for
|
|
60
|
+
# formats pikuri handles specially, or +nil+ for "unrecognised" (text,
|
|
91
61
|
# opaque binary, ...).
|
|
92
62
|
#
|
|
93
|
-
# @param input [String, Pathname]
|
|
94
|
-
#
|
|
95
|
-
#
|
|
96
|
-
# path exists; missing-file errors propagate as +Errno::ENOENT+.
|
|
63
|
+
# @param input [String, Pathname] bytes to inspect, or a +Pathname+
|
|
64
|
+
# opened binary and read up to {SAMPLE_BYTES}. Caller verifies the path
|
|
65
|
+
# exists; missing-file errors propagate as +Errno::ENOENT+.
|
|
97
66
|
# @return [String, nil]
|
|
98
67
|
def detect_mime(input)
|
|
99
68
|
bytes = sample_of(input)
|
|
@@ -109,19 +78,13 @@ module Pikuri
|
|
|
109
78
|
nil
|
|
110
79
|
end
|
|
111
80
|
|
|
112
|
-
#
|
|
113
|
-
#
|
|
114
|
-
#
|
|
115
|
-
#
|
|
116
|
-
# are >127 so they sit outside the non-printable ranges and pass
|
|
117
|
-
# through unflagged, letting UTF-8 text read fine. An empty
|
|
118
|
-
# sample is treated as not-binary (callers reading an empty file
|
|
119
|
-
# take the empty-text path).
|
|
81
|
+
# Text-vs-binary heuristic (opencode's): any +NUL+ forces +true+; else
|
|
82
|
+
# ratio bytes outside printable (+\t\n\v\f\r+ + ASCII 32..126) against the
|
|
83
|
+
# sample. UTF-8 continuation bytes (0x80-0xBF) sit above 127 so they pass
|
|
84
|
+
# unflagged; an empty sample is not-binary.
|
|
120
85
|
#
|
|
121
|
-
# @param input [String, Pathname]
|
|
122
|
-
#
|
|
123
|
-
# to {SAMPLE_BYTES} from. Caller is responsible for verifying
|
|
124
|
-
# the path exists.
|
|
86
|
+
# @param input [String, Pathname] bytes, or a +Pathname+ opened binary
|
|
87
|
+
# and read up to {SAMPLE_BYTES}. Caller verifies the path exists.
|
|
125
88
|
# @return [Boolean]
|
|
126
89
|
def binary?(input)
|
|
127
90
|
bytes = sample_of(input)
|
|
@@ -136,37 +99,19 @@ module Pikuri
|
|
|
136
99
|
non_printable.to_f / bytes.bytesize > BINARY_NONPRINTABLE_THRESHOLD
|
|
137
100
|
end
|
|
138
101
|
|
|
139
|
-
# Read +path+
|
|
140
|
-
# through
|
|
141
|
-
#
|
|
142
|
-
#
|
|
143
|
-
#
|
|
144
|
-
# PDF with no extractable text comes back as the empty String, a
|
|
145
|
-
# deliberate silent skip callers detect by length if they care).
|
|
146
|
-
#
|
|
147
|
-
# Refusal cases — all raise rather than returning a sentinel
|
|
148
|
-
# because the callers are internal pikuri code, not an LLM
|
|
149
|
-
# tool. The LLM-facing +Workspace::Read+ does its own routing and
|
|
150
|
-
# returns "Error: ..." observations; that's a separate concern.
|
|
151
|
-
#
|
|
152
|
-
# * Path doesn't exist → +Errno::ENOENT+.
|
|
153
|
-
# * Path is a directory → +ArgumentError+.
|
|
154
|
-
# * Image (PNG / JPEG / GIF / WebP per {.detect_mime}) →
|
|
155
|
-
# +ArgumentError+; images aren't text.
|
|
156
|
-
# * Content no extractor claims (opaque binary) →
|
|
157
|
-
# +ArgumentError+, mapped from {Extractor::Unsupported}.
|
|
158
|
-
# * Extraction failure (malformed PDF, ...) → +RuntimeError+ with
|
|
159
|
-
# the path included, mapped from {Extractor::Error} so callers
|
|
160
|
-
# don't need to know any extractor's exception hierarchy.
|
|
102
|
+
# Read +path+ as plain UTF-8 text through the {Extractor} registry:
|
|
103
|
+
# unrecognised-but-textual passes through verbatim; with pikuri-pdf
|
|
104
|
+
# registered, PDFs get +"--- Page N ---"+ markers (a scanned-image PDF
|
|
105
|
+
# comes back empty — a deliberate silent skip callers detect by length).
|
|
106
|
+
# Refusals raise (callers are internal pikuri code, not an LLM tool).
|
|
161
107
|
#
|
|
162
108
|
# @param path [Pathname] file to read.
|
|
163
|
-
# @return [String] UTF-8 text
|
|
164
|
-
#
|
|
165
|
-
#
|
|
166
|
-
# a directory, is an image, or is binary.
|
|
109
|
+
# @return [String] UTF-8 text; may be empty (empty file, scanned PDF).
|
|
110
|
+
# @raise [ArgumentError] if +path+ isn't a +Pathname+, is a directory, an
|
|
111
|
+
# image, or opaque binary (the last mapped from {Extractor::Unsupported}).
|
|
167
112
|
# @raise [Errno::ENOENT] if +path+ doesn't exist.
|
|
168
|
-
# @raise [RuntimeError] on an extraction failure (
|
|
169
|
-
#
|
|
113
|
+
# @raise [RuntimeError] on an extraction failure (mapped from
|
|
114
|
+
# {Extractor::Error}, path included).
|
|
170
115
|
def read_as_text(path)
|
|
171
116
|
mime = guard_extractable(path)
|
|
172
117
|
path.open('rb') { |io| Extractor.extract(io, content_type: mime) }
|
|
@@ -176,29 +121,21 @@ module Pikuri
|
|
|
176
121
|
raise "Cannot extract text from #{path}: #{e.message}"
|
|
177
122
|
end
|
|
178
123
|
|
|
179
|
-
# Extract +path+
|
|
180
|
-
#
|
|
181
|
-
#
|
|
182
|
-
#
|
|
183
|
-
# {.read_as_text}; the windowing semantics (including the lazy
|
|
184
|
-
# +extract_lines+ consumption that stops parsing once the window
|
|
185
|
-
# fills) are {Extractor.extract_paged}'s.
|
|
186
|
-
# The LLM-facing callers map the exceptions into +"Error: ..."+
|
|
187
|
-
# observations themselves.
|
|
124
|
+
# Extract +path+ into a windowed {Extractor::Page} (lines from +offset+ up
|
|
125
|
+
# to +limit+, byte-capped, long lines truncated). Same routing and refusal
|
|
126
|
+
# contract as {.read_as_text}; windowing semantics are
|
|
127
|
+
# {Extractor.extract_paged}'s.
|
|
188
128
|
#
|
|
189
129
|
# @param path [Pathname] file to read.
|
|
190
|
-
# @param offset [Integer] 1-indexed first line
|
|
191
|
-
#
|
|
192
|
-
# @param limit [Integer] maximum lines to collect. Caller
|
|
193
|
-
# validates +limit >= 1+.
|
|
130
|
+
# @param offset [Integer] 1-indexed first line; caller validates +>= 1+.
|
|
131
|
+
# @param limit [Integer] max lines to collect; caller validates +>= 1+.
|
|
194
132
|
# @param max_bytes [Integer] hard byte cap on collected content.
|
|
195
133
|
# @param max_line_length [Integer] per-line truncation threshold.
|
|
196
|
-
# @return [Extractor::Page]
|
|
197
|
-
# @raise [ArgumentError] if +path+ isn't a +Pathname+, is a
|
|
198
|
-
#
|
|
134
|
+
# @return [Extractor::Page]
|
|
135
|
+
# @raise [ArgumentError] if +path+ isn't a +Pathname+, is a directory, an
|
|
136
|
+
# image, or binary.
|
|
199
137
|
# @raise [Errno::ENOENT] if +path+ doesn't exist.
|
|
200
|
-
# @raise [RuntimeError] on an extraction failure
|
|
201
|
-
# unsupported PDF, ...).
|
|
138
|
+
# @raise [RuntimeError] on an extraction failure.
|
|
202
139
|
def read_as_text_paged(path, offset: 1, limit: Extractor::PAGE_DEFAULT_LIMIT,
|
|
203
140
|
max_bytes: Extractor::PAGE_MAX_BYTES,
|
|
204
141
|
max_line_length: Extractor::PAGE_MAX_LINE_LENGTH)
|
|
@@ -213,16 +150,13 @@ module Pikuri
|
|
|
213
150
|
raise "Cannot extract text from #{path}: #{e.message}"
|
|
214
151
|
end
|
|
215
152
|
|
|
216
|
-
#
|
|
217
|
-
#
|
|
218
|
-
#
|
|
219
|
-
# model, never text). Returns the {.detect_mime} result so the
|
|
220
|
-
# caller can pass it to the {Extractor} as the content-type hint.
|
|
153
|
+
# Shared path-level refusals for the read-as-text pair: an existing
|
|
154
|
+
# non-directory +Pathname+, not an image (images are vision-model data).
|
|
155
|
+
# Returns the {.detect_mime} result as the content-type hint.
|
|
221
156
|
#
|
|
222
157
|
# @param path [Pathname]
|
|
223
158
|
# @return [String, nil] the sniffed MIME type.
|
|
224
|
-
# @raise [ArgumentError] on a non-Pathname, a directory, or an
|
|
225
|
-
# image.
|
|
159
|
+
# @raise [ArgumentError] on a non-Pathname, a directory, or an image.
|
|
226
160
|
# @raise [Errno::ENOENT] if +path+ doesn't exist.
|
|
227
161
|
def guard_extractable(path)
|
|
228
162
|
raise ArgumentError, "expected Pathname, got #{path.class}" unless path.is_a?(Pathname)
|
|
@@ -236,24 +170,18 @@ module Pikuri
|
|
|
236
170
|
end
|
|
237
171
|
private_class_method :guard_extractable
|
|
238
172
|
|
|
239
|
-
# Coerce
|
|
240
|
-
# +String+
|
|
241
|
-
# +Pathname+ inputs are opened in binary mode and up to
|
|
242
|
-
# {SAMPLE_BYTES} are read off the front. Empty files come back
|
|
243
|
-
# as an empty String — {.binary?} treats that as not-binary and
|
|
244
|
-
# {.detect_mime} returns +nil+ for it, which is what the
|
|
245
|
-
# empty-text path wants.
|
|
173
|
+
# Coerce +input+ into up to {SAMPLE_BYTES} binary bytes for the sniffs: a
|
|
174
|
+
# +String+ is sliced, a +Pathname+ opened binary and read. Empty files → empty String (not-binary, +nil+ mime).
|
|
246
175
|
#
|
|
247
176
|
# @param input [String, Pathname]
|
|
248
|
-
# @return [String] raw bytes
|
|
249
|
-
#
|
|
250
|
-
#
|
|
251
|
-
# +Pathname+ — refuses to guess, since a bare String could be
|
|
252
|
-
# either a path or actual bytes.
|
|
177
|
+
# @return [String] raw bytes.
|
|
178
|
+
# @raise [ArgumentError] if neither — refuses to guess, since a bare
|
|
179
|
+
# String could be a path or actual bytes.
|
|
253
180
|
def sample_of(input)
|
|
254
181
|
case input
|
|
255
182
|
when String
|
|
256
|
-
|
|
183
|
+
# Binary: a UTF-8 String holding non-ASCII raises against the magic prefixes.
|
|
184
|
+
input.byteslice(0, SAMPLE_BYTES).b
|
|
257
185
|
when Pathname
|
|
258
186
|
input.open('rb') { |io| io.read(SAMPLE_BYTES) || +'' }
|
|
259
187
|
else
|
data/lib/pikuri/finalizers.rb
CHANGED
|
@@ -1,54 +1,44 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Pikuri
|
|
4
|
-
# Process-global teardown registry: one +at_exit+ for the whole
|
|
5
|
-
#
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
# the same LIFO + per-handler-rescue + idempotent shape, one level
|
|
10
|
-
# up.
|
|
4
|
+
# Process-global teardown registry: one +at_exit+ for the whole process,
|
|
5
|
+
# with everything owning a resource that needs orderly shutdown (agents,
|
|
6
|
+
# a container a gem supervises, background workers) registering here instead of
|
|
7
|
+
# growing its own +at_exit+. {Agent#on_close} promoted per-agent → per-process
|
|
8
|
+
# — the same LIFO + per-handler-rescue + idempotent shape.
|
|
11
9
|
#
|
|
12
10
|
# == Why one chokepoint
|
|
13
11
|
#
|
|
14
|
-
# Independent +at_exit+ hooks fire in
|
|
15
|
-
#
|
|
16
|
-
#
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
#
|
|
20
|
-
# time) have closed gracefully, while the subprocess machinery they
|
|
21
|
-
# shell out to during close (e.g. {VectorDb::Server::Chroma#close}'s
|
|
22
|
-
# +docker stop+) is still live.
|
|
12
|
+
# Independent +at_exit+ hooks fire in file-load order — invisible and fragile.
|
|
13
|
+
# One registry makes the order explicit: the SIGTERM-the-strays backstop
|
|
14
|
+
# ({Subprocess.cleanup!}) registers at *load* time, so it sits at the bottom
|
|
15
|
+
# of the LIFO stack and runs *last* — after agents and servers (registered at
|
|
16
|
+
# *construction* time) close gracefully, while the subprocess machinery they
|
|
17
|
+
# shell out to during close (a +docker stop+) is still live.
|
|
23
18
|
#
|
|
24
19
|
# == Contract
|
|
25
20
|
#
|
|
26
|
-
# A registrant MUST respond to +#close+, and
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
# +Finalizers.register { Pikuri::Subprocess.cleanup! }+).
|
|
21
|
+
# A registrant MUST respond to +#close+, idempotent and tolerant of running at
|
|
22
|
+
# exit (the host may have closed it earlier). Pass a block instead for
|
|
23
|
+
# teardown with no natural +#close+
|
|
24
|
+
# (+Finalizers.register { Pikuri::Subprocess.cleanup! }+).
|
|
31
25
|
#
|
|
32
26
|
# == Order: LIFO
|
|
33
27
|
#
|
|
34
|
-
# Last registered, first closed
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
# tears down first. Registration order is therefore dependency order;
|
|
38
|
-
# register the dependency before its dependents.
|
|
28
|
+
# Last registered, first closed. A registrant depending on an earlier one (a
|
|
29
|
+
# background indexing daemon writing into a database container) registers later
|
|
30
|
+
# and tears down first — registration order is dependency order.
|
|
39
31
|
#
|
|
40
32
|
# == Errors are contained
|
|
41
33
|
#
|
|
42
|
-
# Each +#close+ runs
|
|
43
|
-
# {
|
|
44
|
-
# teardown can't strand the rest. {.run!} drains the registry, so a
|
|
45
|
-
# second call (an explicit one, then the +at_exit+) closes nothing.
|
|
34
|
+
# Each +#close+ runs in its own +rescue+ (a raise is logged, the sweep
|
|
35
|
+
# continues). {.run!} drains the registry, so a second call closes nothing.
|
|
46
36
|
module Finalizers
|
|
47
37
|
# @return [Logger] subsystem logger for contained teardown failures.
|
|
48
38
|
LOGGER = Pikuri.logger_for('Finalizers')
|
|
49
39
|
|
|
50
40
|
# Adapts a teardown block to the +#close+ protocol, so a block and a
|
|
51
|
-
# closeable
|
|
41
|
+
# closeable share one registry.
|
|
52
42
|
Closer = Struct.new(:block) do
|
|
53
43
|
# @return [void]
|
|
54
44
|
def close
|
|
@@ -60,17 +50,14 @@ module Pikuri
|
|
|
60
50
|
@mutex = Mutex.new
|
|
61
51
|
|
|
62
52
|
class << self
|
|
63
|
-
# Register a closeable (or a block) to
|
|
64
|
-
#
|
|
65
|
-
#
|
|
66
|
-
# should drop out so it can be garbage-collected rather than
|
|
67
|
-
# pinned alive until the process dies.
|
|
53
|
+
# Register a closeable (or a block) to tear down at exit. Returns the
|
|
54
|
+
# handle so the caller can later {.unregister} it — a resource closed
|
|
55
|
+
# explicitly before exit should drop out to be GC'd, not pinned alive.
|
|
68
56
|
#
|
|
69
|
-
# @param closeable [#close, nil] resource to close
|
|
70
|
-
#
|
|
71
|
-
# @
|
|
72
|
-
#
|
|
73
|
-
# the {Closer} wrapping the block; pass it to {.unregister}
|
|
57
|
+
# @param closeable [#close, nil] resource to close; omit when passing a block
|
|
58
|
+
# @yield teardown for resources with no +#close+
|
|
59
|
+
# @return [#close] the registered handle (the object, or the {Closer}
|
|
60
|
+
# wrapping the block); pass to {.unregister}
|
|
74
61
|
# @raise [ArgumentError] if neither an object nor a block is given
|
|
75
62
|
def register(closeable = nil, &block)
|
|
76
63
|
unless closeable || block
|
|
@@ -82,8 +69,7 @@ module Pikuri
|
|
|
82
69
|
handle
|
|
83
70
|
end
|
|
84
71
|
|
|
85
|
-
# Drop a previously-registered handle. Idempotent
|
|
86
|
-
# something already gone (or never registered) is a no-op.
|
|
72
|
+
# Drop a previously-registered handle. Idempotent.
|
|
87
73
|
#
|
|
88
74
|
# @param handle [#close] the value returned by {.register}
|
|
89
75
|
# @return [void]
|
|
@@ -92,10 +78,9 @@ module Pikuri
|
|
|
92
78
|
nil
|
|
93
79
|
end
|
|
94
80
|
|
|
95
|
-
# Close every registrant in LIFO order, each guarded by its own
|
|
96
|
-
#
|
|
97
|
-
#
|
|
98
|
-
# no-op and keeps it safe against a concurrent caller.
|
|
81
|
+
# Close every registrant in LIFO order, each guarded by its own +rescue+.
|
|
82
|
+
# Wired to +at_exit+ below. Draining under the lock makes a repeat call a
|
|
83
|
+
# no-op and stays safe against a concurrent caller.
|
|
99
84
|
#
|
|
100
85
|
# @return [void]
|
|
101
86
|
def run!
|
data/lib/pikuri/paths.rb
CHANGED
|
@@ -1,23 +1,35 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'pathname'
|
|
4
|
+
require 'tmpdir'
|
|
5
|
+
require 'fileutils'
|
|
4
6
|
|
|
5
7
|
module Pikuri
|
|
6
|
-
# Standardized on-disk locations for pikuri's local state
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
# {
|
|
10
|
-
#
|
|
8
|
+
# Standardized on-disk locations for pikuri's local state, centralizing XDG
|
|
9
|
+
# resolution so every disk-caching component roots under one place instead of
|
|
10
|
+
# re-deriving it. The load-bearing distinction is {.cache} (shared across all
|
|
11
|
+
# pikuri processes, persistent) vs. {.new_temp} (private to this process,
|
|
12
|
+
# ephemeral) — reach for the wrong one and two concurrent processes clobber
|
|
13
|
+
# each other's scratch.
|
|
11
14
|
module Paths
|
|
12
|
-
#
|
|
13
|
-
#
|
|
15
|
+
# +temp-*+ umbrellas older than this are reaped by {.sweep_stale_temps!} at
|
|
16
|
+
# core load — generous enough not to disturb a long-lived process in another
|
|
17
|
+
# shell, tight enough that one killed last week doesn't leak forever.
|
|
14
18
|
#
|
|
15
|
-
#
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
#
|
|
19
|
-
#
|
|
20
|
-
#
|
|
19
|
+
# @return [Integer] seconds
|
|
20
|
+
TEMP_STALE_SECONDS = 7 * 24 * 60 * 60
|
|
21
|
+
|
|
22
|
+
# Pikuri's cache root: +$XDG_CACHE_HOME/pikuri+ or +~/.cache/pikuri+.
|
|
23
|
+
#
|
|
24
|
+
# Shared across ALL pikuri processes and PERSISTENT (survives exit) — the
|
|
25
|
+
# home for state meant to be reused: the URL cache, mem0 / Qdrant
|
|
26
|
+
# server data. For scratch private to *this* process that should vanish when
|
|
27
|
+
# it exits, use {.new_temp}; reaching for +cache+ + a fixed subdir is how two
|
|
28
|
+
# concurrent processes end up overwriting each other's files.
|
|
29
|
+
#
|
|
30
|
+
# A method, not a constant: a constant would snapshot +XDG_CACHE_HOME+ at
|
|
31
|
+
# +require+ time, breaking env-stubbing in tests and a runtime change. Not
|
|
32
|
+
# created — callers +mkdir_p+ the subdir they need.
|
|
21
33
|
#
|
|
22
34
|
# @return [Pathname] the +<cache home>/pikuri+ directory
|
|
23
35
|
def self.cache
|
|
@@ -25,5 +37,84 @@ module Pikuri
|
|
|
25
37
|
home = File.expand_path('~/.cache') if home.nil? || home.empty?
|
|
26
38
|
Pathname.new(home).join('pikuri')
|
|
27
39
|
end
|
|
40
|
+
|
|
41
|
+
# A fresh scratch dir private to this process, reaped at exit via
|
|
42
|
+
# {Pikuri::Finalizers} (and stale-swept by {.sweep_stale_temps!} for strays a
|
|
43
|
+
# crash leaves behind). Mint one per concurrent-safe scratch need and hold
|
|
44
|
+
# the result:
|
|
45
|
+
#
|
|
46
|
+
# dir = Paths.new_temp # ~/.cache/pikuri/temp-XXXX/
|
|
47
|
+
# snap = Snapshot.new(source: db, dir: dir) # can't collide with another process
|
|
48
|
+
#
|
|
49
|
+
# Minted IN HOME under {.cache} — never +/tmp+ — for two concrete reasons: a
|
|
50
|
+
# +cp --reflink=auto+ copy shares extents only on the source's filesystem
|
|
51
|
+
# (the copied DB lives under +$HOME+), and the bubblewrap sandbox binds its
|
|
52
|
+
# scratch at +/tmp+ inside, so an umbrella already under +/tmp+ would bind on
|
|
53
|
+
# top of itself.
|
|
54
|
+
#
|
|
55
|
+
# A factory — a *new* dir each call (hence +new_+, not a memoized accessor
|
|
56
|
+
# like {.cache}); a caller memoizes if it wants one stable dir, but per
|
|
57
|
+
# *consumer*, not per process. Memoizing one at process scope and handing it
|
|
58
|
+
# to several agents puts two writers on one path — the collision this exists
|
|
59
|
+
# to prevent, moved inside the process.
|
|
60
|
+
#
|
|
61
|
+
# @return [Pathname] absolute, symlink-resolved path of the freshly-made dir
|
|
62
|
+
def self.new_temp
|
|
63
|
+
FileUtils.mkdir_p(cache)
|
|
64
|
+
path = Pathname.new(Dir.mktmpdir('temp-', cache)).realpath
|
|
65
|
+
Pikuri::Finalizers.register { FileUtils.remove_entry(path.to_s) if path.exist? }
|
|
66
|
+
path
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Reap +temp-*+ umbrellas under {.cache} older than {TEMP_STALE_SECONDS}.
|
|
70
|
+
# Called once at core load (so every binary sweeps, regardless of which gems
|
|
71
|
+
# it wires) — the net for a process killed before its {Pikuri::Finalizers}
|
|
72
|
+
# reap could fire. Failures are swallowed (best-effort; the Finalizers
|
|
73
|
+
# removal is the load-bearing path), and recent dirs are left for concurrent
|
|
74
|
+
# processes.
|
|
75
|
+
#
|
|
76
|
+
# @return [void]
|
|
77
|
+
def self.sweep_stale_temps!
|
|
78
|
+
return unless File.directory?(cache)
|
|
79
|
+
|
|
80
|
+
cutoff = Time.now - TEMP_STALE_SECONDS
|
|
81
|
+
Dir.children(cache).each do |entry|
|
|
82
|
+
next unless entry.start_with?('temp-')
|
|
83
|
+
|
|
84
|
+
path = File.join(cache, entry)
|
|
85
|
+
next unless File.directory?(path)
|
|
86
|
+
next if File.mtime(path) > cutoff
|
|
87
|
+
|
|
88
|
+
FileUtils.remove_entry(path)
|
|
89
|
+
rescue StandardError
|
|
90
|
+
# best-effort sweep; never block the host on dead state
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Pikuri's config root: +$XDG_CONFIG_HOME/pikuri+ or +~/.config/pikuri+.
|
|
95
|
+
# Home for user-owned, hand-edited config (the OS helper's +MACHINE.md+).
|
|
96
|
+
#
|
|
97
|
+
# A method for the same reason as {.cache}. Not created — config is
|
|
98
|
+
# user-owned, so callers +mkdir_p+ only when writing.
|
|
99
|
+
#
|
|
100
|
+
# @return [Pathname] the +<config home>/pikuri+ directory
|
|
101
|
+
def self.config
|
|
102
|
+
home = ENV['XDG_CONFIG_HOME']
|
|
103
|
+
home = File.expand_path('~/.config') if home.nil? || home.empty?
|
|
104
|
+
Pathname.new(home).join('pikuri')
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Pikuri's state root: +$XDG_STATE_HOME/pikuri+ or +~/.local/state/pikuri+.
|
|
108
|
+
# Created automatically. For state that persists across restarts but isn't
|
|
109
|
+
# portable enough for XDG_DATA_HOME (logs, history, layout).
|
|
110
|
+
#
|
|
111
|
+
# @return [Pathname] the +<state home>/pikuri+ directory
|
|
112
|
+
def self.state
|
|
113
|
+
state = ENV['XDG_STATE_HOME']
|
|
114
|
+
state = File.expand_path('~/.local/state') if state.nil? || state.empty?
|
|
115
|
+
state = Pathname.new(state).join('pikuri')
|
|
116
|
+
FileUtils.mkdir_p(state.to_path)
|
|
117
|
+
state
|
|
118
|
+
end
|
|
28
119
|
end
|
|
29
120
|
end
|