@bom0792/herdgate 0.9.36 → 0.9.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,13 @@
1
- // Native Herdr terminal controller. Never grants takeover and never interprets a shell command.
2
- import { spawn } from "node:child_process";
1
+ // Native Herdr terminal controller. Never interprets a shell command. --takeover makes the
2
+ // latest viewer own the stream: Herdr allows one controller per terminal, and this way a
3
+ // reconnect or a second viewer (phone, desktop) replaces the old one instead of failing.
4
+ import { execFile, spawn } from "node:child_process";
5
+ import { randomBytes } from "node:crypto";
3
6
 
4
7
  const maxLine = 1024 * 1024;
5
8
 
6
9
  export async function openTerminal(bin, id, cols, rows) {
7
- const child = spawn(bin, ["terminal", "session", "control", id, "--cols", String(cols), "--rows", String(rows)], {
10
+ const child = spawn(bin, ["terminal", "session", "control", id, "--takeover", "--cols", String(cols), "--rows", String(rows)], {
8
11
  stdio: ["pipe", "pipe", "ignore"], windowsHide: true, shell: false,
9
12
  });
10
13
  child.on("error", () => child.stdout.destroy());
@@ -59,3 +62,259 @@ async function* readFrames(stdout) {
59
62
  export function validDimension(value) {
60
63
  return Number.isInteger(value) && value > 0 && value <= 65535;
61
64
  }
65
+
66
+ // POSIX cksum (CRC-32 0x04C11DB7 + byte length, the same `cksum` coreutils prints):
67
+ // a content stamp that needs no stat and survives any transport. `readChangedCommand`
68
+ // stamps file reads with exactly this, so the phone can ask "same as before?" in one line.
69
+ const cksumTable = new Uint32Array(256).map((_, n) => {
70
+ let c = n << 24;
71
+ for (let k = 0; k < 8; k++) c = ((c << 1) ^ ((c & 0x80000000) !== 0 ? 0x04c11db7 : 0)) >>> 0;
72
+ return c >>> 0;
73
+ });
74
+ export function cksum(bytes) {
75
+ let crc = 0;
76
+ for (let i = 0; i < bytes.length; i++) crc = ((crc << 8) ^ cksumTable[((crc >>> 24) ^ bytes[i]) & 0xff]) >>> 0;
77
+ let len = bytes.length;
78
+ while (len > 0) { // the length rides the same CRC, low byte first, no leading zeros
79
+ crc = ((crc << 8) ^ cksumTable[((crc >>> 24) ^ (len & 0xff)) & 0xff]) >>> 0;
80
+ len = Math.floor(len / 256);
81
+ }
82
+ return `${(crc ^ 0xffffffff) >>> 0} ${bytes.length}`;
83
+ }
84
+
85
+ const newMarker = () => "HC_" + randomBytes(16).toString("hex");
86
+
87
+ /** One persistent `/bin/sh` with marker framing: every request is `<command> </dev/null`
88
+ * followed by a printf that emits `\n<unique marker> <exit code>\n`. stdin is detached
89
+ * per command so a hungry command cannot swallow the next request; a reply is decoded
90
+ * only once its whole delimiter is present, so chunked output never corrupts multibyte
91
+ * text. The shell is probed with `true` before it counts as alive; any channel failure
92
+ * (write error, early exit, timeout, over-limit) kills the whole generation — the
93
+ * written request is never retried, later requests fall back to one exec per command,
94
+ * and only an explicit reinit() may open the next generation. */
95
+ export function openShellManager({
96
+ exec,
97
+ shell = process.env.HERDR_SHELL || "/bin/sh",
98
+ probeCommand = "true",
99
+ timeoutMs = 4_000,
100
+ limitBytes = 8 * 1024 * 1024,
101
+ } = {}) {
102
+ const fallbackRun = exec || ((command) => new Promise((resolve) => {
103
+ execFile(shell, ["-c", command], { timeout: timeoutMs, maxBuffer: 512 * 1024, windowsHide: true }, (err, stdout) => {
104
+ resolve({ stdout: Buffer.from(stdout ?? ""), code: err && typeof err.code === "number" ? err.code : err ? 1 : 0 });
105
+ });
106
+ }));
107
+ let generation = 1;
108
+ let state = "idle"; // idle | opening | live | dead
109
+ let fails = 0;
110
+ let child = null;
111
+ let buffer = Buffer.alloc(0);
112
+ let waiting = []; // framed requests in submission order; head is decoded first
113
+ const stats = { shells: 0, shellRequests: 0, fallbackRequests: 0 };
114
+
115
+ const failChannel = (reason) => {
116
+ if (state === "dead") return;
117
+ const written = [];
118
+ const unwritten = [];
119
+ for (const req of waiting) (req.written ? written : unwritten).push(req);
120
+ waiting = [];
121
+ state = "dead";
122
+ fails++;
123
+ buffer = Buffer.alloc(0);
124
+ if (child) { try { child.stdin.end(); } catch { /* already gone */ } child.kill(); child = null; }
125
+ for (const req of written) {
126
+ clearTimeout(req.timer);
127
+ req.reject(Object.assign(new Error(`persistent shell failed: ${reason}`), { kind: "shell_failed", noRetry: true }));
128
+ }
129
+ // These were never written, so running them through the fallback is not a retry.
130
+ for (const req of unwritten) fallbackRun(req.command).then(req.resolve, req.reject);
131
+ };
132
+
133
+ const open = () => {
134
+ stats.shells++;
135
+ state = "opening";
136
+ const kid = spawn(shell, [], { stdio: ["pipe", "pipe", "ignore"], windowsHide: true });
137
+ child = kid;
138
+ kid.stdout.on("data", (chunk) => {
139
+ buffer = Buffer.concat([buffer, chunk]);
140
+ if (buffer.length > limitBytes) return failChannel("frame over limit");
141
+ for (;;) {
142
+ const head = waiting[0];
143
+ if (!head || !head.written) break;
144
+ const start = buffer.indexOf(head.delimiter);
145
+ if (start < 0) break;
146
+ const end = buffer.indexOf(10, start + head.delimiter.length);
147
+ if (end < 0) break;
148
+ const code = Number.parseInt(buffer.subarray(start + head.delimiter.length, end).toString("ascii"), 10);
149
+ if (!Number.isInteger(code) || code < 0) return failChannel("corrupt frame");
150
+ const stdout = buffer.subarray(0, start);
151
+ buffer = buffer.subarray(end + 1);
152
+ waiting.shift();
153
+ clearTimeout(head.timer);
154
+ if (head.probe) {
155
+ if (code !== 0) return failChannel("probe failed");
156
+ state = "live";
157
+ }
158
+ head.resolve({ stdout, code });
159
+ pump();
160
+ }
161
+ });
162
+ kid.stdin.on("error", () => { if (child === kid) failChannel("stdin write failed"); });
163
+ kid.on("error", () => { if (child === kid) failChannel("shell did not start"); });
164
+ kid.on("exit", () => { if (child === kid && state !== "dead") failChannel("shell exited"); });
165
+ const probe = { probe: true, command: probeCommand, marker: newMarker(), written: false };
166
+ probe.delimiter = Buffer.from(`\n${probe.marker} `);
167
+ probe.promise = new Promise((resolve, reject) => { probe.resolve = resolve; probe.reject = reject; });
168
+ waiting.push(probe);
169
+ probe.promise.catch(() => {}); // the probe's own failure is reported by failChannel
170
+ };
171
+
172
+ function pump() {
173
+ if (state === "dead" || !child) return;
174
+ const head = waiting[0];
175
+ if (!head || head.written) return;
176
+ if (!head.probe && state !== "live") return; // nothing runs before the probe answers
177
+ head.written = true;
178
+ stats.shellRequests++;
179
+ head.timer = setTimeout(() => failChannel("request timed out"), timeoutMs);
180
+ head.timer.unref?.();
181
+ // The redirect wraps the whole command (braces), so pipelines keep their stdin while
182
+ // no command can ever read the marker stream meant for the next request.
183
+ const frame = `{ ${head.command} ; } </dev/null\nprintf '\\n%s %s\\n' '${head.marker}' "$?"\n`;
184
+ child.stdin.write(frame, (err) => { if (err) failChannel("stdin write failed"); });
185
+ }
186
+
187
+ return {
188
+ execute(command) {
189
+ if (state === "dead") { stats.fallbackRequests++; return fallbackRun(command); }
190
+ if (state === "idle") open();
191
+ const req = { command, marker: newMarker(), written: false };
192
+ req.delimiter = Buffer.from(`\n${req.marker} `);
193
+ const promise = new Promise((resolve, reject) => { req.resolve = resolve; req.reject = reject; });
194
+ req.promise = promise;
195
+ waiting.push(req);
196
+ pump();
197
+ return promise;
198
+ },
199
+ // The only way a failed generation comes back: an explicit new one.
200
+ reinit() {
201
+ failChannel("reinit");
202
+ generation++;
203
+ fails = 0;
204
+ state = "idle";
205
+ },
206
+ // End of the line for this manager: close the shell without failing anyone.
207
+ close() {
208
+ if (!child) { state = "dead"; return; }
209
+ const kid = child;
210
+ child = null;
211
+ state = "dead";
212
+ try { kid.stdin.end(); } catch { /* already gone */ }
213
+ kid.kill();
214
+ },
215
+ get generation() { return generation; },
216
+ get state() { return state; },
217
+ get fails() { return fails; },
218
+ stats,
219
+ };
220
+ }
221
+
222
+ const resyncError = (reason = "pane input needs resync; re-read, never replay") =>
223
+ Object.assign(new Error(reason), { kind: "needs_resync" });
224
+
225
+ /** Keystrokes from the phone ride a bounded FIFO (PaneLiveInputQueue semantics): adjacent
226
+ * keystroke batches for the same pane coalesce Nagle-style, every input is acknowledged
227
+ * in submission order before the next batch is written, and any ambiguous failure —
228
+ * delivery unknown, or the queue overflows — flips the whole queue to needsResync and
229
+ * drops it: the client must resync from a fresh read, and old input is never replayed. */
230
+ export class PaneLiveInputQueue {
231
+ constructor(apply, { limit = 256 } = {}) {
232
+ if (typeof apply !== "function") throw new Error("PaneLiveInputQueue needs an apply function");
233
+ this.apply = apply;
234
+ this.limit = limit;
235
+ this.state = "live"; // live | needsResync
236
+ this.enqueuedCount = 0;
237
+ this.acknowledgedCount = 0;
238
+ this.droppedCount = 0;
239
+ this.pending = [];
240
+ this.pumping = false;
241
+ }
242
+
243
+ enqueue(paneId, kind, payload = {}) {
244
+ if (this.state !== "live") return Promise.reject(resyncError());
245
+ if (this.pending.length >= this.limit) {
246
+ this.resync("input queue overflow");
247
+ return Promise.reject(resyncError("input queue overflow"));
248
+ }
249
+ const tail = this.pending[this.pending.length - 1];
250
+ if (kind === "keys" && tail && !tail.sent && tail.paneId === paneId && tail.kind === "keys" && Array.isArray(payload.keys)) {
251
+ tail.keys = tail.keys.concat(payload.keys); // one send-keys for a burst of keystrokes
252
+ return this.claim(tail);
253
+ }
254
+ const event = { paneId, kind, text: payload.text, keys: payload.keys ? [...payload.keys] : undefined, sent: false, waiters: [] };
255
+ this.pending.push(event);
256
+ const promise = this.claim(event);
257
+ queueMicrotask(() => this.pump()); // a beat to gather a burst before the write starts
258
+ return promise;
259
+ }
260
+
261
+ /** Drop not-yet-sent input for a pane (the viewer switched away from it). */
262
+ abandon(paneId) {
263
+ if (this.state !== "live") return;
264
+ this.pending = this.pending.filter((event) => {
265
+ if (event.paneId !== paneId || event.sent) return true;
266
+ this.droppedCount += event.waiters.length;
267
+ for (const waiter of event.waiters) waiter.reject(Object.assign(new Error("pane input abandoned"), { kind: "abandoned" }));
268
+ return false;
269
+ });
270
+ }
271
+
272
+ resync(reason = "pane input needs resync") {
273
+ if (this.state !== "live") return;
274
+ this.state = "needsResync";
275
+ const events = this.pending;
276
+ this.pending = [];
277
+ for (const event of events) {
278
+ this.droppedCount += event.waiters.length;
279
+ for (const waiter of event.waiters) waiter.reject(resyncError(reason));
280
+ }
281
+ }
282
+
283
+ claim(event) {
284
+ const seq = ++this.enqueuedCount;
285
+ return new Promise((resolve, reject) => event.waiters.push({ seq, resolve, reject }));
286
+ }
287
+
288
+ async pump() {
289
+ if (this.pumping || this.state !== "live") return;
290
+ this.pumping = true;
291
+ try {
292
+ while (this.pending.length && this.state === "live") {
293
+ const event = this.pending[0];
294
+ event.sent = true;
295
+ let failure = null;
296
+ try {
297
+ await this.apply({ paneId: event.paneId, kind: event.kind, text: event.text, keys: event.keys });
298
+ } catch (err) {
299
+ // The host answered no: a refusal is a refusal — this input is done, the rest drain on.
300
+ // Anything ambiguous (delivery unknown) leaves the channel untrustworthy: resync.
301
+ if (err && err.definitive === true) failure = err;
302
+ else return this.resync(err?.message ?? "pane input delivery failed");
303
+ }
304
+ this.pending.shift();
305
+ this.acknowledge(event, failure);
306
+ }
307
+ } finally {
308
+ this.pumping = false;
309
+ }
310
+ }
311
+
312
+ /** Acks come back strictly in submission order, one count per accepted input. */
313
+ acknowledge(event, failure) {
314
+ for (const waiter of event.waiters) {
315
+ this.acknowledgedCount++;
316
+ if (failure) waiter.reject(failure);
317
+ else waiter.resolve(this.acknowledgedCount);
318
+ }
319
+ }
320
+ }
@@ -0,0 +1,183 @@
1
+ // Journal GC for the experimental Pi receipt journal (see experiments/pi-receipt).
2
+ // "The Luggage" school of retention: eat everything past its days, keep what
3
+ // might still matter. Fail-closed like every other journal reader — an entry
4
+ // whose status is 'unknown' is evidence not yet understood, so it is ALWAYS
5
+ // kept; a scope that cannot be read is reported, never thrown; a watch store
6
+ // that cannot be parsed is never rewritten. Dry-run by default: nothing is
7
+ // written unless apply: true, and even then every rewrite is tmp+rename.
8
+
9
+ import { readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
10
+ import { join } from 'node:path';
11
+ import { RECEIPT_ID, readJournal } from './pi-receipts.mjs';
12
+ import { watchFile } from './pi-receipt-watch.mjs';
13
+
14
+ const SCOPE = /^[0-9a-f]{16}$/; // one hashed dir per Pi workspace
15
+ const RECEIPTS = 'receipts.jsonl';
16
+ const DAY_MS = 86400e3;
17
+ const DEVICE = /^[\w.-]{1,64}$/; // same shape the watch store validates
18
+
19
+ /**
20
+ * The durable watch intents at base/watch.json are the only "cursors" over the
21
+ * journals (requestId -> receipt id). Missing file: no references. Unreadable
22
+ * or corrupt file: no references AND never rewritten (fail-closed — apply must
23
+ * not destroy intents it could not read).
24
+ */
25
+ function loadCursors(base) {
26
+ const file = watchFile(base);
27
+ let raw;
28
+ try { raw = JSON.parse(readFileSync(file, 'utf8')); }
29
+ catch (error) {
30
+ return { subs: [], file, writable: false, broken: error.code !== 'ENOENT' };
31
+ }
32
+ const list = Array.isArray(raw?.subscriptions) ? raw.subscriptions : [];
33
+ const subs = list.filter(s => DEVICE.test(s?.device ?? '') && RECEIPT_ID.test(s?.requestId ?? '')
34
+ && typeof s.wantBody === 'boolean' && Number.isFinite(s.createdAt) && Number.isFinite(s.expiresAt));
35
+ return { subs, file, writable: true };
36
+ }
37
+
38
+ /** Same atomic-private pattern as the journal writer (experiments/pi-receipt/journal.mjs). */
39
+ function atomicWrite(file, text) {
40
+ const tmp = `${file}.tmp`;
41
+ writeFileSync(tmp, text, { mode: 0o600 });
42
+ rmSync(file, { force: true });
43
+ renameSync(tmp, file);
44
+ }
45
+
46
+ /**
47
+ * Sweep every workspace scope under base and collect receipts older than
48
+ * `days` for deletion — except:
49
+ * - status 'unknown' (fail-closed: unclear evidence is always kept), and
50
+ * - receipts still referenced by a watch intent (a device may still be
51
+ * waiting to be shown the result).
52
+ * Body archives (responses/<id>.md) of deleted receipts join the candidates;
53
+ * their bytes fold into the receipt's candidate bytes. Watch intents pointing
54
+ * at ids no journal holds anymore are orphan cursors, reported always and
55
+ * removed only on apply. Never throws; unreadable scopes land in report.errors.
56
+ *
57
+ * apply=false returns { scopes, candidates, orphanCursors, bytesReclaimable,
58
+ * keptUnknown, errors } and touches nothing. apply=true additionally rewrites
59
+ * each affected journal without the deleted entries (tmp+rename), removes the
60
+ * archived bodies, retires orphan cursors, and adds { applied, deleted,
61
+ * bytesFreed } to the report.
62
+ */
63
+ export function gcJournal({ base, days = 90, now = Date.now(), apply = false } = {}) {
64
+ const at = Number.isFinite(now) ? now : Date.now();
65
+ const span = Number.isFinite(days) ? Math.max(0, Math.floor(days)) : 90;
66
+ const cutoff = at - span * DAY_MS;
67
+
68
+ const report = {
69
+ scopes: 0, // scope dirs successfully read
70
+ candidates: [], // {id, scope, ts, status, bytes} — bytes = journal line + body archive
71
+ orphanCursors: [], // {device, requestId}
72
+ bytesReclaimable: 0,
73
+ keptUnknown: 0, // old entries kept because their status is unknown
74
+ errors: [], // {scope, code}
75
+ };
76
+
77
+ let names = [];
78
+ try { names = readdirSync(base).filter(name => SCOPE.test(name)).sort(); }
79
+ catch (error) {
80
+ report.errors.push({ scope: null, code: error.code ?? 'unreadable_base' });
81
+ return report; // nothing can be read, nothing will be written
82
+ }
83
+
84
+ const byScope = new Map(); // scope -> entries (validated by readJournal)
85
+ for (const name of names) {
86
+ try {
87
+ byScope.set(name, readJournal(join(base, name, RECEIPTS)));
88
+ report.scopes += 1;
89
+ } catch (error) {
90
+ report.errors.push({ scope: name, code: error.code ?? 'unreadable_scope' });
91
+ }
92
+ }
93
+
94
+ const cursors = loadCursors(base);
95
+ if (cursors.broken) report.errors.push({ scope: 'watch', code: 'unreadable_watch' });
96
+ const referenced = new Set(cursors.subs.map(sub => sub.requestId));
97
+
98
+ const allIds = new Set(); // every receipt id any journal holds (post-scan)
99
+ for (const entries of byScope.values()) for (const entry of entries) allIds.add(entry.id);
100
+
101
+ // Pass 1 — pick deletion candidates per entry, keeping the fail-closed set.
102
+ const keptIds = new Set(); // ids that must survive somewhere
103
+ const delByScope = new Map(); // scope -> Set<id to drop>
104
+ const lineBytes = new Map(); // "scope\0id" -> journal-line bytes
105
+ for (const [scope, entries] of byScope) {
106
+ for (const entry of entries) {
107
+ const old = entry.ts < cutoff;
108
+ if (old && entry.status === 'unknown') {
109
+ report.keptUnknown += 1;
110
+ keptIds.add(entry.id);
111
+ continue;
112
+ }
113
+ if (old && !referenced.has(entry.id)) {
114
+ const bytes = Buffer.byteLength(JSON.stringify(entry), 'utf8') + 1; // line + \n
115
+ report.candidates.push({ id: entry.id, scope, ts: entry.ts, status: entry.status, bytes });
116
+ lineBytes.set(`${scope}\0${entry.id}`, bytes);
117
+ const ids = delByScope.get(scope) ?? new Set();
118
+ ids.add(entry.id);
119
+ delByScope.set(scope, ids);
120
+ } else {
121
+ keptIds.add(entry.id); // fresh, or still referenced by a watch intent
122
+ }
123
+ }
124
+ }
125
+
126
+ // Pass 2 — archived bodies of deleted receipts (the writer files them in the
127
+ // same scope). A body stays when any surviving receipt elsewhere owns the id.
128
+ const bodyFiles = []; // {id, scope, path, size}
129
+ for (const candidate of report.candidates) {
130
+ if (keptIds.has(candidate.id)) continue;
131
+ const path = join(base, candidate.scope, 'responses', `${candidate.id}.md`);
132
+ try {
133
+ const size = statSync(path).size;
134
+ candidate.bytes += size;
135
+ bodyFiles.push({ id: candidate.id, scope: candidate.scope, path, size });
136
+ } catch { /* no retained body: fine */ }
137
+ }
138
+
139
+ report.orphanCursors = cursors.subs
140
+ .filter(sub => !allIds.has(sub.requestId))
141
+ .map(sub => ({ device: sub.device, requestId: sub.requestId }))
142
+ .sort((a, b) => `${a.device}${a.requestId}`.localeCompare(`${b.device}${b.requestId}`));
143
+
144
+ report.bytesReclaimable = report.candidates.reduce((sum, c) => sum + c.bytes, 0);
145
+
146
+ if (!apply) return report;
147
+
148
+ // apply — every write tmp+rename, every failure reported, never thrown.
149
+ let applied = true;
150
+ let deleted = 0;
151
+ let bytesFreed = 0;
152
+ for (const [scope, ids] of delByScope) {
153
+ try {
154
+ const kept = byScope.get(scope).filter(entry => !ids.has(entry.id));
155
+ atomicWrite(join(base, scope, RECEIPTS), kept.map(entry => JSON.stringify(entry)).join('\n') + (kept.length ? '\n' : ''));
156
+ deleted += ids.size;
157
+ for (const id of ids) bytesFreed += lineBytes.get(`${scope}\0${id}`) ?? 0;
158
+ } catch (error) {
159
+ applied = false;
160
+ report.errors.push({ scope, code: error.code ?? 'rewrite_failed' });
161
+ }
162
+ }
163
+ for (const body of bodyFiles) {
164
+ try {
165
+ rmSync(body.path, { force: true });
166
+ bytesFreed += body.size;
167
+ } catch (error) {
168
+ applied = false;
169
+ report.errors.push({ scope: body.scope, code: error.code ?? 'body_delete_failed' });
170
+ }
171
+ }
172
+ if (report.orphanCursors.length && cursors.writable) {
173
+ try {
174
+ const keep = cursors.subs.filter(sub => allIds.has(sub.requestId));
175
+ atomicWrite(cursors.file, JSON.stringify({ v: 1, subscriptions: keep }) + '\n');
176
+ } catch (error) {
177
+ applied = false;
178
+ report.errors.push({ scope: 'watch', code: error.code ?? 'watch_rewrite_failed' });
179
+ }
180
+ }
181
+
182
+ return { ...report, applied, deleted, bytesFreed };
183
+ }
@@ -0,0 +1,217 @@
1
+ // BM25 keyword search over the local Pi receipt journal — device-safe core.
2
+ // The scoring may read archived bodies and Pi session transcripts (host-local
3
+ // files its user can already read); the RESULT handed to a device carries
4
+ // publicReceipt-shaped metadata only: id, score, status, reason, usage.
5
+ // No snippet, no sessionId, no session file, no timestamps — and no title
6
+ // (prompt-derived text) unless the caller explicitly opts in with withTitle.
7
+
8
+ import { closeSync, openSync, readdirSync, readFileSync, readSync, statSync } from 'node:fs';
9
+ import { homedir } from 'node:os';
10
+ import { join } from 'node:path';
11
+ import { publicReceipt } from './pi-receipts.mjs';
12
+ import { workerBase } from './pi-worker.mjs';
13
+
14
+ const K1 = 1.2;
15
+ const B = 0.75;
16
+ const TRANSCRIPT_CAP = 512 * 1024;
17
+ const BODY_CAP = 128 * 1024;
18
+ const SCOPE = /^[0-9a-f]{16}$/;
19
+
20
+ /** Letters+digits (any script), plus inner . _ / - of words; NFC-folded. */
21
+ export function tokenize(text) {
22
+ const out = [];
23
+ const re = /[\p{L}\p{N}]+(?:[._/\-][\p{L}\p{N}]+)*/gu;
24
+ let m;
25
+ while ((m = re.exec(String(text ?? '').normalize('NFC')))) out.push(m[0].toLowerCase());
26
+ return out;
27
+ }
28
+
29
+ export function buildIndex(docs) {
30
+ const terms = docs.map(d => tokenize(d.text));
31
+ const lengths = terms.map(t => t.length);
32
+ const avgdl = lengths.reduce((a, b) => a + b, 0) / (docs.length || 1);
33
+ const df = new Map();
34
+ terms.forEach(t => {
35
+ const seen = new Set(t);
36
+ for (const term of seen) df.set(term, (df.get(term) ?? 0) + 1);
37
+ });
38
+ return {
39
+ docs, terms, lengths, avgdl, df, n: docs.length,
40
+ idf(term) {
41
+ const d = this.df.get(term) ?? 0;
42
+ return Math.log(1 + (this.n - d + 0.5) / (d + 0.5));
43
+ },
44
+ };
45
+ }
46
+
47
+ /** Score every document against the query; ties break on newest ts. */
48
+ export function searchIndex(index, query, limit = 10) {
49
+ const q = tokenize(query);
50
+ if (!q.length || !index.n) return [];
51
+ const scored = [];
52
+ for (let i = 0; i < index.n; i++) {
53
+ const tf = new Map();
54
+ for (const t of index.terms[i]) tf.set(t, (tf.get(t) ?? 0) + 1);
55
+ let score = 0;
56
+ for (const term of new Set(q)) {
57
+ const f = tf.get(term);
58
+ if (!f) continue;
59
+ const norm = K1 * (1 - B + B * (index.lengths[i] / (index.avgdl || 1)));
60
+ score += index.idf(term) * (f * (K1 + 1)) / (f + norm);
61
+ }
62
+ if (score > 0) scored.push({ doc: index.docs[i], score });
63
+ }
64
+ scored.sort((a, b) => b.score - a.score || (b.doc.meta.ts ?? 0) - (a.doc.meta.ts ?? 0));
65
+ return limit >= scored.length ? scored : scored.slice(0, limit);
66
+ }
67
+
68
+ /** Best window around the first query-term hit (host-local display only). */
69
+ export function snippet(text, query, width = 160) {
70
+ const hay = String(text ?? '');
71
+ const terms = tokenize(query);
72
+ const lower = hay.toLowerCase();
73
+ let at = -1;
74
+ for (const term of terms) { at = lower.indexOf(term); if (at >= 0) break; }
75
+ if (at < 0) return hay.slice(0, width).replace(/\s+/g, ' ').trim();
76
+ const start = Math.max(0, at - Math.floor(width / 2));
77
+ const cut = hay.slice(start, start + width).replace(/\s+/g, ' ').trim();
78
+ return start > 0 ? `…${cut}` : cut;
79
+ }
80
+
81
+ function readJsonl(file, cap) {
82
+ try {
83
+ const stat = statSync(file);
84
+ const fd = openSync(file, 'r');
85
+ try {
86
+ const size = Math.min(stat.size, cap);
87
+ const buf = Buffer.alloc(size);
88
+ let read = 0;
89
+ while (read < size) {
90
+ const n = readSync(fd, buf, read, size - read, null);
91
+ if (!n) break;
92
+ read += n;
93
+ }
94
+ const out = [];
95
+ for (const line of buf.toString('utf8').split('\n')) {
96
+ try { out.push(JSON.parse(line)); } catch { /* torn tail or junk */ }
97
+ }
98
+ return out;
99
+ } finally { closeSync(fd); }
100
+ } catch { return []; }
101
+ }
102
+
103
+ function textOfContent(content) {
104
+ if (typeof content === 'string') return content;
105
+ if (!Array.isArray(content)) return '';
106
+ return content.map(b => (typeof b?.text === 'string' ? b.text : '')).join(' ');
107
+ }
108
+
109
+ function transcriptText(file) {
110
+ const parts = [];
111
+ for (const e of readJsonl(file, TRANSCRIPT_CAP)) {
112
+ if (e?.type !== 'message') continue;
113
+ const role = e.message?.role;
114
+ if (role !== 'user' && role !== 'assistant') continue;
115
+ const text = textOfContent(e.message.content).trim();
116
+ if (text) parts.push(text);
117
+ }
118
+ return parts.join('\n');
119
+ }
120
+
121
+ /** Match a Pi session transcript by its id: <root>/<slug>/<ts>_<sessionId>.jsonl.
122
+ * Two roots hold transcripts: Pi's default storage for TUI sessions, and the
123
+ * worker base for SDK-created job sessions (they live outside Pi's storage —
124
+ * the same reason restore-by-file insists on --session <file>). */
125
+ export function sessionRoots(env = process.env) {
126
+ return [
127
+ join(env.PI_HOME ?? join(homedir(), '.pi'), 'agent', 'sessions'),
128
+ join(workerBase(env), 'sessions'),
129
+ ];
130
+ }
131
+
132
+ export function findSessionFile(sessionsRoots, sessionId) {
133
+ if (!sessionId) return null;
134
+ const roots = sessionsRoots === null || sessionsRoots === undefined ? []
135
+ : Array.isArray(sessionsRoots) ? sessionsRoots : [sessionsRoots];
136
+ for (const sessionsDir of roots) {
137
+ try {
138
+ const suffix = `_${sessionId}.jsonl`;
139
+ for (const name of readdirSync(sessionsDir)) {
140
+ // Two layouts: Pi's <root>/<cwd-slug>/<ts>_<id>.jsonl and the worker's
141
+ // flat <root>/<ts>_<id>.jsonl (SDK sessions carry no cwd slug).
142
+ if (name.endsWith(suffix)) return join(sessionsDir, name);
143
+ const dir = join(sessionsDir, name);
144
+ let names;
145
+ try { names = readdirSync(dir); } catch { continue; }
146
+ for (const inner of names) {
147
+ if (inner.endsWith(suffix)) return join(dir, inner);
148
+ }
149
+ }
150
+ } catch { /* root missing: fine */ }
151
+ }
152
+ return null;
153
+ }
154
+
155
+ /** Where Pi stores session transcripts on this host (PI_HOME wins, like the CLI). */
156
+ export function sessionsDir(env = process.env) {
157
+ return sessionRoots(env)[0];
158
+ }
159
+
160
+ function metaLine(meta) {
161
+ const bits = [meta.status, meta.reason, meta.paneId, meta.principal, meta.title];
162
+ if (meta.usage) bits.push(`${meta.usage.totalTokens ?? 0} tokens`, `$${meta.usage.costUsd ?? 0}`);
163
+ return bits.filter(Boolean).join(' ');
164
+ }
165
+
166
+ /** Every receipt in every workspace scope becomes one searchable document (host-local). */
167
+ export function collectDocs({ base, sessionsRoot, withTranscripts = true } = {}) {
168
+ const docs = [];
169
+ let scopes = [];
170
+ try { scopes = readdirSync(base).filter(n => SCOPE.test(n)); }
171
+ catch { return docs; }
172
+ for (const scope of scopes) {
173
+ const dir = join(base, scope);
174
+ for (const entry of readJsonl(join(dir, 'receipts.jsonl'), 4 * 1024 * 1024)) {
175
+ if (!entry?.v || !entry.id || !entry.sessionId) continue;
176
+ const meta = {
177
+ id: entry.id, sessionId: entry.sessionId, ts: entry.ts ?? 0,
178
+ status: entry.status, reason: entry.reason, paneId: entry.paneId, usage: entry.usage,
179
+ ...(entry.principal ? { principal: entry.principal } : {}),
180
+ ...(entry.title ? { title: entry.title } : {}),
181
+ };
182
+ let text = metaLine(meta);
183
+ let sessionFile = null;
184
+ if (withTranscripts) {
185
+ sessionFile = findSessionFile(sessionsRoot ?? sessionRoots(), entry.sessionId);
186
+ if (sessionFile) text += '\n' + transcriptText(sessionFile);
187
+ }
188
+ try { text += '\n' + readFileSync(join(dir, 'responses', `${entry.id}.md`), 'utf8').slice(0, BODY_CAP); }
189
+ catch { /* not retained: fine */ }
190
+ docs.push({ id: entry.id, text, meta, sessionFile });
191
+ }
192
+ }
193
+ return docs;
194
+ }
195
+
196
+ /**
197
+ * Device-safe search: full-text scoring host-side, publicReceipt-shaped hits.
198
+ * One hit per session by default (a long transcript owns many receipts; without
199
+ * collapsing it would flood the caller's result page). Never throws.
200
+ */
201
+ export function searchReceipts({ base, sessionsRoot, query, limit = 10, dedupe = true, all = false, withTitle = false } = {}) {
202
+ try {
203
+ const docs = collectDocs({ base, sessionsRoot });
204
+ const ranked = searchIndex(buildIndex(docs), query, docs.length);
205
+ const hits = [];
206
+ const seen = new Set();
207
+ for (const { doc, score } of ranked) {
208
+ if (dedupe && !all && doc.meta.sessionId && seen.has(doc.meta.sessionId)) continue;
209
+ if (doc.meta.sessionId) seen.add(doc.meta.sessionId);
210
+ hits.push({ id: doc.id, score: Math.round(score * 1000) / 1000, ...publicReceipt({ ...doc.meta, v: 1, ts: doc.meta.ts }, { withTitle }) });
211
+ if (hits.length >= Math.max(1, Math.min(50, limit))) break;
212
+ }
213
+ return hits;
214
+ } catch {
215
+ return []; // unreadable journal base: fail closed, no hits
216
+ }
217
+ }