@staix/agent-mem 0.19.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +4 -4
- package/package.json +1 -1
- package/src/cli.ts +106 -4
- package/src/commands-table.ts +5 -0
- package/src/hook-contract.ts +5 -2
- package/src/hook-rss-wire.ts +2 -0
- package/src/hook-spool-replay.ts +332 -0
- package/src/hook-spool.ts +271 -55
- package/src/hook-transcript.ts +22 -6
- package/src/hooks.ts +121 -60
- package/src/memory/doctor.ts +32 -2
- package/src/memory/metrics.ts +4 -3
- package/src/memory/public-prune-view.ts +6 -2
- package/src/session-brief-worker.ts +105 -54
- package/src/setup-drift.ts +3 -3
- package/src/setup-jcode.ts +17 -8
- package/src/setup-owned.ts +62 -6
- package/src/setup-toml.ts +58 -0
- package/src/setup.ts +62 -15
package/src/hook-spool.ts
CHANGED
|
@@ -22,12 +22,18 @@ import { HOOK_EVENTS, HOOK_INPUT_BYTES } from "./hook-contract.ts";
|
|
|
22
22
|
// `hook-` nor `hook-v2-` for it, so they leave it alone. Claude native rows stay format 2, byte for byte, so 0.18.0 keeps draining them.
|
|
23
23
|
const SCHEMA = "agent-mem.hook-capture/v1", FAMILY = "agent-mem.hook-capture", FORMAT = 2, PREFIX = "hook-v2-", FORMAT_V3 = 3, PREFIX_V3 = "hook-v3-";
|
|
24
24
|
const RECORD_BYTES = HOOK_INPUT_BYTES + 8192, MAX_FILES = 256, MAX_BYTES = 2 * 1024 * 1024;
|
|
25
|
-
|
|
25
|
+
// One hook previously performed one replay of at most 32 files / 256 KiB.
|
|
26
|
+
// Interleaved subpasses share these same ceilings; they do not each get a fresh allowance.
|
|
27
|
+
export const HOOK_REPLAY_MAX_FILES = 32, HOOK_REPLAY_MAX_BYTES = 256 * 1024;
|
|
28
|
+
const DRAIN_FILES = HOOK_REPLAY_MAX_FILES, DRAIN_BYTES = HOOK_REPLAY_MAX_BYTES;
|
|
29
|
+
export interface HookReplayBudget { files: number; bytes: number; seen: Set<string> }
|
|
30
|
+
export const hookReplayBudget = (): HookReplayBudget => ({ files: HOOK_REPLAY_MAX_FILES, bytes: HOOK_REPLAY_MAX_BYTES, seen: new Set() });
|
|
26
31
|
// ponytail: a finite native admission reserve covers deferred replay, not unlimited locked intake.
|
|
27
32
|
// Upgrade: coordinated append-only capacity accounting and observable backpressure at exhaustion.
|
|
28
33
|
const CAPACITY_FILES = 32, CAPACITY_BYTES = 256 * 1024;
|
|
29
34
|
const namePattern = /^(hook-(?:v[23]-)?([a-f0-9]{64})-([a-f0-9]{64})-([a-f0-9]{32})\.json)(?:\.draining-[a-f0-9]{24})?$/;
|
|
30
35
|
const hash = (value: string | Uint8Array) => createHash("sha256").update(value).digest("hex");
|
|
36
|
+
const summaryEvent = (event: string) => ["Title", "Recap", "Compact"].includes(event);
|
|
31
37
|
interface Record { schema: string; format?: number; cwd: string; scope: string | null; root?: string; nonce: string; capture: NormalizedCaptureFields }
|
|
32
38
|
export interface PendingHookCapture { name: string; cwd: string; receipt: string; deferred?: true; backlog?: true }
|
|
33
39
|
export class HookSpoolFull extends Error { constructor() { super("hook storage full"); } }
|
|
@@ -43,7 +49,9 @@ function belowFloor(base: string): boolean {
|
|
|
43
49
|
function physicalRoot(cwd: string): string {
|
|
44
50
|
const stat = lstatSync(cwd);
|
|
45
51
|
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("hook root unavailable");
|
|
46
|
-
|
|
52
|
+
// macOS can assign a different st_dev after reboot. Keep the directory's inode
|
|
53
|
+
// and birth time; open-file ownership still compares dev as well as inode.
|
|
54
|
+
return hash(`${stat.ino}:${stat.birthtimeMs}`);
|
|
47
55
|
}
|
|
48
56
|
|
|
49
57
|
function privateDirectory(dir: string): void {
|
|
@@ -52,23 +60,92 @@ function privateDirectory(dir: string): void {
|
|
|
52
60
|
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("hook storage unavailable");
|
|
53
61
|
chmodSync(dir, 0o700);
|
|
54
62
|
}
|
|
55
|
-
function entries(base: string, capacity = false): { name: string; size: number; modified: number }[] {
|
|
63
|
+
function entries(base: string, capacity = false, coverage?: { complete: boolean }): { name: string; size: number; modified: number; dev: number; ino: number }[] {
|
|
64
|
+
if (coverage) coverage.complete = false;
|
|
56
65
|
const dir = capacity ? join(directory(base), ".capacity") : directory(base), maximum = capacity ? CAPACITY_FILES : MAX_FILES, stat = lstatSync(dir, { throwIfNoEntry: false });
|
|
57
|
-
if (!stat) return [];
|
|
66
|
+
if (!stat) { if (coverage) coverage.complete = true; return []; }
|
|
58
67
|
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error("hook storage unavailable");
|
|
59
|
-
const reader = opendirSync(dir), rows: { name: string; size: number; modified: number }[] = [];
|
|
68
|
+
const reader = opendirSync(dir), rows: { name: string; size: number; modified: number; dev: number; ino: number }[] = [];
|
|
60
69
|
try {
|
|
61
70
|
for (let scanned = 0; scanned <= maximum + 4; scanned++) {
|
|
62
|
-
const entry = reader.readSync(); if (!entry) return rows;
|
|
71
|
+
const entry = reader.readSync(); if (!entry) { if (coverage) coverage.complete = true; return rows; }
|
|
63
72
|
if ([".writing", ".quarantine", ".capacity", ".maintenance"].includes(entry.name) && entry.isDirectory()) continue;
|
|
64
73
|
if (rows.length === maximum) return rows;
|
|
65
74
|
const item = lstatSync(join(dir, entry.name), { throwIfNoEntry: false });
|
|
66
|
-
if (item) rows.push({ name: entry.name, size: item.size, modified: item.mtimeMs });
|
|
75
|
+
if (item) rows.push({ name: entry.name, size: item.size, modified: item.mtimeMs, dev: item.dev, ino: item.ino });
|
|
67
76
|
}
|
|
68
77
|
return rows;
|
|
69
78
|
} finally { reader.closeSync(); }
|
|
70
79
|
}
|
|
71
80
|
|
|
81
|
+
/** A bounded diagnostic count of this cwd's active publications (names only, no stat).
|
|
82
|
+
* Task 110 schedules replay by publication/backpressure progress, not this legacy count. */
|
|
83
|
+
export function hookPendingCount(cwd: string, base: string): number {
|
|
84
|
+
try {
|
|
85
|
+
const dir = directory(canonicalPath(base)), wanted = hash(realPath(cwd)), reader = opendirSync(dir);
|
|
86
|
+
let count = 0;
|
|
87
|
+
try {
|
|
88
|
+
for (let scanned = 0; scanned < MAX_FILES; scanned++) {
|
|
89
|
+
const entry = reader.readSync(); if (!entry) break;
|
|
90
|
+
const match = namePattern.exec(entry.name);
|
|
91
|
+
if (match && match[2] === wanted) count++;
|
|
92
|
+
}
|
|
93
|
+
} finally { reader.closeSync(); }
|
|
94
|
+
return count;
|
|
95
|
+
} catch { return 0; }
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Read-only bounded metadata inventory. Counts are lower bounds when complete is false. */
|
|
99
|
+
export function readHookSpoolStatus(base: string) {
|
|
100
|
+
assertFormatFloor(base, "hook-spool");
|
|
101
|
+
let claimed = 0, oldest = Infinity, complete = true;
|
|
102
|
+
const publications: { folder: string; name: string; size: number; modified: number }[] = [];
|
|
103
|
+
const tier = (capacity: boolean) => {
|
|
104
|
+
const dir = capacity ? join(directory(base), ".capacity") : directory(base);
|
|
105
|
+
const maxFiles = capacity ? CAPACITY_FILES : MAX_FILES, maxBytes = capacity ? CAPACITY_BYTES : MAX_BYTES;
|
|
106
|
+
const named = lstatSync(dir, { throwIfNoEntry: false });
|
|
107
|
+
let files = 0, bytes = 0;
|
|
108
|
+
if (!named) return { files, bytes, maxFiles, maxBytes };
|
|
109
|
+
if (!named.isDirectory() || named.isSymbolicLink()) throw new Error("hook storage unavailable");
|
|
110
|
+
const reader = opendirSync(dir);
|
|
111
|
+
try {
|
|
112
|
+
for (let scanned = 0; scanned <= maxFiles + 4; scanned++) {
|
|
113
|
+
const entry = reader.readSync();
|
|
114
|
+
if (!entry) return { files, bytes, maxFiles, maxBytes };
|
|
115
|
+
if (!capacity && [".writing", ".quarantine", ".capacity", ".maintenance"].includes(entry.name) && entry.isDirectory()) continue;
|
|
116
|
+
if (files === maxFiles) break;
|
|
117
|
+
const item = lstatSync(join(dir, entry.name), { throwIfNoEntry: false });
|
|
118
|
+
if (!item) { complete = false; continue; }
|
|
119
|
+
files++; bytes += item.size; oldest = Math.min(oldest, item.mtimeMs);
|
|
120
|
+
publications.push({ folder: dir, name: entry.name, size: item.size, modified: item.mtimeMs });
|
|
121
|
+
if (namePattern.test(entry.name) && entry.name.includes(".draining-")) claimed++;
|
|
122
|
+
}
|
|
123
|
+
complete = false;
|
|
124
|
+
return { files, bytes, maxFiles, maxBytes };
|
|
125
|
+
} finally { reader.closeSync(); }
|
|
126
|
+
};
|
|
127
|
+
const active = tier(false), reserve = tier(true), budget = maintenanceBudget();
|
|
128
|
+
let native = 0, summaries = 0;
|
|
129
|
+
for (const row of publications) {
|
|
130
|
+
const match = namePattern.exec(row.name);
|
|
131
|
+
if (!match || row.size > RECORD_BYTES) continue;
|
|
132
|
+
if (!budget.files || row.size > budget.bytes) break;
|
|
133
|
+
budget.files--; budget.bytes -= row.size;
|
|
134
|
+
let file: OwnedFile | undefined;
|
|
135
|
+
try {
|
|
136
|
+
file = openOwned(join(row.folder, row.name));
|
|
137
|
+
if (file.initial.size !== row.size || file.initial.mtimeMs !== row.modified) continue;
|
|
138
|
+
const record = decoded(file);
|
|
139
|
+
if (!record || record.unknown || hash(record.value.cwd) !== match[2] || (record.value.scope ?? "0".repeat(64)) !== match[3] || record.value.nonce !== match[4]) continue;
|
|
140
|
+
if (summaryEvent(record.value.capture.event)) summaries++; else native++;
|
|
141
|
+
} catch { /* Body classification stays unknown; metadata does not become a false zero. */ }
|
|
142
|
+
finally { if (file) closeSync(file.fd); }
|
|
143
|
+
}
|
|
144
|
+
const unclassified = active.files + reserve.files - native - summaries;
|
|
145
|
+
const kinds = { native, summaries, unclassified, complete: complete && unclassified === 0 };
|
|
146
|
+
return { active, reserve, claimed, oldestAt: Number.isFinite(oldest) ? new Date(oldest).toISOString() : null, complete, kinds };
|
|
147
|
+
}
|
|
148
|
+
|
|
72
149
|
/** Validate ordinary Git value syntax even for ignored keys: invalid unused configuration can make
|
|
73
150
|
* native Git refuse the whole file, so ignoring its syntax could falsely certify project resolution. */
|
|
74
151
|
function validateConfigValue(value: string): void {
|
|
@@ -139,11 +216,16 @@ function identityConfig(text: string, include?: (path: string, condition?: strin
|
|
|
139
216
|
* ponytail: strict unsupported Git configuration remains unknown, not guessed. A native historical-origin
|
|
140
217
|
* receipt would broaden that supported subset without relaxing the physical replacement boundary. */
|
|
141
218
|
export function hookCaptureScope(cwd: string, base: string, cacheValidity = false, extraGitConfig?: string): string | undefined {
|
|
219
|
+
return captureScope(cwd, base, cacheValidity, extraGitConfig, false);
|
|
220
|
+
}
|
|
221
|
+
/** 0.19.0 and older write a device-bound scope. Only its current-boot value is knowable;
|
|
222
|
+
* never guess an old device or silently rebind a record whose HOME/Git identity changed. */
|
|
223
|
+
function captureScope(cwd: string, base: string, cacheValidity: boolean, extraGitConfig: string | undefined, legacyDevice: boolean): string | undefined {
|
|
142
224
|
try {
|
|
143
225
|
const parts: string[] = [], identity = (file: string) => {
|
|
144
226
|
const stat = lstatSync(file);
|
|
145
227
|
if (!stat.isDirectory()) throw new Error("physical scope unavailable");
|
|
146
|
-
return `${stat.dev}
|
|
228
|
+
return `${legacyDevice || cacheValidity ? `${stat.dev}:` : ""}${stat.ino}:${stat.birthtimeMs}`;
|
|
147
229
|
};
|
|
148
230
|
const content = (file: string, maximum = 64 * 1024): string => {
|
|
149
231
|
const named = lstatSync(file, { throwIfNoEntry: false });
|
|
@@ -365,11 +447,13 @@ function retireClaim(base: string, file: OwnedFile, preserve: boolean, batch?: Q
|
|
|
365
447
|
return true;
|
|
366
448
|
}
|
|
367
449
|
|
|
368
|
-
const RETIRE_AFTER_MS = 7 * 24 * 3600_000;
|
|
450
|
+
const RETIRE_AFTER_MS = 7 * 24 * 3600_000, MISSING_CWD_AFTER_MS = 3600_000;
|
|
451
|
+
interface MaintenanceBudget { files: number; bytes: number }
|
|
452
|
+
const maintenanceBudget = (): MaintenanceBudget => ({ files: DRAIN_FILES, bytes: DRAIN_BYTES });
|
|
369
453
|
/** Private numeric maintenance cursor: each invocation inspects a different bounded slice. */
|
|
370
|
-
function retirementCursor(base: string, capacity: boolean, length: number): { start: number; save(n: number): void } {
|
|
454
|
+
function retirementCursor(base: string, capacity: boolean, length: number, pressure = false): { start: number; save(n: number): void } {
|
|
371
455
|
const dir = join(directory(base), ".maintenance"); privateDirectory(dir);
|
|
372
|
-
const name = join(dir, capacity ? "reserve" : "active");
|
|
456
|
+
const name = join(dir, `${capacity ? "reserve" : "active"}${pressure ? "-pressure" : ""}`);
|
|
373
457
|
let start = 0, file: OwnedFile | undefined;
|
|
374
458
|
try {
|
|
375
459
|
file = openOwned(name);
|
|
@@ -385,62 +469,125 @@ function retirementCursor(base: string, capacity: boolean, length: number): { st
|
|
|
385
469
|
finally { rmSync(temporary, { force: true }); }
|
|
386
470
|
} };
|
|
387
471
|
}
|
|
388
|
-
/**
|
|
389
|
-
*
|
|
390
|
-
*
|
|
391
|
-
|
|
392
|
-
|
|
472
|
+
/** Bounded, counted retirement: missing cwds after one hour, other unresolvable known
|
|
473
|
+
* scopes after seven days, and fair-share pressure. Newer schemas retain their seven-day
|
|
474
|
+
* protection. A semantic/physical scope mismatch alone never proves a cwd was replaced.
|
|
475
|
+
* ponytail: fair shares are cooperative admission, not an atomic multi-process quota.
|
|
476
|
+
* Upgrade: coordinated byte/file reservations with crash recovery. */
|
|
477
|
+
function retireUnresolved(base: string, admittingCwd: string | undefined, inventory: ReturnType<typeof entries>, incomingBytes: number, minimal: boolean, capacity = false, budget = maintenanceBudget(), scanLimit = DRAIN_FILES): Set<string> {
|
|
478
|
+
const retired = new Set<string>(), cwdHash = admittingCwd === undefined ? undefined : hash(admittingCwd);
|
|
393
479
|
let count = inventory.length, bytes = inventory.reduce((sum, row) => sum + row.size, 0);
|
|
394
|
-
const
|
|
395
|
-
const
|
|
396
|
-
|
|
480
|
+
const maximum = capacity ? CAPACITY_FILES : MAX_FILES, byteMaximum = capacity ? CAPACITY_BYTES : MAX_BYTES, share = maximum / 4;
|
|
481
|
+
const groups = new Map<string, number>();
|
|
482
|
+
for (const row of inventory) {
|
|
483
|
+
const match = namePattern.exec(row.name); if (match) groups.set(match[2]!, (groups.get(match[2]!) ?? 0) + 1);
|
|
484
|
+
}
|
|
485
|
+
const quota = () => cwdHash !== undefined && (groups.get(cwdHash) ?? 0) >= share;
|
|
486
|
+
const pressure = () => cwdHash !== undefined && (quota() || count >= maximum || bytes + incomingBytes > byteMaximum);
|
|
487
|
+
const due = inventory.some(row => Date.now() - row.modified >= MISSING_CWD_AFTER_MS);
|
|
488
|
+
const pressured = pressure();
|
|
489
|
+
if (!inventory.length || minimal && !pressured) return retired;
|
|
397
490
|
const active = [...inventory].sort((a, b) => a.modified - b.modified || a.name.localeCompare(b.name));
|
|
398
491
|
let cursor: ReturnType<typeof retirementCursor>;
|
|
399
|
-
try { cursor = retirementCursor(base, capacity, active.length); } catch { return retired; }
|
|
400
|
-
let
|
|
492
|
+
try { cursor = retirementCursor(base, capacity, active.length, pressured); } catch { return retired; }
|
|
493
|
+
let scanned = 0;
|
|
401
494
|
const scopes = new Map<string, string | undefined>();
|
|
495
|
+
// Pressure first visits the largest share; HOME variants of one cwd share one allowance.
|
|
496
|
+
const ordered = pressure() ? [...active].sort((a, b) => {
|
|
497
|
+
const group = (row: typeof a) => namePattern.exec(row.name)?.[2];
|
|
498
|
+
const score = (row: typeof a) => { const key = group(row); return quota() ? Number(key === cwdHash) * (maximum + 1) : groups.get(key ?? "") ?? 0; };
|
|
499
|
+
return score(b) - score(a) || a.modified - b.modified || a.name.localeCompare(b.name);
|
|
500
|
+
}) : active;
|
|
501
|
+
const candidates = ordered.slice(cursor.start).concat(ordered.slice(0, cursor.start));
|
|
502
|
+
const held: { file: OwnedFile; row: typeof inventory[number]; group: string }[] = [];
|
|
503
|
+
let classified = 0, nativeSeen = false;
|
|
402
504
|
try {
|
|
403
|
-
for (
|
|
404
|
-
if (
|
|
405
|
-
const
|
|
406
|
-
|
|
407
|
-
|
|
505
|
+
for (const row of candidates) {
|
|
506
|
+
if (!budget.files || scanned >= scanLimit || !pressure() && (minimal || pressured && !due)) break;
|
|
507
|
+
const match = namePattern.exec(row.name); if (!match) { scanned++; continue; }
|
|
508
|
+
if (row.size <= RECORD_BYTES && row.size > budget.bytes) break;
|
|
509
|
+
budget.files--; scanned++;
|
|
408
510
|
let file: OwnedFile | undefined;
|
|
409
511
|
try {
|
|
410
512
|
file = openOwned(join(capacity ? join(directory(base), ".capacity") : directory(base), row.name));
|
|
411
513
|
if (file.initial.size !== row.size || file.initial.mtimeMs !== row.modified) continue;
|
|
412
|
-
if (row.size <= RECORD_BYTES)
|
|
514
|
+
if (row.size <= RECORD_BYTES) budget.bytes -= row.size;
|
|
413
515
|
const record = decoded(file);
|
|
414
516
|
const valid = record && hash(record.value.cwd) === match[2] && record.value.nonce === match[4] && (record.value.scope ?? "0".repeat(64)) === match[3];
|
|
415
517
|
let reason: CaptureDropReason | undefined;
|
|
416
518
|
if (valid && record.unknown) {
|
|
417
|
-
// #199: a newer writer's record waits for a reader that knows it, then leaves with a counted reason, never at once.
|
|
418
519
|
if (Date.now() - row.modified < RETIRE_AFTER_MS) continue;
|
|
419
520
|
reason = "unknown_schema_expired";
|
|
420
521
|
} else if (valid) {
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
522
|
+
classified++;
|
|
523
|
+
const summary = summaryEvent(record.value.capture.event);
|
|
524
|
+
if (!summary) nativeSeen = true;
|
|
525
|
+
const age = Date.now() - Date.parse(record.value.capture.at);
|
|
526
|
+
if (age >= MISSING_CWD_AFTER_MS && Date.now() - row.modified >= MISSING_CWD_AFTER_MS) {
|
|
527
|
+
try { lstatSync(record.value.cwd); }
|
|
528
|
+
catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") reason = "cwd_removed"; }
|
|
529
|
+
}
|
|
530
|
+
if (!reason && age >= RETIRE_AFTER_MS) {
|
|
425
531
|
if (record.value.scope === null) reason = "scope_unknown";
|
|
426
532
|
else {
|
|
427
533
|
if (!scopes.has(record.value.cwd)) scopes.set(record.value.cwd, hookCaptureScope(record.value.cwd, base));
|
|
428
534
|
const currentScope = scopes.get(record.value.cwd);
|
|
429
535
|
if (currentScope === undefined) reason = "scope_unknown";
|
|
430
|
-
else if (currentScope !== record.value.scope) reason = "undrainable_expired";
|
|
536
|
+
else if (currentScope !== record.value.scope && captureScope(record.value.cwd, base, false, undefined, true) !== record.value.scope) reason = "undrainable_expired";
|
|
431
537
|
}
|
|
432
538
|
}
|
|
539
|
+
if (!reason && pressure()) {
|
|
540
|
+
if (summary) {
|
|
541
|
+
// An advanced transcript cursor relies on accepted summaries. Hold their
|
|
542
|
+
// descriptors only to prove a complete all-summary foreign inventory.
|
|
543
|
+
if (!quota()) { held.push({ file, row, group: match[2]! }); file = undefined; }
|
|
544
|
+
continue;
|
|
545
|
+
}
|
|
546
|
+
if (!quota() || match[2] === cwdHash) reason = "scope_capacity";
|
|
547
|
+
}
|
|
433
548
|
if (!reason) continue;
|
|
434
549
|
}
|
|
435
550
|
const claim = claimOwned(base, row.name, file);
|
|
436
551
|
if (!claim || !owns(claim)) continue;
|
|
437
|
-
|
|
552
|
+
if (reason === "cwd_removed") {
|
|
553
|
+
try { lstatSync(record!.value.cwd); continue; }
|
|
554
|
+
catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") continue; }
|
|
555
|
+
}
|
|
556
|
+
// Persist the numeric loss receipt BEFORE removing a replayable body.
|
|
438
557
|
if (reason) appendMetrics(base, [{ at: new Date().toISOString(), kind: "hook", captureDrops: 1, captureDropReason: reason }]);
|
|
439
|
-
if (retireClaim(base, claim, false)) {
|
|
558
|
+
if (retireClaim(base, claim, false)) {
|
|
559
|
+
retired.add(row.name); count--; bytes -= row.size;
|
|
560
|
+
groups.set(match[2]!, Math.max(0, (groups.get(match[2]!) ?? 0) - 1));
|
|
561
|
+
}
|
|
440
562
|
} catch { /* Transient I/O and changed evidence stay pending. */ }
|
|
441
563
|
finally { if (file) closeSync(file.fd); }
|
|
442
564
|
}
|
|
443
|
-
|
|
565
|
+
// ponytail: a cap of summaries from four or more orphaned cwds can still
|
|
566
|
+
// refuse intake until TTL/replay: 32 files/256 KiB cannot prove native absence
|
|
567
|
+
// across the whole cap. Upgrade: a bounded derived classification cache.
|
|
568
|
+
if (pressure() && !quota() && !nativeSeen && classified === inventory.length && held.length === inventory.length
|
|
569
|
+
&& held.every(item => owns(item.file))) {
|
|
570
|
+
const coverage = { complete: false }, current = entries(base, capacity, coverage);
|
|
571
|
+
if (coverage.complete && current.length === inventory.length && current.every(row => inventory.some(prior => prior.name === row.name && prior.size === row.size && prior.modified === row.modified))) {
|
|
572
|
+
const victims = held.filter(item => item.group !== cwdHash).sort((a, b) => (groups.get(b.group) ?? 0) - (groups.get(a.group) ?? 0) || a.row.modified - b.row.modified || a.row.name.localeCompare(b.row.name));
|
|
573
|
+
for (const victim of victims) {
|
|
574
|
+
if (!pressure()) break;
|
|
575
|
+
try {
|
|
576
|
+
const claim = claimOwned(base, victim.row.name, victim.file);
|
|
577
|
+
if (!claim || !owns(claim)) continue;
|
|
578
|
+
appendMetrics(base, [{ at: new Date().toISOString(), kind: "hook", captureDrops: 1, captureDropReason: "summary_capacity" }]);
|
|
579
|
+
if (retireClaim(base, claim, false)) {
|
|
580
|
+
retired.add(victim.row.name); count--; bytes -= victim.row.size;
|
|
581
|
+
groups.set(victim.group, Math.max(0, (groups.get(victim.group) ?? 0) - 1));
|
|
582
|
+
}
|
|
583
|
+
} catch { /* Failed counting or changed ownership preserves the body. */ }
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
} finally {
|
|
588
|
+
for (const item of held) closeSync(item.file.fd);
|
|
589
|
+
try { cursor.save(retired.size ? 0 : cursor.start + Math.max(1, scanned)); } catch { /* A derived cursor never acknowledges capture. */ }
|
|
590
|
+
}
|
|
444
591
|
return retired;
|
|
445
592
|
}
|
|
446
593
|
|
|
@@ -532,32 +679,67 @@ function publishCapture(event: string, capture: NormalizedCaptureFields, options
|
|
|
532
679
|
if (Buffer.byteLength(bytes) > RECORD_BYTES) throw new Error("hook record bound");
|
|
533
680
|
const dir = directory(base), writing = join(dir, ".writing"); privateDirectory(dir); privateDirectory(writing);
|
|
534
681
|
let destination = dir, deferred = false, backlog = false;
|
|
682
|
+
const budget = maintenanceBudget(), incomingBytes = Buffer.byteLength(bytes), cwdHash = hash(cwd);
|
|
683
|
+
const withinShare = (rows: ReturnType<typeof entries>, maximum: number) => rows.filter(row => namePattern.exec(row.name)?.[2] === cwdHash).length < maximum / 4;
|
|
684
|
+
const name = `${v3 ? PREFIX_V3 : PREFIX}${cwdHash}-${scope ?? "0".repeat(64)}-${nonce}.json`;
|
|
685
|
+
let activeInventory: ReturnType<typeof entries> | undefined, reserveInventory: ReturnType<typeof entries> | undefined;
|
|
686
|
+
if (["Title", "Recap", "Compact"].includes(event)) {
|
|
687
|
+
// Transcript cursor recovery can repeat a deterministic publication. An already
|
|
688
|
+
// durable identical body needs no new slot, including when a drainer owns it.
|
|
689
|
+
for (const capacity of [false, true]) {
|
|
690
|
+
const inventory = entries(base, capacity);
|
|
691
|
+
if (capacity) reserveInventory = inventory; else activeInventory = inventory;
|
|
692
|
+
for (const row of inventory) {
|
|
693
|
+
if (!namePattern.test(row.name) || row.name !== name && !row.name.startsWith(name + ".draining-")) continue;
|
|
694
|
+
if (!budget.files || row.size > budget.bytes) break;
|
|
695
|
+
budget.files--; budget.bytes -= row.size;
|
|
696
|
+
let existing: OwnedFile | undefined;
|
|
697
|
+
try {
|
|
698
|
+
existing = openOwned(join(capacity ? join(dir, ".capacity") : dir, row.name));
|
|
699
|
+
if (existing.initial.size !== incomingBytes) continue;
|
|
700
|
+
const held = Buffer.alloc(incomingBytes);
|
|
701
|
+
if (readSync(existing.fd, held, 0, held.length, 0) === held.length && owns(existing) && held.equals(Buffer.from(bytes))) {
|
|
702
|
+
return { name, cwd, receipt: hash(bytes), backlog: true, ...(capacity ? { deferred: true as const } : {}) };
|
|
703
|
+
}
|
|
704
|
+
} catch { /* Changed or unavailable evidence is not acknowledgement. */ }
|
|
705
|
+
finally { if (existing) closeSync(existing.fd); }
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
}
|
|
535
709
|
if (event === "SessionStart") {
|
|
536
710
|
// ponytail: reserve-only admission scans at most 32 metadata entries, not O(1).
|
|
537
711
|
// Never inspect/retire the active tier on the brief deadline. Fixed-slot byte
|
|
538
712
|
// reservations with crash recovery are the upgrade if this bound is too costly.
|
|
539
713
|
destination = join(dir, ".capacity"); privateDirectory(destination);
|
|
540
|
-
const
|
|
541
|
-
|
|
714
|
+
const inventory = entries(base, true);
|
|
715
|
+
const retired = retireUnresolved(base, cwd, inventory, incomingBytes, true, true, budget);
|
|
716
|
+
const reserved = inventory.filter(row => !retired.has(row.name));
|
|
717
|
+
if (!withinShare(reserved, CAPACITY_FILES) || reserved.length >= CAPACITY_FILES || reserved.reduce((sum, row) => sum + row.size, 0) + Buffer.byteLength(bytes) > CAPACITY_BYTES) throw new HookSpoolFull();
|
|
542
718
|
deferred = true; backlog = reserved.length > 0;
|
|
543
719
|
} else {
|
|
544
|
-
const inventory = entries(base)
|
|
545
|
-
|
|
720
|
+
const inventory = activeInventory ?? entries(base);
|
|
721
|
+
if (summaryEvent(event) && (!withinShare(inventory, MAX_FILES) || inventory.length >= MAX_FILES || inventory.reduce((sum, row) => sum + row.size, 0) + incomingBytes > MAX_BYTES)) throw new HookSpoolFull();
|
|
722
|
+
let reserved = reserveInventory ?? entries(base, true);
|
|
723
|
+
// Split a nonempty two-tier backlog's inspection budget so neither tier starves.
|
|
724
|
+
const retired = retireUnresolved(base, cwd, inventory, incomingBytes, false, false, budget, reserved.length ? DRAIN_FILES / 2 : DRAIN_FILES);
|
|
546
725
|
const queued = inventory.filter(row => !retired.has(row.name));
|
|
547
|
-
|
|
548
|
-
//
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
726
|
+
const activeFits = withinShare(queued, MAX_FILES) && queued.length < MAX_FILES && queued.reduce((sum, row) => sum + row.size, 0) + incomingBytes <= MAX_BYTES;
|
|
727
|
+
// If active admission failed, reserve pressure gets the remaining budget first.
|
|
728
|
+
if (activeFits) {
|
|
729
|
+
const retiredCapacity = retireUnresolved(base, undefined, reserved, 0, false, true, budget);
|
|
730
|
+
reserved = reserved.filter(row => !retiredCapacity.has(row.name));
|
|
731
|
+
}
|
|
552
732
|
backlog = queued.length > 0 || reserved.length > 0;
|
|
553
|
-
if (
|
|
733
|
+
if (!activeFits) {
|
|
554
734
|
if (options.deferAtCapacity !== true) throw new HookSpoolFull();
|
|
555
735
|
destination = join(dir, ".capacity"); privateDirectory(destination);
|
|
556
|
-
|
|
736
|
+
const retiredReserve = retireUnresolved(base, cwd, reserved, incomingBytes, true, true, budget);
|
|
737
|
+
reserved = reserved.filter(row => !retiredReserve.has(row.name));
|
|
738
|
+
if (!withinShare(reserved, CAPACITY_FILES) || reserved.length >= CAPACITY_FILES || reserved.reduce((sum, row) => sum + row.size, 0) + Buffer.byteLength(bytes) > CAPACITY_BYTES) throw new HookSpoolFull();
|
|
557
739
|
deferred = true;
|
|
558
740
|
}
|
|
559
741
|
}
|
|
560
|
-
const
|
|
742
|
+
const temporary = join(writing, name);
|
|
561
743
|
try { writeFileSync(temporary, bytes, { mode: 0o600, flag: "wx" }); renameSync(temporary, join(destination, name)); }
|
|
562
744
|
finally { rmSync(temporary, { force: true }); }
|
|
563
745
|
return { name, cwd, receipt: hash(bytes), ...(deferred ? { deferred: true as const } : {}), ...(backlog ? { backlog: true as const } : {}) };
|
|
@@ -572,32 +754,40 @@ export function hookCapturePending(base: string, name: string): boolean {
|
|
|
572
754
|
} catch { return true; } // Unreadable evidence is pending, never an acknowledgement.
|
|
573
755
|
}
|
|
574
756
|
|
|
575
|
-
export async function drainHookCaptures(options: Options & { project: string; expectedScope?: string; current?: string; timeRemaining?: () => number },
|
|
757
|
+
export async function drainHookCaptures(options: Options & { project: string; expectedScope?: string; current?: string; timeRemaining?: () => number; replayBudget?: HookReplayBudget },
|
|
576
758
|
append: (input: CaptureInput, receipt: string) => CaptureAppend): Promise<{ captured: string[]; blocked: number }> {
|
|
759
|
+
const budget = options.replayBudget ?? hookReplayBudget();
|
|
760
|
+
if (!Number.isSafeInteger(budget.files) || budget.files < 0 || budget.files > HOOK_REPLAY_MAX_FILES
|
|
761
|
+
|| !Number.isSafeInteger(budget.bytes) || budget.bytes < 0 || budget.bytes > HOOK_REPLAY_MAX_BYTES
|
|
762
|
+
|| !(budget.seen instanceof Set) || budget.seen.size > HOOK_REPLAY_MAX_FILES) throw new Error("hook replay budget is invalid; create a fresh hookReplayBudget for this hook");
|
|
577
763
|
const cwd = realPath(options.cwd), base = canonicalPath(options.base), stamp = hookCaptureScope(cwd, base), captured: string[] = [];
|
|
578
764
|
if (belowFloor(base)) return { captured, blocked: 1 };
|
|
579
765
|
if (!stamp || options.expectedScope !== undefined && options.expectedScope !== stamp) return { captured, blocked: 1 };
|
|
580
766
|
// F4: the configured policy applies at drain too. `options.policy` carries it, so the anchored line below (a test diagnostic matches its exact text) stays as it was.
|
|
581
767
|
options = { ...options, policy: configuredPolicyOptions(base, cwd, options.policy) };
|
|
582
768
|
const policy = createStandalonePolicy(options.policy), dir = directory(base);
|
|
769
|
+
const legacyStamp = captureScope(cwd, base, false, undefined, true);
|
|
770
|
+
const accepts = (scope: string | null) => scope === stamp || legacyStamp !== undefined && scope === legacyStamp;
|
|
583
771
|
const matching = (rows: ReturnType<typeof entries>, capacity: boolean) => rows.filter(row => {
|
|
584
|
-
const m = namePattern.exec(row.name); return m && m[2] === hash(cwd) && m[3]
|
|
772
|
+
const m = namePattern.exec(row.name); return m && m[2] === hash(cwd) && accepts(m[3]!);
|
|
585
773
|
}).sort((a, b) => a.modified - b.modified || a.name.localeCompare(b.name)).map(row => ({ ...row, capacity }));
|
|
586
774
|
const rows = [...matching(entries(base, true), true), ...matching(entries(base), false)]
|
|
587
775
|
.sort((a, b) => a.modified - b.modified || a.name.localeCompare(b.name));
|
|
588
|
-
|
|
589
|
-
|
|
776
|
+
const identity = (row: typeof rows[number]) => `${namePattern.exec(row.name)![1]}:${row.dev}:${row.ino}`;
|
|
777
|
+
let remaining = budget.bytes, blocked = 0;
|
|
778
|
+
for (const row of rows.filter(row => !budget.seen.has(identity(row))).slice(0, budget.files)) {
|
|
590
779
|
if (options.timeRemaining && options.timeRemaining() < 8 || row.size <= RECORD_BYTES && row.size > remaining) break;
|
|
591
780
|
const match = namePattern.exec(row.name)!, name = match[1]!;
|
|
781
|
+
budget.files--; budget.seen.add(identity(row));
|
|
592
782
|
let file: OwnedFile | undefined;
|
|
593
783
|
try {
|
|
594
784
|
file = openOwned(join(row.capacity ? join(dir, ".capacity") : dir, row.name));
|
|
595
785
|
if (file.initial.size !== row.size || file.initial.mtimeMs !== row.modified) { blocked++; continue; }
|
|
596
786
|
const claim = claimOwned(base, row.name, file);
|
|
597
787
|
if (!claim) { blocked++; continue; }
|
|
598
|
-
if (claim.initial.size <= RECORD_BYTES) remaining -= claim.initial.size;
|
|
788
|
+
if (claim.initial.size <= RECORD_BYTES) { remaining -= claim.initial.size; budget.bytes -= claim.initial.size; }
|
|
599
789
|
const record = decoded(claim);
|
|
600
|
-
if (!record || record.value.cwd !== cwd || record.value.scope !==
|
|
790
|
+
if (!record || record.value.cwd !== cwd || record.value.scope !== match[3] || record.value.nonce !== match[4]) {
|
|
601
791
|
retireClaim(base, claim, false); blocked++; continue;
|
|
602
792
|
}
|
|
603
793
|
// #199: an unknown or newer schema stays claimed and pending (later passes reclaim it) until the TTL, then retires with a counted reason.
|
|
@@ -623,7 +813,7 @@ export async function drainHookCaptures(options: Options & { project: string; ex
|
|
|
623
813
|
...(c.tool === null ? {} : { tool: c.tool }), ...(c.status === null ? {} : { status: c.status }), excerpt: c.excerpt,
|
|
624
814
|
...(c.session === undefined ? {} : { session: c.session }), ...(c.subagent === undefined ? {} : { subagent: c.subagent }) };
|
|
625
815
|
if (!await policy.screen(value) || !await policy.screen(input)) { retireClaim(base, claim, true); blocked++; continue; }
|
|
626
|
-
if (!owns(claim) || hookCaptureScope(cwd, base) !== stamp) { blocked++; continue; }
|
|
816
|
+
if (!owns(claim) || hookCaptureScope(cwd, base) !== stamp || value.scope === legacyStamp && captureScope(cwd, base, false, undefined, true) !== legacyStamp) { blocked++; continue; }
|
|
627
817
|
append(input, hash(record.bytes));
|
|
628
818
|
if (owns(claim)) { rmSync(claim.file); captured.push(name); }
|
|
629
819
|
else blocked++;
|
|
@@ -632,3 +822,29 @@ export async function drainHookCaptures(options: Options & { project: string; ex
|
|
|
632
822
|
}
|
|
633
823
|
return { captured, blocked };
|
|
634
824
|
}
|
|
825
|
+
|
|
826
|
+
/**
|
|
827
|
+
* #217 PR2: the narrow internal surface the CLI-only replay (src/hook-spool-replay.ts) reuses, so replay keeps exactly the
|
|
828
|
+
* hook path's owned-file, decoder and claim rules. No format or ownership fork; the replay module is never imported by hooks.
|
|
829
|
+
*/
|
|
830
|
+
export const hookSpoolReplayInternals = {
|
|
831
|
+
namePattern, RECORD_BYTES, MAX_FILES,
|
|
832
|
+
openOwned: (file: string) => openOwned(file),
|
|
833
|
+
owns: (file: OwnedFile) => owns(file),
|
|
834
|
+
decoded: (file: OwnedFile) => decoded(file),
|
|
835
|
+
belowFloor: (base: string) => belowFloor(base),
|
|
836
|
+
/** Claim under a caller-chosen deterministic suffix (#217 decision B: hash(plan digest, publication)), never a random one. */
|
|
837
|
+
claimAs(file: OwnedFile, name: string, suffix: string): OwnedFile | undefined {
|
|
838
|
+
const match = namePattern.exec(name);
|
|
839
|
+
if (!match || !/^[a-f0-9]{24}$/.test(suffix) || !owns(file)) return undefined;
|
|
840
|
+
const claim = join(dirname(file.file), `${match[1]}.draining-${suffix}`);
|
|
841
|
+
renameSync(file.file, claim);
|
|
842
|
+
const owned = { ...file, file: claim };
|
|
843
|
+
return owns(owned) ? owned : undefined;
|
|
844
|
+
},
|
|
845
|
+
/** Remove exactly the owned claim, only while it is still the inspected inode. */
|
|
846
|
+
retireOwned(file: OwnedFile): boolean { if (!owns(file)) return false; rmSync(file.file); return true; },
|
|
847
|
+
};
|
|
848
|
+
export type HookSpoolOwnedFile = OwnedFile;
|
|
849
|
+
/** The 0.19.0-and-older device-bound scope for this cwd, computable only for the current boot (drain accepts it too). */
|
|
850
|
+
export function hookLegacyCaptureScope(cwd: string, base: string): string | undefined { return captureScope(cwd, base, false, undefined, true); }
|
package/src/hook-transcript.ts
CHANGED
|
@@ -6,7 +6,7 @@ import { join } from "node:path";
|
|
|
6
6
|
import { canonicalPath, realPath } from "./memory/platform.ts";
|
|
7
7
|
import { configuredPolicyOptions } from "./memory/config.ts";
|
|
8
8
|
import { createStandalonePolicy, type StandalonePolicyOptions } from "./memory/policy.ts";
|
|
9
|
-
import { HookSpoolFull, stageTranscriptCapture } from "./hook-spool.ts";
|
|
9
|
+
import { HookSpoolFull, stageTranscriptCapture, type PendingHookCapture } from "./hook-spool.ts";
|
|
10
10
|
import { appendMetrics } from "./memory/metrics.ts";
|
|
11
11
|
import type { HookTranscriptProgress } from "./hook-contract.ts";
|
|
12
12
|
|
|
@@ -14,7 +14,13 @@ export const HOOK_TRANSCRIPT_BYTES = 256 * 1024;
|
|
|
14
14
|
const hash = (value: string) => createHash("sha256").update(value).digest("hex");
|
|
15
15
|
type SummaryEvent = "Title" | "Recap" | "Compact";
|
|
16
16
|
interface Cursor { offset: number; dev: string; ino: string; size: number; title: string; at: string; readStart: number; readAt: string; skipping: number }
|
|
17
|
-
|
|
17
|
+
/** `deadline` is the hard end of the scan. With `workBudgetMs`, the line loop gets that budget measured from the end of setup
|
|
18
|
+
* (policy, cursor database, read), never past `deadline` (#226 task 110: on a slow runner setup ate a fixed window). */
|
|
19
|
+
interface Options { cwd: string; base: string; deadline: number; workBudgetMs?: number; policy?: StandalonePolicyOptions;
|
|
20
|
+
/** Worker-local replay opportunities; no text crosses these callbacks. */
|
|
21
|
+
onPublication?: (pending: PendingHookCapture) => Promise<void>;
|
|
22
|
+
onBackpressure?: () => Promise<boolean>;
|
|
23
|
+
}
|
|
18
24
|
const object = (value: unknown): Record<string, unknown> | undefined => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
|
|
19
25
|
const timestamp = (value: unknown) => typeof value === "string" && Number.isFinite(Date.parse(value)) ? new Date(value).toISOString() : undefined;
|
|
20
26
|
|
|
@@ -127,7 +133,10 @@ export async function scanHookTranscriptTail(input: unknown, options: Options):
|
|
|
127
133
|
const after = fstatSync(fd), live = lstatSync(value.transcript_path);
|
|
128
134
|
if (length !== bytes.length || after.size < stat.size || live.dev !== stat.dev || live.ino !== stat.ino || realPath(value.transcript_path) !== transcript) return;
|
|
129
135
|
let start = 0, stopReason: HookTranscriptProgress["stopReason"];
|
|
130
|
-
|
|
136
|
+
const workStart = performance.now();
|
|
137
|
+
// The caller's deadline is read live (a caller may move it); the work budget only adds an earlier end after setup.
|
|
138
|
+
const due = () => performance.now() >= options.deadline || options.workBudgetMs !== undefined && performance.now() - workStart >= options.workBudgetMs;
|
|
139
|
+
while (start < length && !due()) {
|
|
131
140
|
const newline = bytes.indexOf(10, start);
|
|
132
141
|
if (cursor.skipping) {
|
|
133
142
|
start = newline < 0 ? length : newline + 1;
|
|
@@ -162,9 +171,16 @@ export async function scanHookTranscriptTail(input: unknown, options: Options):
|
|
|
162
171
|
at = (db.query("SELECT at FROM fallback WHERE digest=?").get(digest) as { at: string }).at;
|
|
163
172
|
if (!timestamp(at)) throw new Error("transcript cursor unavailable");
|
|
164
173
|
}
|
|
165
|
-
const
|
|
166
|
-
...(summary
|
|
174
|
+
const publish = () => stageTranscriptCapture({ agent: "claude", event: summary!.event, excerpt: summary!.text, at, session,
|
|
175
|
+
...(summary!.subagent || typeof value.agent_id === "string" && value.agent_id.length > 0 ? { subagent: true } : {}) }, { cwd, base, policy: options.policy, deferAtCapacity: true }, summary!.event === "Title" ? `title:${cursor.offset + start}` : undefined);
|
|
176
|
+
let staged: PendingHookCapture | undefined;
|
|
177
|
+
try { staged = await publish(); }
|
|
178
|
+
catch (error) {
|
|
179
|
+
if (!(error instanceof HookSpoolFull) || due() || !options.onBackpressure || !await options.onBackpressure() || due()) throw error;
|
|
180
|
+
staged = await publish(); // Same line/receipt, no second read or cursor advance on backpressure.
|
|
181
|
+
}
|
|
167
182
|
if (staged && title) cursor.title = title;
|
|
183
|
+
if (staged) await options.onPublication?.(staged);
|
|
168
184
|
} catch (error) {
|
|
169
185
|
// Advance only through durable/refused rows; a full spool retries this line
|
|
170
186
|
// after the ordinary hook worker drains earlier publications.
|
|
@@ -177,7 +193,7 @@ export async function scanHookTranscriptTail(input: unknown, options: Options):
|
|
|
177
193
|
const at = transcriptTimestamp(line); if (at) cursor.at = at;
|
|
178
194
|
start = newline + 1;
|
|
179
195
|
}
|
|
180
|
-
if (!stopReason && start < length &&
|
|
196
|
+
if (!stopReason && start < length && due()) stopReason = "deadline";
|
|
181
197
|
if (!stopReason && cursor.offset + start < stat.size && length === HOOK_TRANSCRIPT_BYTES) stopReason = "cap";
|
|
182
198
|
assertFormatFloor(base, "hook-spool");
|
|
183
199
|
db.query("UPDATE cursor SET offset=?,dev=?,ino=?,size=?,title=?,at=?,skipping=? WHERE id=1").run(cursor.offset + start, String(stat.dev), String(stat.ino), stat.size, cursor.title, cursor.at, cursor.skipping);
|