@staix/agent-mem 0.9.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +17 -5
- package/package.json +1 -1
- package/src/cli.ts +22 -9
- package/src/commands-table.ts +5 -4
- package/src/index.ts +1 -0
- package/src/ingest.ts +98 -7
- package/src/mcp-memory.ts +47 -35
- package/src/mcp.ts +1 -1
- package/src/memory/candidates.ts +44 -1
- package/src/memory/capture.ts +24 -6
- package/src/memory/commands.ts +23 -4
- package/src/memory/consolidate.ts +9 -6
- package/src/memory/consolidation-clusters.ts +12 -7
- package/src/memory/eval-fixture.ts +7 -3
- package/src/memory/eval.ts +17 -8
- package/src/memory/external-import.ts +8 -1
- package/src/memory/platform.ts +8 -8
- package/src/memory/policy.ts +62 -3
- package/src/memory/project-import.ts +31 -14
- package/src/memory/search-fts.ts +68 -3
- package/src/memory/search-score.ts +66 -0
- package/src/memory/store.ts +40 -13
- package/src/session-brief-worker.ts +5 -1
- package/src/setup-agy.ts +9 -0
- package/src/setup-jcode.ts +7 -0
- package/src/setup-json.ts +151 -0
- package/src/setup-owned.ts +40 -19
- package/src/setup.ts +147 -34
package/src/memory/candidates.ts
CHANGED
|
@@ -13,6 +13,15 @@ export const CANDIDATE_DB_BUSY_TIMEOUT_MS = 3000;
|
|
|
13
13
|
/** First line of every note written by accepting a consolidation candidate; `consolidationClusters` reads it to avoid proposing summaries of its own summaries. */
|
|
14
14
|
export const ACCEPTED_CONSOLIDATION_PREFIX = "Accepted consolidation candidate: ";
|
|
15
15
|
export const CANDIDATE_RETENTION = { count: 500, ageMs: 30 * 86400_000 } as const;
|
|
16
|
+
export const CONSOLIDATION_REVIEW_LIMIT = 10_000;
|
|
17
|
+
export class ConsolidationReviewLimit extends Error {
|
|
18
|
+
readonly code = "consolidation_review_limit";
|
|
19
|
+
readonly limit = CONSOLIDATION_REVIEW_LIMIT;
|
|
20
|
+
constructor(readonly project: string, readonly count: number) {
|
|
21
|
+
super(`consolidation review cap reached: project ${project}, count ${count}, limit ${CONSOLIDATION_REVIEW_LIMIT}; no reviewed identities evicted`);
|
|
22
|
+
this.name = "ConsolidationReviewLimit";
|
|
23
|
+
}
|
|
24
|
+
}
|
|
16
25
|
export const COUNTER_RETENTION = { count: 10_000, ageMs: 90 * 86400_000 } as const;
|
|
17
26
|
export type CandidateVerdict = "public" | "private" | "pii" | "unknown";
|
|
18
27
|
export interface CandidateTarget { id: string; revision: string }
|
|
@@ -27,6 +36,7 @@ export type CandidateInput = { project: string } & (
|
|
|
27
36
|
);
|
|
28
37
|
export type CandidateStatus = "pending" | "accepting" | "accepted" | "rejected";
|
|
29
38
|
export interface Candidate { id: string; createdAt: string; status: CandidateStatus; input: CandidateInput; acceptedNoteId: string | null }
|
|
39
|
+
export interface ConsolidationReview { status: Exclude<CandidateStatus, "pending">; acceptedNoteId: string | null }
|
|
30
40
|
export interface CandidateFilter { project: string; status?: CandidateStatus | "all"; kind?: CandidateInput["kind"]; limit?: number }
|
|
31
41
|
export interface CounterKey { entryId: string; project: string; task?: string | null; turn?: string | null; session?: string | null; deliveryId: string }
|
|
32
42
|
export interface CounterValues { rendered?: number | null; delivered?: number | null; cited?: number | null }
|
|
@@ -146,13 +156,27 @@ export class CandidateStore {
|
|
|
146
156
|
// F7 / AC4: when a candidate was first shown to the person. In place and nullable: an older database gains it here, and a reader without it treats every row as unshown.
|
|
147
157
|
// Several processes open this store at once (every `amem mcp` builds it at startup), so the check and the ALTER are one immediate transaction: the loser waits, then sees the column.
|
|
148
158
|
if (!hasShownColumn(this.db)) this.db.transaction(() => { if (!hasShownColumn(this.db)) this.db.run("ALTER TABLE candidates ADD COLUMN shown_at INTEGER"); }).immediate();
|
|
159
|
+
this.db.run("CREATE TABLE IF NOT EXISTS consolidation_reviews (id TEXT PRIMARY KEY, project TEXT NOT NULL, status TEXT NOT NULL, note_id TEXT)");
|
|
149
160
|
this.db.run("CREATE TABLE IF NOT EXISTS counters (id TEXT PRIMARY KEY, project TEXT NOT NULL, payload TEXT NOT NULL, at INTEGER NOT NULL, rendered INTEGER, delivered INTEGER, cited INTEGER)");
|
|
150
161
|
this.prune();
|
|
151
162
|
}
|
|
152
163
|
close(): void { this.db.close(); }
|
|
164
|
+
/** Keep only identity and disposition after payload retention, never source text or target lists. */
|
|
165
|
+
private retainConsolidationReviews(): void {
|
|
166
|
+
const overflow = this.db.query(`SELECT project, count(*) AS n FROM consolidation_reviews GROUP BY project HAVING count(*) > ?
|
|
167
|
+
UNION ALL SELECT c.project, count(*) + (SELECT count(*) FROM consolidation_reviews WHERE project = c.project) AS n FROM candidates c LEFT JOIN consolidation_reviews r ON r.id = c.id
|
|
168
|
+
WHERE c.kind = 'consolidation' AND c.status IN ('accepting', 'accepted', 'rejected') AND r.id IS NULL
|
|
169
|
+
GROUP BY c.project HAVING count(*) + (SELECT count(*) FROM consolidation_reviews WHERE project = c.project) > ? LIMIT 1`)
|
|
170
|
+
.get(CONSOLIDATION_REVIEW_LIMIT, CONSOLIDATION_REVIEW_LIMIT) as { project: string; n: number } | null;
|
|
171
|
+
if (overflow) throw new ConsolidationReviewLimit(overflow.project, overflow.n);
|
|
172
|
+
this.db.run(`INSERT INTO consolidation_reviews (id, project, status, note_id)
|
|
173
|
+
SELECT id, project, status, note_id FROM candidates WHERE kind = 'consolidation' AND status IN ('accepting', 'accepted', 'rejected')
|
|
174
|
+
ON CONFLICT(id) DO UPDATE SET status = excluded.status, note_id = excluded.note_id`);
|
|
175
|
+
}
|
|
153
176
|
private prune(): void {
|
|
154
177
|
this.db.transaction(() => {
|
|
155
178
|
const now = this.now();
|
|
179
|
+
this.retainConsolidationReviews();
|
|
156
180
|
// Candidates (AC4, no candidate expires unshown): by age only a decided one, or a pending one that was shown and has had its grace period, goes. By count three tiers decide who is
|
|
157
181
|
// kept: unshown pending first, then shown pending, then decided, so a cap eviction takes decided rows before a shown one (which is owed its grace) and shown ones before an unseen one.
|
|
158
182
|
this.db.query("DELETE FROM candidates WHERE at <= ? AND (status IN ('accepted', 'rejected') OR (shown_at IS NOT NULL AND shown_at <= ?))").run(now - CANDIDATE_RETENTION.ageMs, now - CANDIDATE_SHOWN_GRACE_MS);
|
|
@@ -174,10 +198,19 @@ export class CandidateStore {
|
|
|
174
198
|
// Prune, count, insert and prune again are ONE immediate transaction: concurrent proposers (several agents) cannot all pass the count and then let the cap evict an unseen candidate.
|
|
175
199
|
return this.db.transaction((): string => {
|
|
176
200
|
this.prune();
|
|
201
|
+
if (input.kind === "consolidation" && this.reviewedConsolidation(id, input.project)) return id;
|
|
177
202
|
// AC4: a new candidate never pushes an unshown one out. With the cap's worth waiting the person has to review first, and the refusal says so (a named error an MCP tool can map).
|
|
178
203
|
const waiting = (this.db.query("SELECT COUNT(*) AS n FROM candidates WHERE project = ? AND status IN ('pending', 'accepting') AND shown_at IS NULL").get(input.project) as { n: number }).n;
|
|
179
204
|
const known = this.db.query("SELECT 1 FROM candidates WHERE id = ?").get(id);
|
|
180
205
|
if (!known && waiting >= CANDIDATE_RETENTION.count) throw new CandidateQueueFull(waiting);
|
|
206
|
+
if (!known && input.kind === "consolidation") {
|
|
207
|
+
const allocated = this.db.query(`SELECT (SELECT count(*) FROM consolidation_reviews WHERE project = ?) +
|
|
208
|
+
(SELECT count(*) FROM candidates c WHERE project = ? AND kind = 'consolidation' AND status IN ('pending', 'accepting')
|
|
209
|
+
AND NOT EXISTS (SELECT 1 FROM consolidation_reviews r WHERE r.id = c.id)) AS n`).get(input.project, input.project) as { n: number };
|
|
210
|
+
// ponytail: 10,000 lasting review identities per project; refuse new proposals instead of forgetting a person's decision.
|
|
211
|
+
// Upgrade with an explicit person-audited receipt archive before increasing this bounded sidecar capacity.
|
|
212
|
+
if (allocated.n >= CONSOLIDATION_REVIEW_LIMIT) throw new ConsolidationReviewLimit(input.project, allocated.n);
|
|
213
|
+
}
|
|
181
214
|
this.db.query("INSERT OR IGNORE INTO candidates (id, project, kind, payload, at, status) VALUES (?, ?, ?, ?, ?, 'pending')").run(id, input.project, input.kind, payload, this.now());
|
|
182
215
|
this.prune();
|
|
183
216
|
return id;
|
|
@@ -189,6 +222,11 @@ export class CandidateStore {
|
|
|
189
222
|
const row = this.db.query("SELECT * FROM candidates WHERE id = ? AND project IN (SELECT value FROM json_each(?))").get(id, this.scope(project)) as Parameters<CandidateStore["decode"]>[0] | null;
|
|
190
223
|
return row ? this.decode(row) : undefined;
|
|
191
224
|
}
|
|
225
|
+
reviewedConsolidation(id: string, project: string): ConsolidationReview | undefined {
|
|
226
|
+
if (!candidateId(id) || !text(project, 400)) throw new Error("invalid candidate identity");
|
|
227
|
+
const row = this.db.query("SELECT status, note_id FROM consolidation_reviews WHERE id = ? AND project IN (SELECT value FROM json_each(?))").get(id, this.scope(project)) as { status: ConsolidationReview["status"]; note_id: string | null } | null;
|
|
228
|
+
return row ? { status: row.status, acceptedNoteId: row.note_id } : undefined;
|
|
229
|
+
}
|
|
192
230
|
list(filter: CandidateFilter): Candidate[] {
|
|
193
231
|
if (!text(filter.project, 400) || filter.status !== undefined && !["pending", "accepting", "accepted", "rejected", "all"].includes(filter.status) || filter.kind !== undefined && !["correction", "fail", "decay", "memory", "promotion", "consolidation", "model-answer"].includes(filter.kind)) throw new Error("invalid candidate filter");
|
|
194
232
|
const limit = filter.limit ?? CANDIDATE_RETENTION.count;
|
|
@@ -221,6 +259,7 @@ export class CandidateStore {
|
|
|
221
259
|
if (!candidate) throw new Error("candidate unavailable");
|
|
222
260
|
if (candidate.status === "accepted" || candidate.status === "accepting") throw new Error("candidate acceptance already started; inspect its ordinary note before any further action");
|
|
223
261
|
this.db.query("UPDATE candidates SET status = 'rejected' WHERE id = ? AND project IN (SELECT value FROM json_each(?)) AND status = 'pending'").run(id, this.scope(context.project));
|
|
262
|
+
if (candidate.input.kind === "consolidation") this.retainConsolidationReviews();
|
|
224
263
|
}).immediate();
|
|
225
264
|
}
|
|
226
265
|
accept(id: string, context: CandidateAcceptance): CandidateAcceptanceResult {
|
|
@@ -321,7 +360,10 @@ export class CandidateStore {
|
|
|
321
360
|
// report every confirmed effect; recovery requires person inspection before any replay.
|
|
322
361
|
throw Error(`consolidation interrupted: inspect note ${noteId}; superseded ${applied.join(", ") || "none"}; remaining or uncertain ${data.targets.filter(target => !applied.includes(target.id)).map(target => target.id).join(", ")}`);
|
|
323
362
|
}
|
|
324
|
-
this.db.
|
|
363
|
+
this.db.transaction(() => {
|
|
364
|
+
this.db.query("UPDATE candidates SET status = 'accepted', note_id = ? WHERE id = ? AND project IN (SELECT value FROM json_each(?))").run(noteId, candidate.id, this.scope(context.project));
|
|
365
|
+
this.db.query("UPDATE consolidation_reviews SET status = 'accepted', note_id = ? WHERE id = ? AND project IN (SELECT value FROM json_each(?))").run(noteId, candidate.id, this.scope(context.project));
|
|
366
|
+
}).immediate();
|
|
325
367
|
return { candidateId: candidate.id, noteId, alreadyAccepted: false };
|
|
326
368
|
});
|
|
327
369
|
});
|
|
@@ -379,6 +421,7 @@ export class CandidateStore {
|
|
|
379
421
|
if (current.status !== "pending") throw Error("candidate is rejected or acceptance was interrupted; no automatic replay");
|
|
380
422
|
before?.();
|
|
381
423
|
this.db.query("UPDATE candidates SET status = 'accepting' WHERE id = ? AND project IN (SELECT value FROM json_each(?))").run(id, this.scope(project));
|
|
424
|
+
if (current.input.kind === "consolidation") this.retainConsolidationReviews();
|
|
382
425
|
return undefined;
|
|
383
426
|
}).immediate();
|
|
384
427
|
}
|
package/src/memory/capture.ts
CHANGED
|
@@ -619,17 +619,15 @@ export class CaptureStore {
|
|
|
619
619
|
/** Exact-project, bounded body pages; MCP screens before filling its public candidate window. */
|
|
620
620
|
readPage(window: ReadWindow & { query?: string; match?: "substring" }): { rows: CaptureRow[]; next?: ReadRef; engine?: "fts5" | "substring" } {
|
|
621
621
|
this.checkReadBinding();
|
|
622
|
-
const { columns, where, bind } = readWindowSql(window, "id");
|
|
622
|
+
const { columns, where, bind } = readWindowSql(window, "id", "integer");
|
|
623
623
|
if (this.recentReadIds) {
|
|
624
624
|
if (window.query !== undefined) throw new Error("recent capture brief does not support query indexes");
|
|
625
625
|
where.push("id IN (SELECT value FROM json_each(?))"); bind.push(JSON.stringify(this.recentReadIds));
|
|
626
626
|
}
|
|
627
627
|
const engine = window.query === undefined ? undefined : window.match !== "substring" && this.ensureMcpFts() ? "fts5" : "substring";
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
}
|
|
632
|
-
const rows = this.db.query(`SELECT *, ${columns} FROM rows WHERE ${where.join(" AND ")} ORDER BY candidate_position, candidate_id LIMIT ?`)
|
|
628
|
+
const plan = captureReadPlan(window, engine, where, bind);
|
|
629
|
+
if (!plan) return { rows: [], engine };
|
|
630
|
+
const rows = this.db.query(`SELECT rows.*, ${columns} FROM ${plan.from} WHERE ${where.join(" AND ")} ORDER BY ${plan.order} LIMIT ?`)
|
|
633
631
|
.all(...bind, window.limit) as (RawRow & { candidate_id: string; candidate_position: number })[];
|
|
634
632
|
return { rows: rows.map(toRow), ...(engine ? { engine } : {}), ...(rows.length ? { next: readRef(rows[rows.length - 1]!) } : {}) };
|
|
635
633
|
}
|
|
@@ -644,6 +642,26 @@ export class CaptureStore {
|
|
|
644
642
|
close(): void { this.db.close(); }
|
|
645
643
|
}
|
|
646
644
|
|
|
645
|
+
/** Keep chronological readers intact while exhaustive search consumes FTS's rowid range directly. */
|
|
646
|
+
function captureReadPlan(window: ReadWindow & { query?: string; match?: "substring" }, engine: "fts5" | "substring" | undefined,
|
|
647
|
+
where: string[], bind: (string | number)[]): { from: string; order: string } | undefined {
|
|
648
|
+
let from = "rows", order = "candidate_position, candidate_id";
|
|
649
|
+
if (window.query !== undefined && engine === "fts5") {
|
|
650
|
+
const match = ftsTerms(window.query); if (!match) return;
|
|
651
|
+
if (window.order === "id") {
|
|
652
|
+
from = "mcp_fts JOIN rows ON rows.id = mcp_fts.rowid"; order = "mcp_fts.rowid";
|
|
653
|
+
where.push("mcp_fts MATCH ?"); bind.push(match);
|
|
654
|
+
if (window.after) {
|
|
655
|
+
const cursor = where.indexOf("id > ?");
|
|
656
|
+
if (cursor < 0) throw Error("missing capture identity cursor");
|
|
657
|
+
// Qualify the existing predicate so FTS receives the rowid range, without a second condition/bind.
|
|
658
|
+
where[cursor] = "mcp_fts.rowid > ?";
|
|
659
|
+
}
|
|
660
|
+
} else { where.push("id IN (SELECT rowid FROM mcp_fts WHERE mcp_fts MATCH ?)"); bind.push(match); }
|
|
661
|
+
}
|
|
662
|
+
return { from, order };
|
|
663
|
+
}
|
|
664
|
+
|
|
647
665
|
/** One shard's capture totals for the catalog and the capacity advisor, without opening a writer. */
|
|
648
666
|
export function captureShardStats(dir: string): { rows: number; bytes: number; lastAt: string | null } {
|
|
649
667
|
if (!existsSync(join(dir, "capture.db"))) return { rows: 0, bytes: 0, lastAt: null };
|
package/src/memory/commands.ts
CHANGED
|
@@ -15,6 +15,10 @@ export const AMEM_NAMES: CommandNames = { memory: "amem", goal: "amem goal" };
|
|
|
15
15
|
export interface MemoryCommandOptions {
|
|
16
16
|
/** Host-authenticated person intent. Never accept this from model arguments. Required for retained history/as-of. */
|
|
17
17
|
person?: boolean;
|
|
18
|
+
/** Standalone person CLI opt-in. Hosts omit this to retain their screened substring recall. */
|
|
19
|
+
rankedSearch?: boolean;
|
|
20
|
+
/** Standalone person CLI opt-in; requires rankedSearch and person intent. */
|
|
21
|
+
explainSearch?: boolean;
|
|
18
22
|
/** Defaults to amem's own names. */
|
|
19
23
|
names?: CommandNames;
|
|
20
24
|
/** Startup-bound canonical project supplied by a host; no model-facing selector. */
|
|
@@ -67,6 +71,11 @@ export async function rawMemoryCommand(args: string[], cwd: string, opts: Memory
|
|
|
67
71
|
const sub = rest[0] && !rest[0].startsWith("--") ? rest.shift()! : "list";
|
|
68
72
|
const names = opts.names ?? AMEM_NAMES, usage = memoryUsage(names.memory);
|
|
69
73
|
if (args.some(arg => arg === "--history" || arg === "--as-of" || arg.startsWith("--history=") || arg.startsWith("--as-of=")) && opts.person !== true) fail("historical recall requires authenticated person intent from the host");
|
|
74
|
+
if (opts.rankedSearch && opts.person !== true) fail("ranked CLI search requires person intent");
|
|
75
|
+
if (opts.explainSearch && (opts.person !== true || opts.rankedSearch !== true)) fail("score explanation requires ranked search and person intent");
|
|
76
|
+
if (opts.explainSearch && sub !== "search") fail("score explanation applies only to search");
|
|
77
|
+
if (args.includes("--explain") && (opts.person !== true || opts.rankedSearch !== true || opts.explainSearch !== true)) fail("score explanation requires ranked search and person intent");
|
|
78
|
+
for (let index = rest.indexOf("--explain"); index >= 0; index = rest.indexOf("--explain")) rest.splice(index, 1);
|
|
70
79
|
const store = opts.store ?? opts.open();
|
|
71
80
|
const actor = "user";
|
|
72
81
|
const project = opts.project ?? memoryProjectKey(cwd);
|
|
@@ -89,8 +98,16 @@ export async function rawMemoryCommand(args: string[], cwd: string, opts: Memory
|
|
|
89
98
|
if (kind && !KINDS.includes(kind as Kind)) fail(`--kind takes ${KINDS.join(", ")}`);
|
|
90
99
|
if (limit && !(Number(limit) > 0)) fail("--limit takes a positive number");
|
|
91
100
|
const q = sub === "search" ? rest.join(" ") : undefined;
|
|
92
|
-
const
|
|
93
|
-
|
|
101
|
+
const ranked = opts.rankedSearch && q?.trim() && !history && asOf === undefined;
|
|
102
|
+
if (opts.explainSearch && (!ranked || !q?.trim())) fail("--explain requires a non-empty current-project ranked search without --history or --as-of");
|
|
103
|
+
const state = recallRows(store, project, { person: opts.person, history, asOf, query: ranked ? undefined : q, limit: Number.MAX_SAFE_INTEGER });
|
|
104
|
+
// Retained history has its own snapshot; the current index cannot rank it. Empty queries keep list semantics.
|
|
105
|
+
// ponytail: fetch up to the current curated-view count before display caps, so metadata/readable exclusions consume no slots.
|
|
106
|
+
// Upgrade to paged ranked views if this full-pool person read reaches the R3 ceiling.
|
|
107
|
+
const rankedResult = ranked ? store.withDeferredPromotions(() => store.search(q!, { project, includeCapture: false, limit: Math.max(1, state.views.length), ...(opts.explainSearch ? { explain: true } : {}) })) : undefined;
|
|
108
|
+
const scoreDetails = opts.explainSearch ? new Map(rankedResult?.hits.flatMap(hit => hit.view && hit.scoreDetails ? [[hit.view.id, hit.scoreDetails] as const] : []) ?? []) : undefined;
|
|
109
|
+
const candidates = rankedResult ? rankedResult.hits.flatMap(hit => hit.view ? [hit.view] : []) : state.views;
|
|
110
|
+
const all = candidates.filter(view => (!tier || view.tier === tier) && (!kind || view.kind === kind) && (!peer || view.peer === peer) && (!machine || view.machine === machine) && (!since || Date.parse(view.at) >= Date.parse(since)));
|
|
94
111
|
const readable = opts.readable(all), rows = readable.slice(0, limit ? Number(limit) : 50);
|
|
95
112
|
ready(rows);
|
|
96
113
|
if (history || asOf !== undefined) {
|
|
@@ -99,10 +116,12 @@ export async function rawMemoryCommand(args: string[], cwd: string, opts: Memory
|
|
|
99
116
|
const result = { at: state.at, entries: rows.map(decorated), forgotten, relations: state.relations.filter(edge => publicIds.has(edge.from) && publicIds.has(edge.to)), incomplete: emitted.incomplete || readable.length > rows.length || forgotten.length < allForgotten.length, notices: emitted.notices };
|
|
100
117
|
return json ? JSON.stringify(result, null, 2) : [`retained memory as of ${state.at}`, ...result.notices, ...rows.map(view => `${stale(view) ? "(stale?) " : ""}${one(view)}${view.expiresAt && Date.parse(view.expiresAt) <= Date.parse(state.at) ? ` (expired at ${view.expiresAt})` : ""}`), ...forgotten.map(row => `forgotten ${row.id} at ${row.at} (no retained text shown)`), ...(result.incomplete ? ["(retained state is incomplete)"] : [])].join("\n");
|
|
101
118
|
}
|
|
102
|
-
if (json) return JSON.stringify(rows, null, 2);
|
|
119
|
+
if (json) return JSON.stringify(opts.explainSearch ? rows.map(view => ({ ...view, ...(scoreDetails?.has(view.id) ? { scoreDetails: scoreDetails.get(view.id) } : {}) })) : rows, null, 2);
|
|
103
120
|
const counted = opts.readable(store.withDeferredPromotions(() => store.list({ project, limit: Number.MAX_SAFE_INTEGER, includeSuperseded: true })));
|
|
104
121
|
const withheld = all.length - readable.length;
|
|
105
|
-
return [`memory for ${project} on machine ${store.machine.label}: ${counted.filter(view => view.tier === "long").length} long-term, ${counted.filter(view => view.tier === "short").length} short-term`, ...(rows.length ?
|
|
122
|
+
return [`memory for ${project} on machine ${store.machine.label}: ${counted.filter(view => view.tier === "long").length} long-term, ${counted.filter(view => view.tier === "short").length} short-term`, ...(rows.length ? opts.explainSearch
|
|
123
|
+
? rows.flatMap(view => [`${stale(view) ? "(stale?) " : ""}${one(view)}`, ...(scoreDetails?.has(view.id) ? [`score: ${JSON.stringify(scoreDetails.get(view.id))}`] : [])])
|
|
124
|
+
: rows.map(view => `${stale(view) ? "(stale?) " : ""}${one(view)}`) : ["(nothing)"]),
|
|
106
125
|
...(withheld ? [`(${withheld} withheld: they match a PII pattern; the dashboard shows them)`] : [])].join("\n");
|
|
107
126
|
}
|
|
108
127
|
case "show": {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CandidateStore, candidateInputId, candidateTargetRevision, type CandidateInput } from "./candidates.ts";
|
|
1
|
+
import { CandidateStore, ConsolidationReviewLimit, candidateInputId, candidateTargetRevision, type CandidateInput } from "./candidates.ts";
|
|
2
2
|
import { consolidationClusters } from "./consolidation-clusters.ts";
|
|
3
3
|
import { memoryPolicySnapshot, type StandalonePolicy } from "./policy.ts";
|
|
4
4
|
import { recallViewIsLive } from "./recall.ts";
|
|
@@ -20,7 +20,10 @@ export interface ConsolidationReport {
|
|
|
20
20
|
model?: ModelConsolidationReport;
|
|
21
21
|
}
|
|
22
22
|
export class ConsolidationProposalError extends Error {
|
|
23
|
-
constructor(readonly proposed: string[]
|
|
23
|
+
constructor(readonly proposed: string[], readonly refusal?: ConsolidationReviewLimit) {
|
|
24
|
+
super(`consolidation proposal interrupted; candidates already queued: ${proposed.join(", ") || "none"}; no curated notes changed${refusal ? `; ${refusal.message}` : ""}`);
|
|
25
|
+
this.name = "ConsolidationProposalError";
|
|
26
|
+
}
|
|
24
27
|
}
|
|
25
28
|
const modelAnswer = (row: View) => row.peer === "amem-model" || row.text.startsWith("Model answer:");
|
|
26
29
|
/** On-demand candidate writes only. Every source is screened once under one policy and source revision. */
|
|
@@ -67,12 +70,12 @@ export async function proposeConsolidation(store: MemoryStore, candidates: Candi
|
|
|
67
70
|
// put() is INSERT OR IGNORE: a candidate the person already accepted or rejected keeps its status, so it is neither written again nor reported as queued.
|
|
68
71
|
for (const input of prepared) {
|
|
69
72
|
check();
|
|
70
|
-
const known = candidates.get(
|
|
71
|
-
if (known && known.status !== "pending") { report.alreadyReviewed++; continue; }
|
|
72
|
-
|
|
73
|
+
const id = candidateInputId(input), known = candidates.get(id, project);
|
|
74
|
+
if (known && known.status !== "pending" || candidates.reviewedConsolidation(id, project)) { report.alreadyReviewed++; continue; }
|
|
75
|
+
candidates.put(input, "public");
|
|
73
76
|
report.candidates.push({ id, action: input.action, sources: input.targets.map(target => `mem:${target.id}`) });
|
|
74
77
|
}
|
|
75
78
|
if (options.model !== undefined) report.model = await proposeModelConsolidations(store, candidates, synced, options.model, { project, actor: options.actor, check, policy });
|
|
76
|
-
} catch { throw new ConsolidationProposalError(report.candidates.map(row => row.id)); }
|
|
79
|
+
} catch (error) { throw new ConsolidationProposalError(report.candidates.map(row => row.id), error instanceof ConsolidationReviewLimit ? error : undefined); }
|
|
77
80
|
return report;
|
|
78
81
|
}
|
|
@@ -8,9 +8,14 @@ export interface ConsolidationCluster { action: ConsolidationAction; sources: Vi
|
|
|
8
8
|
export const CONSOLIDATION_SOURCE_LIMIT = 256, CONSOLIDATION_PROPOSAL_LIMIT = 32;
|
|
9
9
|
const words = (row: View) => new Set((`${row.title ?? ""} ${row.text}`.toLowerCase().match(/[\p{L}\p{N}_]{3,}/gu) ?? []).filter(word => !["the", "and", "for", "with", "from", "this", "that", "approach", "symptom", "cause", "evidence", "retry", "when"].includes(word)).slice(0, 128));
|
|
10
10
|
/** A note written by accepting a consolidation candidate. Proposing over one would be a summary of summaries. */
|
|
11
|
-
export
|
|
12
|
-
|
|
13
|
-
const
|
|
11
|
+
export function isAcceptedConsolidation(row: Pick<View, "text">): boolean {
|
|
12
|
+
const end = row.text.indexOf("\n\n"); if (end < 0) return false;
|
|
13
|
+
const lines = row.text.slice(0, end).split("\n");
|
|
14
|
+
return lines.length === 3 && lines[0]!.startsWith(ACCEPTED_CONSOLIDATION_PREFIX)
|
|
15
|
+
&& /^[a-f0-9]{64}$/.test(lines[0]!.slice(ACCEPTED_CONSOLIDATION_PREFIX.length))
|
|
16
|
+
&& /^Action: (?:merge|supersession|recurring-fix)$/.test(lines[1]!)
|
|
17
|
+
&& /^Sources: mem:[a-f0-9]{32}(?:, mem:[a-f0-9]{32}){0,7}$/.test(lines[2]!);
|
|
18
|
+
}
|
|
14
19
|
const paths = (row: View) => row.about.filter(node => node.startsWith("path:"));
|
|
15
20
|
const sharedPath = (a: View, b: View) => paths(a).some(node => paths(b).includes(node));
|
|
16
21
|
const overlap = (a: Set<string>, b: Set<string>) => [...a].filter(word => b.has(word)).length / Math.max(1, Math.min(a.size, b.size));
|
|
@@ -18,7 +23,7 @@ const overlap = (a: Set<string>, b: Set<string>) => [...a].filter(word => b.has(
|
|
|
18
23
|
/** Bounded deterministic proposals over already-screened rows. Never mutates a note or runs a model. */
|
|
19
24
|
export function consolidationClusters(input: readonly View[], project: string, now: number): { clusters: ConsolidationCluster[]; truncated: boolean; engine: "fts5" | "substring" } {
|
|
20
25
|
if (!project.trim() || !Number.isFinite(now)) throw Error("consolidation context unavailable");
|
|
21
|
-
const eligible = input.filter(row => row.project === project && recallViewIsLive(row, now))
|
|
26
|
+
const eligible = input.filter(row => row.project === project && recallViewIsLive(row, now) && !isAcceptedConsolidation(row))
|
|
22
27
|
.sort((a, b) => b.at.localeCompare(a.at) || a.id.localeCompare(b.id));
|
|
23
28
|
const rows = [...new Map(eligible.map(row => [row.id, row])).values()].slice(0, CONSOLIDATION_SOURCE_LIMIT);
|
|
24
29
|
const index = new SearchIndex(":memory:"), clusters: ConsolidationCluster[] = [], seen = new Set<string>();
|
|
@@ -36,15 +41,15 @@ export function consolidationClusters(input: readonly View[], project: string, n
|
|
|
36
41
|
for (let i = 0; i < rows.length; i++) {
|
|
37
42
|
const row = rows[i]!, candidates = index.available ? new Set(index.query([...terms.get(row.id)!].slice(0, 32).join(" "), rows.length, project).map(hit => hit.ref)) : undefined;
|
|
38
43
|
for (const peer of rows.slice(i + 1)) {
|
|
39
|
-
if (row.kind !== peer.kind
|
|
44
|
+
if (row.kind !== peer.kind) continue;
|
|
40
45
|
const lexical = overlap(terms.get(row.id)!, terms.get(peer.id)!);
|
|
41
46
|
if (row.text === peer.text) add("supersession", [row, peer], "Identical text; review one replacement for both originals.");
|
|
42
47
|
else if ((!candidates || candidates.has(peer.id)) && lexical >= 0.5 && sharedPath(row, peer)) add("merge", [row, peer], "Shared path and overlapping indexed terms; no equivalence is assumed.");
|
|
43
48
|
}
|
|
44
49
|
// A retained supersession chain is context, never a reason to resurrect an old body.
|
|
45
|
-
if (row.supersedes.length >= 2
|
|
50
|
+
if (row.supersedes.length >= 2) add("merge", [row], "Current head already replaces multiple originals; review a chain summary.");
|
|
46
51
|
}
|
|
47
|
-
const failRows = rows.filter(row => row.kind === "fail"
|
|
52
|
+
const failRows = rows.filter(row => row.kind === "fail"), decisions = rows.filter(row => row.kind === "decision" || row.kind === "contract");
|
|
48
53
|
for (let i = 0; i < failRows.length; i++) for (const peer of failRows.slice(i + 1)) {
|
|
49
54
|
const fail = failRows[i]!;
|
|
50
55
|
if (!sharedPath(fail, peer) || overlap(terms.get(fail.id)!, terms.get(peer.id)!) < 0.5) continue;
|
|
@@ -79,18 +79,22 @@ export function evalSeeded(set: EvalSet, ks?: readonly number[], rank?: RankFn):
|
|
|
79
79
|
const base = join(dir, "memory");
|
|
80
80
|
mkdirSync(base, { recursive: true });
|
|
81
81
|
// One entry per day, so the recency weight (a 30-day scale) is exercised, long-tier entries first and short-tier ones last: a short-term note leaves the index 14 days after it is
|
|
82
|
-
// written, and the clock stops a day after the last entry, so the short ones written in that window are live when the set is searched (a set with more short entries than that
|
|
82
|
+
// written, and the undated clock stops a day after the last entry, so the short ones written in that window are live when the set is searched (a set with more short entries than that
|
|
83
83
|
// window holds loses its oldest, and a question naming one is then reported as not in the store, like any other missing entry).
|
|
84
|
+
// Explicit dates override only their entry; undated entries retain their legacy file-order day.
|
|
85
|
+
// Search a day after the latest actual seed time unless the set supplies its own clock.
|
|
84
86
|
let current = EPOCH;
|
|
85
87
|
store = new MemoryStore({ memoryDir: base, now: () => current, screen: () => "keep" });
|
|
86
88
|
const ids = new Map<string, string>();
|
|
87
89
|
const ordered = [...set.entries.filter(entry => entry.tier !== "short"), ...set.entries.filter(entry => entry.tier === "short")];
|
|
90
|
+
let latest = -Infinity;
|
|
88
91
|
for (const [index, entry] of ordered.entries()) {
|
|
89
|
-
current = EPOCH + index * DAY;
|
|
92
|
+
current = entry.at === undefined ? EPOCH + index * DAY : Date.parse(entry.at);
|
|
93
|
+
latest = Math.max(latest, current);
|
|
90
94
|
ids.set(entry.key, store.note({ op: "note", project: PROJECT, scope: "project", tier: entry.tier ?? "long", kind: entry.kind ?? "finding", peer: "eval", source: "person",
|
|
91
95
|
...(entry.title ? { title: entry.title } : {}), text: entry.text, about: entry.about ?? [] }, entry.tier === "short"));
|
|
92
96
|
}
|
|
93
|
-
current =
|
|
97
|
+
current = set.now === undefined ? latest + DAY : Date.parse(set.now);
|
|
94
98
|
const window = Math.max(...(ks ?? [10]));
|
|
95
99
|
const search = rank ? { rank: (question: string) => rank(store!, question, window), engine: () => "custom" } : ranker(store, PROJECT, window);
|
|
96
100
|
// Resolved through `store.get`, so a note that has already expired is absent, not silently present: coverage stays separate from ranking.
|
package/src/memory/eval.ts
CHANGED
|
@@ -12,10 +12,10 @@ export const MAX_QUESTIONS = 500, MAX_ENTRIES = 200, MAX_QUESTION_CHARS = 500, M
|
|
|
12
12
|
export const DEFAULT_KS: readonly number[] = [1, 5, 10];
|
|
13
13
|
|
|
14
14
|
export const EVAL_KINDS = ["decision", "finding", "contract", "fail"] as const;
|
|
15
|
-
export interface EvalEntry { key: string; title?: string; text: string; kind?: (typeof EVAL_KINDS)[number]; tier?: "short" | "long"; about?: string[] }
|
|
15
|
+
export interface EvalEntry { key: string; title?: string; text: string; kind?: (typeof EVAL_KINDS)[number]; tier?: "short" | "long"; about?: string[]; /** Explicit seed date; absent keeps the legacy positional day. */ at?: string }
|
|
16
16
|
/** `tag` is an optional label for what a question probes (a test counts them); it never changes the scoring. */
|
|
17
17
|
export interface EvalQuestion { q: string; expect: string[]; tag?: string }
|
|
18
|
-
export interface EvalSet { schema: typeof EVAL_SCHEMA; entries?: EvalEntry[]; questions: EvalQuestion[] }
|
|
18
|
+
export interface EvalSet { schema: typeof EVAL_SCHEMA; entries?: EvalEntry[]; questions: EvalQuestion[]; /** Search clock for a self-contained set; defaults to one day after its latest seed. */ now?: string }
|
|
19
19
|
|
|
20
20
|
const fail = (message: string): never => { throw new Error(`invalid eval set: ${message}`); };
|
|
21
21
|
const object = (value: unknown, what: string): Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : fail(`${what} must be an object`);
|
|
@@ -24,17 +24,25 @@ function exact(row: Record<string, unknown>, allowed: readonly string[], what: s
|
|
|
24
24
|
}
|
|
25
25
|
const text = (value: unknown, what: string, max: number): string => typeof value === "string" && value.trim() !== "" && value.length <= max && !/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/.test(value) ? value : fail(`${what} must be a non-empty string of at most ${max} characters`);
|
|
26
26
|
|
|
27
|
+
/** ISO calendar date or canonical UTC timestamp; reject Date.parse's rollover and locale-dependent forms. */
|
|
28
|
+
function date(value: unknown, what: string): string {
|
|
29
|
+
if (typeof value !== "string" || !/^[0-9]{4}-[0-9]{2}-[0-9]{2}(?:T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:[.][0-9]{3})?Z)?$/.test(value)) return fail(`${what} must be an ISO date or UTC timestamp`);
|
|
30
|
+
const ms = Date.parse(value), normalized = value.length === 10 ? value + "T00:00:00.000Z" : value.length === 20 ? value.slice(0, -1) + ".000Z" : value;
|
|
31
|
+
if (!Number.isFinite(ms) || new Date(ms).toISOString() !== normalized) return fail(`${what} must be a valid ISO date or UTC timestamp`);
|
|
32
|
+
return value;
|
|
33
|
+
}
|
|
34
|
+
|
|
27
35
|
/** Strict, bounded parse of untrusted JSON. Unknown keys are errors, so a typo cannot silently change what is measured. */
|
|
28
36
|
export function parseEvalSet(raw: unknown): EvalSet {
|
|
29
37
|
const root = object(raw, "the set");
|
|
30
|
-
exact(root, ["schema", "entries", "questions"], "the set");
|
|
38
|
+
exact(root, ["schema", "entries", "questions", "now"], "the set");
|
|
31
39
|
if (root.schema !== EVAL_SCHEMA) fail(`schema must be "${EVAL_SCHEMA}"`);
|
|
32
40
|
const entries: EvalEntry[] = [], keys = new Set<string>();
|
|
33
41
|
if (root.entries !== undefined) {
|
|
34
42
|
if (!Array.isArray(root.entries) || !root.entries.length || root.entries.length > MAX_ENTRIES) fail(`entries must hold 1 to ${MAX_ENTRIES} items when present`);
|
|
35
43
|
for (const [index, item] of (root.entries as unknown[]).entries()) {
|
|
36
44
|
const row = object(item, `entries[${index}]`);
|
|
37
|
-
exact(row, ["key", "title", "text", "kind", "tier", "about"], `entries[${index}]`);
|
|
45
|
+
exact(row, ["key", "title", "text", "kind", "tier", "about", "at"], `entries[${index}]`);
|
|
38
46
|
const key = text(row.key, `entries[${index}].key`, 80);
|
|
39
47
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(key)) fail(`entries[${index}].key may use letters, digits, ".", "_" and "-"`);
|
|
40
48
|
if (keys.has(key)) fail(`entries[${index}].key "${key}" repeats`);
|
|
@@ -42,9 +50,10 @@ export function parseEvalSet(raw: unknown): EvalSet {
|
|
|
42
50
|
const kind = row.kind === undefined ? undefined : (EVAL_KINDS as readonly unknown[]).includes(row.kind) ? row.kind as EvalEntry["kind"] : fail(`entries[${index}].kind must be one of ${EVAL_KINDS.join(", ")}`);
|
|
43
51
|
const tier = row.tier === undefined ? undefined : row.tier === "short" || row.tier === "long" ? row.tier : fail(`entries[${index}].tier must be short or long`);
|
|
44
52
|
const about = row.about === undefined ? undefined : Array.isArray(row.about) && row.about.length <= 20 ? (row.about as unknown[]).map((value, at) => text(value, `entries[${index}].about[${at}]`, 200)) : fail(`entries[${index}].about must be a list of at most 20 strings`);
|
|
45
|
-
entries.push({ key, text: text(row.text, `entries[${index}].text`, MAX_ENTRY_CHARS), ...(row.title === undefined ? {} : { title: text(row.title, `entries[${index}].title`, 200) }), ...(kind ? { kind } : {}), ...(tier ? { tier } : {}), ...(about ? { about } : {}) });
|
|
53
|
+
entries.push({ key, text: text(row.text, `entries[${index}].text`, MAX_ENTRY_CHARS), ...(row.title === undefined ? {} : { title: text(row.title, `entries[${index}].title`, 200) }), ...(kind ? { kind } : {}), ...(tier ? { tier } : {}), ...(about ? { about } : {}), ...(row.at === undefined ? {} : { at: date(row.at, `entries[${index}].at`) }) });
|
|
46
54
|
}
|
|
47
55
|
}
|
|
56
|
+
if (root.now !== undefined && !entries.length) fail("now requires self-contained entries");
|
|
48
57
|
if (!Array.isArray(root.questions) || !root.questions.length || root.questions.length > MAX_QUESTIONS) fail(`questions must hold 1 to ${MAX_QUESTIONS} items`);
|
|
49
58
|
const questions = (root.questions as unknown[]).map((item, index): EvalQuestion => {
|
|
50
59
|
const row = object(item, `questions[${index}]`);
|
|
@@ -58,7 +67,7 @@ export function parseEvalSet(raw: unknown): EvalSet {
|
|
|
58
67
|
}
|
|
59
68
|
return { q: text(row.q, `questions[${index}].q`, MAX_QUESTION_CHARS), expect, ...(tag ? { tag } : {}) };
|
|
60
69
|
});
|
|
61
|
-
return { schema: EVAL_SCHEMA, ...(entries.length ? { entries } : {}), questions };
|
|
70
|
+
return { schema: EVAL_SCHEMA, ...(entries.length ? { entries } : {}), questions, ...(root.now === undefined ? {} : { now: date(root.now, "now") }) };
|
|
62
71
|
}
|
|
63
72
|
|
|
64
73
|
/** `--k 1,5,10`: ascending, unique, 1 to MAX_K. */
|
|
@@ -88,7 +97,7 @@ export interface EvalQuestionResult {
|
|
|
88
97
|
/** `recall` and `hit` are null when no question counts (no answerable question), never 0: an empty average is not a measured zero. */
|
|
89
98
|
export interface EvalAggregate { questions: number; recall: Record<number, number | null>; hit: Record<number, number | null>; mrr: number | null }
|
|
90
99
|
export interface EvalReport {
|
|
91
|
-
/**
|
|
100
|
+
/** Ranked project-scoped `store.search` (BM25 with tier and recency weights), the engine used by default current curated `amem search`; `--filter` keeps legacy substring matching. */
|
|
92
101
|
engine: string;
|
|
93
102
|
ks: number[];
|
|
94
103
|
/** Every question. A question whose expected entry is not in the store scores 0, as a search that cannot find it would. */
|
|
@@ -142,7 +151,7 @@ const pct = (value: number | null): string => value === null ? " n/a" : value.t
|
|
|
142
151
|
export function evalText(report: EvalReport, label: string): string {
|
|
143
152
|
const head = (name: string, row: EvalAggregate) => `${name.padEnd(11)} ${String(row.questions).padStart(4)} ${report.ks.map(k => `${pct(row.recall[k] ?? null)}/${pct(row.hit[k] ?? null)}`.padStart(13)).join(" ")} ${pct(row.mrr).padStart(6)}`;
|
|
144
153
|
return [
|
|
145
|
-
`eval: ${label} (${report.engine === "substring" ? "substring fallback" : "ranked store.search, engine " + report.engine};
|
|
154
|
+
`eval: ${label} (${report.engine === "substring" ? "substring fallback" : "ranked store.search, engine " + report.engine}; same ranked search path as default \`amem search\`; \`--filter\` keeps legacy substring matching)`,
|
|
146
155
|
`${"".padEnd(11)} ${"n".padStart(4)} ${report.ks.map(k => `@${k} recall/hit`.padStart(13)).join(" ")} ${"MRR".padStart(6)}`,
|
|
147
156
|
head("all", report.all),
|
|
148
157
|
head("answerable", report.answerable),
|
|
@@ -206,7 +206,14 @@ function title(source: ExternalMemorySource, file: SourceFile, record: RawRecord
|
|
|
206
206
|
}
|
|
207
207
|
|
|
208
208
|
function portable(path: string): string { return path.split(sep).join("/"); }
|
|
209
|
-
|
|
209
|
+
/** Claude Code 2.1.311's project-path encoding, including its long-name suffix. */
|
|
210
|
+
function claudeProjectDirectory(cwd: string): string {
|
|
211
|
+
const path = realPath(cwd), encoded = path.replace(/[^a-zA-Z0-9]/g, "-");
|
|
212
|
+
if (encoded.length <= 200) return encoded;
|
|
213
|
+
let hash = 0;
|
|
214
|
+
for (let index = 0; index < path.length; index++) hash = (Math.imul(hash, 31) + path.charCodeAt(index)) | 0;
|
|
215
|
+
return `${encoded.slice(0, 200)}-${Math.abs(hash).toString(36)}`;
|
|
216
|
+
}
|
|
210
217
|
|
|
211
218
|
export async function importExternalMemory(opts: {
|
|
212
219
|
base: string;
|
package/src/memory/platform.ts
CHANGED
|
@@ -103,18 +103,18 @@ export function memoryGitEnv(env: NodeJS.ProcessEnv = process.env, options: { re
|
|
|
103
103
|
/** spawnSync form of the same call: the env option is merged with memoryGitEnv, everything else passes through. */
|
|
104
104
|
export function memoryGitSync(args: string[], options: SpawnSyncOptionsWithStringEncoding): SpawnSyncReturns<string> {
|
|
105
105
|
const argv = memoryGitArgv(args);
|
|
106
|
-
|
|
106
|
+
const timeout = gitTimeoutContext?.getStore();
|
|
107
|
+
return childProcess().spawnSync(argv[0]!, argv.slice(1), { ...(timeout !== undefined ? { timeout, killSignal: "SIGKILL" as const } : {}), ...options, env: memoryGitEnv(options.env ?? process.env) });
|
|
107
108
|
}
|
|
108
|
-
let
|
|
109
|
+
let gitTimeoutContext: import("node:async_hooks").AsyncLocalStorage<number> | undefined;
|
|
109
110
|
/**
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
* ponytail: one global instead of a parameter on every caller between doctor and its git calls (project key, repository root, visibility). Upgrade: thread an options argument through them.
|
|
111
|
+
* Bound `memoryGitSync` calls inside this async operation unless a call supplies its own timeout.
|
|
112
|
+
* Nested and concurrent operations keep independent contexts; unrelated host calls retain their own options.
|
|
113
|
+
* Load async_hooks only when a caller requests a deadline, keeping path-only hook imports unchanged.
|
|
114
114
|
*/
|
|
115
115
|
export async function withGitTimeout<T>(ms: number, work: () => Promise<T>): Promise<T> {
|
|
116
|
-
|
|
117
|
-
|
|
116
|
+
gitTimeoutContext ??= new (require("node:async_hooks").AsyncLocalStorage)();
|
|
117
|
+
return gitTimeoutContext!.run(ms, work);
|
|
118
118
|
}
|
|
119
119
|
|
|
120
120
|
/**
|
package/src/memory/policy.ts
CHANGED
|
@@ -99,6 +99,7 @@ function literals(value: unknown, label: string): string[] {
|
|
|
99
99
|
|
|
100
100
|
function defaultPii(text: string): boolean {
|
|
101
101
|
if (/\b(?:\d{6}-\d{7}|\d{3}-\d{2}-\d{4})\b/.test(text)) return true;
|
|
102
|
+
if (!text.includes("@")) return false;
|
|
102
103
|
// Bound each email candidate before matching; arbitrary user regexes are never run.
|
|
103
104
|
for (const token of text.split(/[\s<>()\[\]{}",;:]+/)) if (token.includes("@")) {
|
|
104
105
|
if (token.length > 320 || /[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}/i.test(token)) return true;
|
|
@@ -106,6 +107,51 @@ function defaultPii(text: string): boolean {
|
|
|
106
107
|
return false;
|
|
107
108
|
}
|
|
108
109
|
|
|
110
|
+
interface DefaultPathProbe { names: string[]; segments: string[] }
|
|
111
|
+
|
|
112
|
+
/** A mandatory literal, not a deny verdict. All other regex grammar uses the canonical checker. */
|
|
113
|
+
function necessaryDenyLiteral(source: string): string | undefined {
|
|
114
|
+
if (source.length > 256) return;
|
|
115
|
+
let index = source.startsWith("^") ? 1 : 0, literal = "";
|
|
116
|
+
for (; index < source.length; index++) {
|
|
117
|
+
const character = source[index]!;
|
|
118
|
+
if (/^[A-Za-z0-9_-]$/.test(character)) literal += character;
|
|
119
|
+
else if (character === "\\" && source[index + 1] === ".") { literal += "."; index++; }
|
|
120
|
+
else break;
|
|
121
|
+
}
|
|
122
|
+
const rest = source.slice(index);
|
|
123
|
+
return literal && (rest === "" || rest === "$" || rest === "(\\.|$)") ? literal : undefined;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Do not add getter reads or assumptions about malformed/mutated exported defaults. */
|
|
127
|
+
function plainDefaultStrings(values: readonly string[]): boolean {
|
|
128
|
+
if (types.isProxy(values) || !Array.isArray(values) || Object.getPrototypeOf(values) !== Array.prototype || values.length > 64 || Object.getOwnPropertySymbols(values).length || "toJSON" in values) return false;
|
|
129
|
+
if (Object.getOwnPropertyNames(values).length !== values.length + 1) return false;
|
|
130
|
+
for (let index = 0; index < values.length; index++) {
|
|
131
|
+
const descriptor = Object.getOwnPropertyDescriptor(values, String(index));
|
|
132
|
+
if (!descriptor || !("value" in descriptor) || typeof descriptor.value !== "string") return false;
|
|
133
|
+
}
|
|
134
|
+
return true;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function defaultPathProbe(): DefaultPathProbe | undefined {
|
|
138
|
+
const needles = DENY_NAMES.map(necessaryDenyLiteral);
|
|
139
|
+
return needles.every((value): value is string => value !== undefined) ? { names: needles, segments: [...DENY_SEGMENTS] } : undefined;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function containsPathLiteral(text: string, probe: DefaultPathProbe, deny: readonly string[]): boolean {
|
|
143
|
+
return text.includes("/.") || text.includes("//") || deny.some(entry => text.includes(entry))
|
|
144
|
+
|| probe.segments.some(entry => text.includes(entry)) || probe.names.some(needle => text.includes(needle));
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function mayNeedPathCheck(text: string, probe: DefaultPathProbe | undefined, deny: readonly string[]): boolean {
|
|
148
|
+
if (!probe || containsPathLiteral(text, probe, deny)) return true;
|
|
149
|
+
// Raw/node/token paths are substrings of text; pathCandidates takes substrings after NFC.
|
|
150
|
+
// Inspect both: removing ignorables or lowercasing after NFKC can leave a non-NFC result.
|
|
151
|
+
const canonical = text.normalize("NFC");
|
|
152
|
+
return canonical !== text && containsPathLiteral(canonical, probe, deny);
|
|
153
|
+
}
|
|
154
|
+
|
|
109
155
|
/** No filesystem, model, network or storage work occurs while constructing or screening this policy. */
|
|
110
156
|
export function createStandalonePolicy(options: StandalonePolicyOptions = {}): StandalonePolicy {
|
|
111
157
|
if (!options || typeof options !== "object" || types.isProxy(options) || Object.getPrototypeOf(options) !== Object.prototype || Object.getOwnPropertySymbols(options).length) throw new Error("invalid standalone policy options");
|
|
@@ -119,16 +165,23 @@ export function createStandalonePolicy(options: StandalonePolicyOptions = {}): S
|
|
|
119
165
|
// Evict the batch when full; a measured need for better hit rates can upgrade this to an LRU.
|
|
120
166
|
const textChecks = new Map<string, boolean>(); let textBytes = 0;
|
|
121
167
|
let defaultDenyRevision = "";
|
|
168
|
+
// ponytail: the negative probe supports at most 64 dense default entries and 256-character simple name sources.
|
|
169
|
+
// Everything else falls back; extend the grammar only with differential proof, never an approximate verdict.
|
|
170
|
+
let pathProbe: DefaultPathProbe | undefined;
|
|
171
|
+
let defaultsWerePlain = false;
|
|
122
172
|
const publicText = (text: string): boolean => {
|
|
123
173
|
const cached = textChecks.get(text);
|
|
124
174
|
if (cached !== undefined) return cached;
|
|
125
175
|
const check = () => {
|
|
126
|
-
if (looksSecret(text)
|
|
176
|
+
if (looksSecret(text)) return false;
|
|
177
|
+
const unquoted = text.replace(/["']/g, "");
|
|
178
|
+
if (unquoted !== text && looksSecret(unquoted)) return false;
|
|
127
179
|
const normalized = fold(text);
|
|
128
180
|
if (defaultPii(normalized) || pii.some(pattern => normalized.includes(pattern))) return false;
|
|
181
|
+
if (!mayNeedPathCheck(normalized, pathProbe, deny)) return true;
|
|
129
182
|
const candidates = normalized.includes("/") || /\.[a-z]/.test(normalized) ? pathCandidates(normalized) : [];
|
|
130
183
|
const paths = [normalized, nodePath(normalized), ...candidates, ...normalized.split(/[\s"'`(){}\[\],;:=<>]+/)];
|
|
131
|
-
return !paths.some(path => path && isDenied(posix.normalize(path), deny));
|
|
184
|
+
return ![...new Set(paths)].some(path => path && isDenied(posix.normalize(path), deny));
|
|
132
185
|
};
|
|
133
186
|
const allowed = check(), bytes = Buffer.byteLength(text);
|
|
134
187
|
if (bytes <= 1024) {
|
|
@@ -140,8 +193,14 @@ export function createStandalonePolicy(options: StandalonePolicyOptions = {}): S
|
|
|
140
193
|
const standalone = (value: unknown): { ok: boolean; value?: unknown } => {
|
|
141
194
|
try {
|
|
142
195
|
// The exported default lists can be tightened by a host between calls.
|
|
196
|
+
// A getter can change behavior without changing its first serialized value: validate shape every time.
|
|
197
|
+
const defaultsPlain = plainDefaultStrings(DENY_NAMES) && plainDefaultStrings(DENY_SEGMENTS);
|
|
143
198
|
const revision = JSON.stringify([DENY_NAMES, DENY_SEGMENTS]);
|
|
144
|
-
|
|
199
|
+
const changed = revision !== defaultDenyRevision;
|
|
200
|
+
if (changed) { textChecks.clear(); textBytes = 0; defaultDenyRevision = revision; }
|
|
201
|
+
if (!defaultsPlain) pathProbe = undefined;
|
|
202
|
+
else if (changed || !defaultsWerePlain) pathProbe = defaultPathProbe();
|
|
203
|
+
defaultsWerePlain = defaultsPlain;
|
|
145
204
|
const complete = snapshot(value);
|
|
146
205
|
if (!complete) return { ok: false };
|
|
147
206
|
// Only pure text checks are reused. Shape/bounds/private flags and the host's detached
|