@staix/agent-mem 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +3 -1
- package/package.json +1 -1
- package/src/cli.ts +16 -6
- package/src/commands-table.ts +1 -1
- package/src/index.ts +1 -0
- package/src/mcp-memory.ts +6 -2
- package/src/mcp.ts +1 -1
- package/src/memory/candidates.ts +44 -1
- package/src/memory/commands.ts +14 -4
- package/src/memory/consolidate.ts +9 -6
- package/src/memory/consolidation-clusters.ts +12 -7
- package/src/memory/eval-fixture.ts +7 -3
- package/src/memory/eval.ts +15 -6
- package/src/memory/external-import.ts +8 -1
- package/src/memory/platform.ts +8 -8
- package/src/memory/project-import.ts +31 -14
- package/src/memory/search-fts.ts +15 -6
- package/src/memory/search-score.ts +66 -0
- package/src/memory/store.ts +40 -13
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Curated memory with append-only JSONL logs, a derived SQLite index, short- and long-term tiers, a typed graph, goals with leases, and git sync between machines. Extracted from [agent-hub](https://github.com/STAIxBWLB/agent-hub), with compatible storage and entry IDs.
|
|
4
4
|
|
|
5
|
-
Version: 0.
|
|
5
|
+
Version: 0.11.0. Requires Bun 1.4.2 or later; no runtime dependencies.
|
|
6
6
|
|
|
7
7
|
## Quick start
|
|
8
8
|
|
|
@@ -213,6 +213,8 @@ Project identity (#15 D2) keeps `memoryProjectKey`: a worktree, a submodule and
|
|
|
213
213
|
|
|
214
214
|
`amem eval [--set ko|<file>] [--k 1,5,10] [--json]` measures recall with no model and no network (agent-mem#40 F13): it replays a question set against ranked `store.search` and reports recall@k, hit@k and MRR. The ordinary curated project `amem search` path now uses that ranked search; `--filter` selects the legacy substring behavior. Without `--set` it scores the built-in synthetic set (26 invented harbour entries, long and short tier, written a day apart, and 32 questions, shipped in `src/memory/eval-set.json`), which is seeded into a throwaway store, so it never reads or writes your memory home and gives the same numbers on every machine. A set of your own either carries its own `entries` (self-contained, named by `key`) or, without entries, names real entry ids (or a prefix of 8 or more hex characters) and is scored against this project's store; each name is resolved like `amem show` does (a promoted short-term id finds its long-term copy, and a superseded id stands for the entry that replaced it, since search returns only live entries), and a prefix that matches more than one live entry is refused. `mem:<id>` as a brief prints it is accepted. Questions whose expected entry is not in the store score 0 in `all` and are left out of `answerable`, and `answerable` is scored over the entries the store holds, so a ranking problem and a coverage problem stay apart (`n/a` when no question is answerable). `--set ko` scores the second built-in set instead: 37 invented Korean entries and 38 questions (particle variants, spaced and unspaced compounds, mixed Korean-English terms, Latin abbreviations, short and long tier), with its baseline recorded in `docs/contracts.md` (agent-mem#62). Set format: `docs/contracts.md#recall-evaluation-f13`.
|
|
215
215
|
|
|
216
|
+
External retrieval measurements for MCP, library search and a plain BM25 baseline are recorded in [LongMemEval S/M](docs/bench/longmemeval.md). The full datasets stay outside the repository; CI runs only synthetic harness fixtures.
|
|
217
|
+
|
|
216
218
|
The R7 capacity advisor reports this machine's load and proposes capture bounds: `amem capacity [--json]` shows capture size and growth, disk, memory, the rolling `metrics.jsonl` readings and the proposal with its binding constraint; `amem capacity apply [--cap-mb N] [--retention-days N]` saves a preview, and only a matching later `--confirm` writes the home default config. Applying is person-only (agent markers refused, a terminal required); an agent may read the proposal through the library, never apply it. Under the 2026-10-17 owner decision (item 2) the proposal is the largest cap and retention that fit, and it may be above the current values: the cap is the disk-share bound itself (default 2% of free space, `--disk-share`), and retention is the share divided by the measured daily growth, up to a 365-day ceiling (a `ponytail:` constant, a person setting is the upgrade path). A hook p95 over 150 ms or hook RSS over 100 MB halves the proposed cap, and search RSS over 200 MB halves the retention. The reason names every binding constraint: `disk-share retention` or `retention ceiling`, `disk-share cap`, `R3 search RSS`, `R2 hook cost`. A proposal is never written by the advisor, and `apply` still takes `--cap-mb` and `--retention-days` to override it. Benchmark samples stay in the measuring script's synthetic home; only real hook and search invocations feed the advisor's rolling metrics.
|
|
217
219
|
|
|
218
220
|
## Repo scope (#15 T4)
|
package/package.json
CHANGED
package/src/cli.ts
CHANGED
|
@@ -43,6 +43,7 @@ import { appendGitattributes, appendRepoLine, boundRepoRoot, planGitattributes,
|
|
|
43
43
|
import type { Visibility } from "./memory/sync.ts";
|
|
44
44
|
import { applySplit, previewSplit } from "./memory/capacity-split.ts";
|
|
45
45
|
import { evalText, parseKs } from "./memory/eval.ts";
|
|
46
|
+
import { unrankedScoreDetails } from "./memory/search-score.ts";
|
|
46
47
|
import { builtinSet, evalSeeded, evalStore, FIXTURE_PATH, readEvalSet } from "./memory/eval-fixture.ts";
|
|
47
48
|
import { markShown, pendingSummary } from "./memory/candidate-pending.ts";
|
|
48
49
|
import { buildNote, FAIL_TEMPLATE, hasFailLabel, noteAdd, parseFailTemplate, type FailFields } from "./memory/note-add.ts";
|
|
@@ -573,7 +574,7 @@ export async function runCli(argv: readonly string[], context: CliContext = {}):
|
|
|
573
574
|
try {
|
|
574
575
|
const result = await importProject(store, { from: options["--from"], project: key, confirm: options["--confirm"] === true,
|
|
575
576
|
allowShared: options["--allow-shared"] === true, policy: note => policy.screenStandalone(note) });
|
|
576
|
-
const counts = `sources ${JSON.stringify(result.provenance.source)}, peers ${JSON.stringify(result.provenance.peer)}, scopes ${JSON.stringify(result.provenance.scope)}`;
|
|
577
|
+
const counts = `sources ${JSON.stringify(result.provenance.source)}, peers ${JSON.stringify(result.provenance.peer)}, scopes ${JSON.stringify(result.provenance.scope)}, machines ${JSON.stringify(result.provenance.machine)}`;
|
|
577
578
|
const summary = result.applied ? `imported ${result.added} of ${result.read} curated notes into ${result.project} with their original ids`
|
|
578
579
|
: `preview: ${result.wouldAdd} of ${result.read} curated notes can be imported into ${result.project} with their original ids (re-run with --confirm to write)`;
|
|
579
580
|
return options["--json"] ? JSON.stringify(result, null, 2)
|
|
@@ -589,6 +590,7 @@ export async function runCli(argv: readonly string[], context: CliContext = {}):
|
|
|
589
590
|
return options["--json"] ? JSON.stringify(written, null, 2) : `${op === "forget" ? "forgot" : op === "archive" ? "archived" : "unarchived"} ${written.key}${requested !== written.key ? ` (alias ${requested})` : ""}`;
|
|
590
591
|
}
|
|
591
592
|
if (sub === "search" && options["--all-projects"] === true) {
|
|
593
|
+
if (options["--explain"] === true) throw new Error("--explain applies to current-project ranked search, not --all-projects");
|
|
592
594
|
if (options["--filter"] === true) throw new Error("--filter applies to project-local search, not --all-projects");
|
|
593
595
|
const words = parsed.positionals.join(" ");
|
|
594
596
|
if (!words.trim()) throw new Error("usage: amem search --all-projects <words> [--limit N] [--include-archived] [--json]");
|
|
@@ -598,6 +600,11 @@ export async function runCli(argv: readonly string[], context: CliContext = {}):
|
|
|
598
600
|
return [`crossed projects: searched ${result.projects.length} (${result.projects.join(", ")})`, ...(result.hits.length ? result.hits.map(one) : ["(nothing)"])].join("\n");
|
|
599
601
|
}
|
|
600
602
|
if (sub === "search" && options["--include-archived"] === true) throw new Error("--include-archived only applies with --all-projects");
|
|
603
|
+
if (sub === "search" && options["--explain"] === true) {
|
|
604
|
+
if (options["--filter"] === true) throw new Error("--explain cannot be combined with --filter");
|
|
605
|
+
if (options["--history"] === true || options["--as-of"] !== undefined) throw new Error("--explain cannot be combined with --history or --as-of");
|
|
606
|
+
if (!parsed.positionals.join(" ").trim()) throw new Error("--explain requires a non-empty search query");
|
|
607
|
+
}
|
|
601
608
|
if (sub === "alias") {
|
|
602
609
|
if (parsed.path[1] === "add") {
|
|
603
610
|
const [from, to] = parsed.positionals;
|
|
@@ -686,10 +693,12 @@ export async function runCli(argv: readonly string[], context: CliContext = {}):
|
|
|
686
693
|
const kind = options["--kind"], since = options["--since"], limit = options["--limit"] !== undefined ? Number(options["--limit"]) : 50;
|
|
687
694
|
const rows = read.views.filter(row => options["--tier"] !== "short" && (kind === undefined || row.kind === kind) && (typeof since !== "string" || Date.parse(row.at) >= Date.parse(since))
|
|
688
695
|
&& words.every(word => `${row.title ?? ""}\n${row.text}\n${row.about.join(" ")}`.toLowerCase().includes(word))).slice(0, limit);
|
|
689
|
-
|
|
696
|
+
const repoDetails = explainSearch ? rows.map(row => ({ ...row, scoreDetails: unrankedScoreDetails("repo", null) })) : rows;
|
|
697
|
+
if (options["--json"]) { try { const base = JSON.parse(output) as unknown[]; return JSON.stringify([...base, ...repoDetails], null, 2); } catch { return output; } }
|
|
690
698
|
const line = (row: RepoView) => `repo:${row.id.slice(0, 8)} ${row.at.slice(0, 16).replace("T", " ")} repo ${row.kind.padEnd(8)} ${`${row.title ? `${row.title}: ` : ""}${row.text}`.replace(/\s+/g, " ").slice(0, 100)}`;
|
|
691
699
|
const note = read.state === "unsafe" ? [`(repo scope unavailable: ${read.reason})`] : read.skipped.malformed + read.skipped.mismatched + read.skipped.withheld ? [`(${read.skipped.malformed + read.skipped.mismatched} repo lines skipped as invalid, ${read.skipped.withheld} withheld by the screen)`] : [];
|
|
692
|
-
|
|
700
|
+
const lines = explainSearch ? rows.flatMap(row => [line(row), `score: ${JSON.stringify(unrankedScoreDetails("repo", null))}`]) : rows.map(line);
|
|
701
|
+
return [output, ...(rows.length ? ["repo scope (.amem/log.jsonl):", ...lines] : []), ...note].join("\n");
|
|
693
702
|
};
|
|
694
703
|
const crosses = definition.policyFree === true;
|
|
695
704
|
const cwd = context.cwd ?? process.cwd();
|
|
@@ -732,13 +741,14 @@ export async function runCli(argv: readonly string[], context: CliContext = {}):
|
|
|
732
741
|
if (args[index] === "--capture-days") { captureArgs.add(index); captureArgs.add(index + 1); }
|
|
733
742
|
else if (args[index]?.startsWith("--capture-days=")) captureArgs.add(index);
|
|
734
743
|
}
|
|
735
|
-
// --filter
|
|
744
|
+
// --filter and --explain select standalone CLI behavior; hosts keep their existing raw command arguments.
|
|
736
745
|
const legacySearch = options["--filter"] === true;
|
|
737
|
-
const
|
|
746
|
+
const explainSearch = sub === "search" && options["--explain"] === true;
|
|
747
|
+
const syncArgs = args.filter((arg, index) => arg !== "--if-due" && arg !== "--force" && !captureArgs.has(index) && (!legacySearch || arg !== "--filter") && (!explainSearch || arg !== "--explain"));
|
|
738
748
|
let output: string;
|
|
739
749
|
const stop = ifDue ? stopSignal(context.syncDeadlineMs ?? SYNC_IF_DUE_DEADLINE_MS, process) : undefined;
|
|
740
750
|
try {
|
|
741
|
-
output = await withRepoRows(await rawMemoryCommand(syncArgs, cwd, { person: true, rankedSearch: !legacySearch, project: boundProject(), open, readable: notes => notes, stale: createRecallStalenessProbe({ cwd: recallRoot ?? cwd, rootConfirmed, policy: () => configured }), outputReady: notes => outputReady(notes.map(view => view.id)), importPolicy: () => text => configured.declinesPersonal(text),
|
|
751
|
+
output = await withRepoRows(await rawMemoryCommand(syncArgs, cwd, { person: true, rankedSearch: !legacySearch, ...(explainSearch ? { explainSearch: true } : {}), project: boundProject(), open, readable: notes => notes, stale: createRecallStalenessProbe({ cwd: recallRoot ?? cwd, rootConfirmed, policy: () => configured }), outputReady: notes => outputReady(notes.map(view => view.id)), importPolicy: () => text => configured.declinesPersonal(text),
|
|
742
752
|
sync: { screen: row => configured.screenStandalone(row), ...(stop ? { signal: stop.signal } : {}) }, // the repository-narrowed policy binds what sync publishes too
|
|
743
753
|
onBridgeStatus: status => { receipt.bridgeStatus = status; } }));
|
|
744
754
|
} catch (error) {
|
package/src/commands-table.ts
CHANGED
|
@@ -33,7 +33,7 @@ export const COMMANDS: Record<string, CommandDefinition> = {
|
|
|
33
33
|
"--all-projects": flag("List every active project's candidates; the reply says it crossed projects"), "--project": text("List one named project's candidates") } },
|
|
34
34
|
list: { description: "List memory", options: { ...filters, ...history } },
|
|
35
35
|
search: { description: "Rank current curated project notes; history/as-of and empty queries retain newest-first filtering", options: { ...filters, ...history,
|
|
36
|
-
"--filter": flag("Current-project curated notes: match every word as a substring, newest first; not with --all-projects"), "--all-projects": flag("Search every active project; the reply says it crossed projects"),
|
|
36
|
+
"--filter": flag("Current-project curated notes: match every word as a substring, newest first; not with --all-projects"), "--explain": flag("Include the score factors for ranked project-local results"), "--all-projects": flag("Search every active project; the reply says it crossed projects"),
|
|
37
37
|
"--include-archived": flag("With --all-projects, also read archived projects") }, args: [{ name: "words", variadic: true }] },
|
|
38
38
|
brief: { description: "Read intent/path memory: fails first, then decisions and contracts; untrusted data within a UTF-8 byte cap", options: { ...json,
|
|
39
39
|
"--for": text("Explicit intent"), "--paths": text("Repeatable named path"), "--symbols": text("Repeatable named symbol"), "--bytes": text("Complete brief-body byte cap (1..8192, default 2048)") } },
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export * from "./memory/store.ts";
|
|
2
|
+
export type { ScoreFactors } from "./memory/search-score.ts";
|
|
2
3
|
export * from "./memory/capture.ts";
|
|
3
4
|
export { fts5Available, ftsTerms, SearchIndex, captureSearch } from "./memory/search-fts.ts";
|
|
4
5
|
export type { CuratedDoc, RankedRef } from "./memory/search-fts.ts";
|
package/src/mcp-memory.ts
CHANGED
|
@@ -187,8 +187,9 @@ export function createMemoryMcp(options: MemoryMcpOptions): MemoryMcp {
|
|
|
187
187
|
} finally { index.close(); }
|
|
188
188
|
},
|
|
189
189
|
async amem_search(args) {
|
|
190
|
-
bound(args, ["query", "limit", "since", "includeCapture"]); const query = textOf(args, "query")!, limit = limitOf(args), since = sinceOf(args);
|
|
190
|
+
bound(args, ["query", "limit", "since", "includeCapture", "explain"]); const query = textOf(args, "query")!, limit = limitOf(args), since = sinceOf(args);
|
|
191
191
|
if (args.includeCapture !== undefined && typeof args.includeCapture !== "boolean") unavailable();
|
|
192
|
+
if (args.explain !== undefined && typeof args.explain !== "boolean") unavailable();
|
|
192
193
|
if (!await currentPolicy().screen({ query })) unavailable();
|
|
193
194
|
// ponytail: scan all matches into private disposable disk state, keeping one body page in JS.
|
|
194
195
|
// Work scales with matches; a policy-bound cache is the upgrade if this reaches the unchanged watchdog.
|
|
@@ -219,7 +220,10 @@ export function createMemoryMcp(options: MemoryMcpOptions): MemoryMcp {
|
|
|
219
220
|
const repoNotes = (await repoRows()).filter(view => (!since || Date.parse(view.at) >= Date.parse(since)) && (useFts || words.every(word => `${view.title ?? ""}\n${view.text}\n${view.about.join(" ")}`.toLowerCase().includes(word))));
|
|
220
221
|
for (const view of repoNotes) append({ source: "repo", evidence: false, view });
|
|
221
222
|
if (batch.length) index.append(batch);
|
|
222
|
-
const selected = index.best(query, limit, store.readNow(), useFts).map(({ value, score }) =>
|
|
223
|
+
const selected = index.best(query, limit, store.readNow(), useFts, args.explain === true).map(({ value, score, scoreDetails }) => {
|
|
224
|
+
const hit = { ...value, score };
|
|
225
|
+
return scoreDetails ? { ...hit, scoreDetails: { source: value.source, ...scoreDetails } } : hit;
|
|
226
|
+
});
|
|
223
227
|
const probe = stale(), result = { engine: useFts ? "fts5" : "substring", hits: selected.map(hit => hit.source === "curated" && "view" in hit && hit.view ? { ...hit, view: decorate(hit.view as View, probe) } : hit) };
|
|
224
228
|
guard(); return observed(result, selected.flatMap(hit => hit.source === "curated" && "view" in hit && hit.view ? [hit.view.id] : []), selected.map(hit => "view" in hit ? hit.view : hit.capture));
|
|
225
229
|
} finally { const position = closers.indexOf(releaseIndex); if (position >= 0) closers.splice(position, 1); releaseIndex(); }
|
package/src/mcp.ts
CHANGED
|
@@ -9,7 +9,7 @@ const tool = <Name extends string>(name: Name, description: string, inputSchema:
|
|
|
9
9
|
|
|
10
10
|
/** Storage scope comes from startup, never tool arguments; candidate proposals remain person-reviewed. */
|
|
11
11
|
export const MCP_TOOLS = [
|
|
12
|
-
tool("amem_search", "Search screened memory in this project. Capture results are recall, not curated evidence.", object({ query: string(1000), limit, since: string(40), includeCapture: { type: "boolean" } }, ["query"])),
|
|
12
|
+
tool("amem_search", "Search screened memory in this project. Capture results are recall, not curated evidence.", object({ query: string(1000), limit, since: string(40), includeCapture: { type: "boolean" }, explain: { type: "boolean" } }, ["query"])),
|
|
13
13
|
tool("amem_evidence", "Return a bounded bundle of cited, live curated entries for a question. Untrusted memory data only: no answer or model call. Capture, repo-only notes, pending candidates and model answers are not evidence.", object({ question: string(1000), limit: { type: "integer", minimum: 1, maximum: 20 }, maxBytes: { type: "integer", minimum: 256, maximum: 8192 } }, ["question"])),
|
|
14
14
|
tool("amem_show", "Show a screened memory entry in this project by its exact id or unique prefix.", object({ id: string(80) }, ["id"])),
|
|
15
15
|
tool("amem_timeline", "Read screened memory near an entry or time in this project.", object({ id: string(80), since: string(40), limit })),
|
package/src/memory/candidates.ts
CHANGED
|
@@ -13,6 +13,15 @@ export const CANDIDATE_DB_BUSY_TIMEOUT_MS = 3000;
|
|
|
13
13
|
/** First line of every note written by accepting a consolidation candidate; `consolidationClusters` reads it to avoid proposing summaries of its own summaries. */
|
|
14
14
|
export const ACCEPTED_CONSOLIDATION_PREFIX = "Accepted consolidation candidate: ";
|
|
15
15
|
export const CANDIDATE_RETENTION = { count: 500, ageMs: 30 * 86400_000 } as const;
|
|
16
|
+
export const CONSOLIDATION_REVIEW_LIMIT = 10_000;
|
|
17
|
+
export class ConsolidationReviewLimit extends Error {
|
|
18
|
+
readonly code = "consolidation_review_limit";
|
|
19
|
+
readonly limit = CONSOLIDATION_REVIEW_LIMIT;
|
|
20
|
+
constructor(readonly project: string, readonly count: number) {
|
|
21
|
+
super(`consolidation review cap reached: project ${project}, count ${count}, limit ${CONSOLIDATION_REVIEW_LIMIT}; no reviewed identities evicted`);
|
|
22
|
+
this.name = "ConsolidationReviewLimit";
|
|
23
|
+
}
|
|
24
|
+
}
|
|
16
25
|
export const COUNTER_RETENTION = { count: 10_000, ageMs: 90 * 86400_000 } as const;
|
|
17
26
|
export type CandidateVerdict = "public" | "private" | "pii" | "unknown";
|
|
18
27
|
export interface CandidateTarget { id: string; revision: string }
|
|
@@ -27,6 +36,7 @@ export type CandidateInput = { project: string } & (
|
|
|
27
36
|
);
|
|
28
37
|
export type CandidateStatus = "pending" | "accepting" | "accepted" | "rejected";
|
|
29
38
|
export interface Candidate { id: string; createdAt: string; status: CandidateStatus; input: CandidateInput; acceptedNoteId: string | null }
|
|
39
|
+
export interface ConsolidationReview { status: Exclude<CandidateStatus, "pending">; acceptedNoteId: string | null }
|
|
30
40
|
export interface CandidateFilter { project: string; status?: CandidateStatus | "all"; kind?: CandidateInput["kind"]; limit?: number }
|
|
31
41
|
export interface CounterKey { entryId: string; project: string; task?: string | null; turn?: string | null; session?: string | null; deliveryId: string }
|
|
32
42
|
export interface CounterValues { rendered?: number | null; delivered?: number | null; cited?: number | null }
|
|
@@ -146,13 +156,27 @@ export class CandidateStore {
|
|
|
146
156
|
// F7 / AC4: when a candidate was first shown to the person. In place and nullable: an older database gains it here, and a reader without it treats every row as unshown.
|
|
147
157
|
// Several processes open this store at once (every `amem mcp` builds it at startup), so the check and the ALTER are one immediate transaction: the loser waits, then sees the column.
|
|
148
158
|
if (!hasShownColumn(this.db)) this.db.transaction(() => { if (!hasShownColumn(this.db)) this.db.run("ALTER TABLE candidates ADD COLUMN shown_at INTEGER"); }).immediate();
|
|
159
|
+
this.db.run("CREATE TABLE IF NOT EXISTS consolidation_reviews (id TEXT PRIMARY KEY, project TEXT NOT NULL, status TEXT NOT NULL, note_id TEXT)");
|
|
149
160
|
this.db.run("CREATE TABLE IF NOT EXISTS counters (id TEXT PRIMARY KEY, project TEXT NOT NULL, payload TEXT NOT NULL, at INTEGER NOT NULL, rendered INTEGER, delivered INTEGER, cited INTEGER)");
|
|
150
161
|
this.prune();
|
|
151
162
|
}
|
|
152
163
|
close(): void { this.db.close(); }
|
|
164
|
+
/** Keep only identity and disposition after payload retention, never source text or target lists. */
|
|
165
|
+
private retainConsolidationReviews(): void {
|
|
166
|
+
const overflow = this.db.query(`SELECT project, count(*) AS n FROM consolidation_reviews GROUP BY project HAVING count(*) > ?
|
|
167
|
+
UNION ALL SELECT c.project, count(*) + (SELECT count(*) FROM consolidation_reviews WHERE project = c.project) AS n FROM candidates c LEFT JOIN consolidation_reviews r ON r.id = c.id
|
|
168
|
+
WHERE c.kind = 'consolidation' AND c.status IN ('accepting', 'accepted', 'rejected') AND r.id IS NULL
|
|
169
|
+
GROUP BY c.project HAVING count(*) + (SELECT count(*) FROM consolidation_reviews WHERE project = c.project) > ? LIMIT 1`)
|
|
170
|
+
.get(CONSOLIDATION_REVIEW_LIMIT, CONSOLIDATION_REVIEW_LIMIT) as { project: string; n: number } | null;
|
|
171
|
+
if (overflow) throw new ConsolidationReviewLimit(overflow.project, overflow.n);
|
|
172
|
+
this.db.run(`INSERT INTO consolidation_reviews (id, project, status, note_id)
|
|
173
|
+
SELECT id, project, status, note_id FROM candidates WHERE kind = 'consolidation' AND status IN ('accepting', 'accepted', 'rejected')
|
|
174
|
+
ON CONFLICT(id) DO UPDATE SET status = excluded.status, note_id = excluded.note_id`);
|
|
175
|
+
}
|
|
153
176
|
private prune(): void {
|
|
154
177
|
this.db.transaction(() => {
|
|
155
178
|
const now = this.now();
|
|
179
|
+
this.retainConsolidationReviews();
|
|
156
180
|
// Candidates (AC4, no candidate expires unshown): by age only a decided one, or a pending one that was shown and has had its grace period, goes. By count three tiers decide who is
|
|
157
181
|
// kept: unshown pending first, then shown pending, then decided, so a cap eviction takes decided rows before a shown one (which is owed its grace) and shown ones before an unseen one.
|
|
158
182
|
this.db.query("DELETE FROM candidates WHERE at <= ? AND (status IN ('accepted', 'rejected') OR (shown_at IS NOT NULL AND shown_at <= ?))").run(now - CANDIDATE_RETENTION.ageMs, now - CANDIDATE_SHOWN_GRACE_MS);
|
|
@@ -174,10 +198,19 @@ export class CandidateStore {
|
|
|
174
198
|
// Prune, count, insert and prune again are ONE immediate transaction: concurrent proposers (several agents) cannot all pass the count and then let the cap evict an unseen candidate.
|
|
175
199
|
return this.db.transaction((): string => {
|
|
176
200
|
this.prune();
|
|
201
|
+
if (input.kind === "consolidation" && this.reviewedConsolidation(id, input.project)) return id;
|
|
177
202
|
// AC4: a new candidate never pushes an unshown one out. With the cap's worth waiting the person has to review first, and the refusal says so (a named error an MCP tool can map).
|
|
178
203
|
const waiting = (this.db.query("SELECT COUNT(*) AS n FROM candidates WHERE project = ? AND status IN ('pending', 'accepting') AND shown_at IS NULL").get(input.project) as { n: number }).n;
|
|
179
204
|
const known = this.db.query("SELECT 1 FROM candidates WHERE id = ?").get(id);
|
|
180
205
|
if (!known && waiting >= CANDIDATE_RETENTION.count) throw new CandidateQueueFull(waiting);
|
|
206
|
+
if (!known && input.kind === "consolidation") {
|
|
207
|
+
const allocated = this.db.query(`SELECT (SELECT count(*) FROM consolidation_reviews WHERE project = ?) +
|
|
208
|
+
(SELECT count(*) FROM candidates c WHERE project = ? AND kind = 'consolidation' AND status IN ('pending', 'accepting')
|
|
209
|
+
AND NOT EXISTS (SELECT 1 FROM consolidation_reviews r WHERE r.id = c.id)) AS n`).get(input.project, input.project) as { n: number };
|
|
210
|
+
// ponytail: 10,000 lasting review identities per project; refuse new proposals instead of forgetting a person's decision.
|
|
211
|
+
// Upgrade with an explicit person-audited receipt archive before increasing this bounded sidecar capacity.
|
|
212
|
+
if (allocated.n >= CONSOLIDATION_REVIEW_LIMIT) throw new ConsolidationReviewLimit(input.project, allocated.n);
|
|
213
|
+
}
|
|
181
214
|
this.db.query("INSERT OR IGNORE INTO candidates (id, project, kind, payload, at, status) VALUES (?, ?, ?, ?, ?, 'pending')").run(id, input.project, input.kind, payload, this.now());
|
|
182
215
|
this.prune();
|
|
183
216
|
return id;
|
|
@@ -189,6 +222,11 @@ export class CandidateStore {
|
|
|
189
222
|
const row = this.db.query("SELECT * FROM candidates WHERE id = ? AND project IN (SELECT value FROM json_each(?))").get(id, this.scope(project)) as Parameters<CandidateStore["decode"]>[0] | null;
|
|
190
223
|
return row ? this.decode(row) : undefined;
|
|
191
224
|
}
|
|
225
|
+
reviewedConsolidation(id: string, project: string): ConsolidationReview | undefined {
|
|
226
|
+
if (!candidateId(id) || !text(project, 400)) throw new Error("invalid candidate identity");
|
|
227
|
+
const row = this.db.query("SELECT status, note_id FROM consolidation_reviews WHERE id = ? AND project IN (SELECT value FROM json_each(?))").get(id, this.scope(project)) as { status: ConsolidationReview["status"]; note_id: string | null } | null;
|
|
228
|
+
return row ? { status: row.status, acceptedNoteId: row.note_id } : undefined;
|
|
229
|
+
}
|
|
192
230
|
list(filter: CandidateFilter): Candidate[] {
|
|
193
231
|
if (!text(filter.project, 400) || filter.status !== undefined && !["pending", "accepting", "accepted", "rejected", "all"].includes(filter.status) || filter.kind !== undefined && !["correction", "fail", "decay", "memory", "promotion", "consolidation", "model-answer"].includes(filter.kind)) throw new Error("invalid candidate filter");
|
|
194
232
|
const limit = filter.limit ?? CANDIDATE_RETENTION.count;
|
|
@@ -221,6 +259,7 @@ export class CandidateStore {
|
|
|
221
259
|
if (!candidate) throw new Error("candidate unavailable");
|
|
222
260
|
if (candidate.status === "accepted" || candidate.status === "accepting") throw new Error("candidate acceptance already started; inspect its ordinary note before any further action");
|
|
223
261
|
this.db.query("UPDATE candidates SET status = 'rejected' WHERE id = ? AND project IN (SELECT value FROM json_each(?)) AND status = 'pending'").run(id, this.scope(context.project));
|
|
262
|
+
if (candidate.input.kind === "consolidation") this.retainConsolidationReviews();
|
|
224
263
|
}).immediate();
|
|
225
264
|
}
|
|
226
265
|
accept(id: string, context: CandidateAcceptance): CandidateAcceptanceResult {
|
|
@@ -321,7 +360,10 @@ export class CandidateStore {
|
|
|
321
360
|
// report every confirmed effect; recovery requires person inspection before any replay.
|
|
322
361
|
throw Error(`consolidation interrupted: inspect note ${noteId}; superseded ${applied.join(", ") || "none"}; remaining or uncertain ${data.targets.filter(target => !applied.includes(target.id)).map(target => target.id).join(", ")}`);
|
|
323
362
|
}
|
|
324
|
-
this.db.
|
|
363
|
+
this.db.transaction(() => {
|
|
364
|
+
this.db.query("UPDATE candidates SET status = 'accepted', note_id = ? WHERE id = ? AND project IN (SELECT value FROM json_each(?))").run(noteId, candidate.id, this.scope(context.project));
|
|
365
|
+
this.db.query("UPDATE consolidation_reviews SET status = 'accepted', note_id = ? WHERE id = ? AND project IN (SELECT value FROM json_each(?))").run(noteId, candidate.id, this.scope(context.project));
|
|
366
|
+
}).immediate();
|
|
325
367
|
return { candidateId: candidate.id, noteId, alreadyAccepted: false };
|
|
326
368
|
});
|
|
327
369
|
});
|
|
@@ -379,6 +421,7 @@ export class CandidateStore {
|
|
|
379
421
|
if (current.status !== "pending") throw Error("candidate is rejected or acceptance was interrupted; no automatic replay");
|
|
380
422
|
before?.();
|
|
381
423
|
this.db.query("UPDATE candidates SET status = 'accepting' WHERE id = ? AND project IN (SELECT value FROM json_each(?))").run(id, this.scope(project));
|
|
424
|
+
if (current.input.kind === "consolidation") this.retainConsolidationReviews();
|
|
382
425
|
return undefined;
|
|
383
426
|
}).immediate();
|
|
384
427
|
}
|
package/src/memory/commands.ts
CHANGED
|
@@ -17,6 +17,8 @@ export interface MemoryCommandOptions {
|
|
|
17
17
|
person?: boolean;
|
|
18
18
|
/** Standalone person CLI opt-in. Hosts omit this to retain their screened substring recall. */
|
|
19
19
|
rankedSearch?: boolean;
|
|
20
|
+
/** Standalone person CLI opt-in; requires rankedSearch and person intent. */
|
|
21
|
+
explainSearch?: boolean;
|
|
20
22
|
/** Defaults to amem's own names. */
|
|
21
23
|
names?: CommandNames;
|
|
22
24
|
/** Startup-bound canonical project supplied by a host; no model-facing selector. */
|
|
@@ -70,6 +72,10 @@ export async function rawMemoryCommand(args: string[], cwd: string, opts: Memory
|
|
|
70
72
|
const names = opts.names ?? AMEM_NAMES, usage = memoryUsage(names.memory);
|
|
71
73
|
if (args.some(arg => arg === "--history" || arg === "--as-of" || arg.startsWith("--history=") || arg.startsWith("--as-of=")) && opts.person !== true) fail("historical recall requires authenticated person intent from the host");
|
|
72
74
|
if (opts.rankedSearch && opts.person !== true) fail("ranked CLI search requires person intent");
|
|
75
|
+
if (opts.explainSearch && (opts.person !== true || opts.rankedSearch !== true)) fail("score explanation requires ranked search and person intent");
|
|
76
|
+
if (opts.explainSearch && sub !== "search") fail("score explanation applies only to search");
|
|
77
|
+
if (args.includes("--explain") && (opts.person !== true || opts.rankedSearch !== true || opts.explainSearch !== true)) fail("score explanation requires ranked search and person intent");
|
|
78
|
+
for (let index = rest.indexOf("--explain"); index >= 0; index = rest.indexOf("--explain")) rest.splice(index, 1);
|
|
73
79
|
const store = opts.store ?? opts.open();
|
|
74
80
|
const actor = "user";
|
|
75
81
|
const project = opts.project ?? memoryProjectKey(cwd);
|
|
@@ -93,12 +99,14 @@ export async function rawMemoryCommand(args: string[], cwd: string, opts: Memory
|
|
|
93
99
|
if (limit && !(Number(limit) > 0)) fail("--limit takes a positive number");
|
|
94
100
|
const q = sub === "search" ? rest.join(" ") : undefined;
|
|
95
101
|
const ranked = opts.rankedSearch && q?.trim() && !history && asOf === undefined;
|
|
102
|
+
if (opts.explainSearch && (!ranked || !q?.trim())) fail("--explain requires a non-empty current-project ranked search without --history or --as-of");
|
|
96
103
|
const state = recallRows(store, project, { person: opts.person, history, asOf, query: ranked ? undefined : q, limit: Number.MAX_SAFE_INTEGER });
|
|
97
104
|
// Retained history has its own snapshot; the current index cannot rank it. Empty queries keep list semantics.
|
|
98
105
|
// ponytail: fetch up to the current curated-view count before display caps, so metadata/readable exclusions consume no slots.
|
|
99
106
|
// Upgrade to paged ranked views if this full-pool person read reaches the R3 ceiling.
|
|
100
|
-
const
|
|
101
|
-
|
|
107
|
+
const rankedResult = ranked ? store.withDeferredPromotions(() => store.search(q!, { project, includeCapture: false, limit: Math.max(1, state.views.length), ...(opts.explainSearch ? { explain: true } : {}) })) : undefined;
|
|
108
|
+
const scoreDetails = opts.explainSearch ? new Map(rankedResult?.hits.flatMap(hit => hit.view && hit.scoreDetails ? [[hit.view.id, hit.scoreDetails] as const] : []) ?? []) : undefined;
|
|
109
|
+
const candidates = rankedResult ? rankedResult.hits.flatMap(hit => hit.view ? [hit.view] : []) : state.views;
|
|
102
110
|
const all = candidates.filter(view => (!tier || view.tier === tier) && (!kind || view.kind === kind) && (!peer || view.peer === peer) && (!machine || view.machine === machine) && (!since || Date.parse(view.at) >= Date.parse(since)));
|
|
103
111
|
const readable = opts.readable(all), rows = readable.slice(0, limit ? Number(limit) : 50);
|
|
104
112
|
ready(rows);
|
|
@@ -108,10 +116,12 @@ export async function rawMemoryCommand(args: string[], cwd: string, opts: Memory
|
|
|
108
116
|
const result = { at: state.at, entries: rows.map(decorated), forgotten, relations: state.relations.filter(edge => publicIds.has(edge.from) && publicIds.has(edge.to)), incomplete: emitted.incomplete || readable.length > rows.length || forgotten.length < allForgotten.length, notices: emitted.notices };
|
|
109
117
|
return json ? JSON.stringify(result, null, 2) : [`retained memory as of ${state.at}`, ...result.notices, ...rows.map(view => `${stale(view) ? "(stale?) " : ""}${one(view)}${view.expiresAt && Date.parse(view.expiresAt) <= Date.parse(state.at) ? ` (expired at ${view.expiresAt})` : ""}`), ...forgotten.map(row => `forgotten ${row.id} at ${row.at} (no retained text shown)`), ...(result.incomplete ? ["(retained state is incomplete)"] : [])].join("\n");
|
|
110
118
|
}
|
|
111
|
-
if (json) return JSON.stringify(rows, null, 2);
|
|
119
|
+
if (json) return JSON.stringify(opts.explainSearch ? rows.map(view => ({ ...view, ...(scoreDetails?.has(view.id) ? { scoreDetails: scoreDetails.get(view.id) } : {}) })) : rows, null, 2);
|
|
112
120
|
const counted = opts.readable(store.withDeferredPromotions(() => store.list({ project, limit: Number.MAX_SAFE_INTEGER, includeSuperseded: true })));
|
|
113
121
|
const withheld = all.length - readable.length;
|
|
114
|
-
return [`memory for ${project} on machine ${store.machine.label}: ${counted.filter(view => view.tier === "long").length} long-term, ${counted.filter(view => view.tier === "short").length} short-term`, ...(rows.length ?
|
|
122
|
+
return [`memory for ${project} on machine ${store.machine.label}: ${counted.filter(view => view.tier === "long").length} long-term, ${counted.filter(view => view.tier === "short").length} short-term`, ...(rows.length ? opts.explainSearch
|
|
123
|
+
? rows.flatMap(view => [`${stale(view) ? "(stale?) " : ""}${one(view)}`, ...(scoreDetails?.has(view.id) ? [`score: ${JSON.stringify(scoreDetails.get(view.id))}`] : [])])
|
|
124
|
+
: rows.map(view => `${stale(view) ? "(stale?) " : ""}${one(view)}`) : ["(nothing)"]),
|
|
115
125
|
...(withheld ? [`(${withheld} withheld: they match a PII pattern; the dashboard shows them)`] : [])].join("\n");
|
|
116
126
|
}
|
|
117
127
|
case "show": {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CandidateStore, candidateInputId, candidateTargetRevision, type CandidateInput } from "./candidates.ts";
|
|
1
|
+
import { CandidateStore, ConsolidationReviewLimit, candidateInputId, candidateTargetRevision, type CandidateInput } from "./candidates.ts";
|
|
2
2
|
import { consolidationClusters } from "./consolidation-clusters.ts";
|
|
3
3
|
import { memoryPolicySnapshot, type StandalonePolicy } from "./policy.ts";
|
|
4
4
|
import { recallViewIsLive } from "./recall.ts";
|
|
@@ -20,7 +20,10 @@ export interface ConsolidationReport {
|
|
|
20
20
|
model?: ModelConsolidationReport;
|
|
21
21
|
}
|
|
22
22
|
export class ConsolidationProposalError extends Error {
|
|
23
|
-
constructor(readonly proposed: string[]
|
|
23
|
+
constructor(readonly proposed: string[], readonly refusal?: ConsolidationReviewLimit) {
|
|
24
|
+
super(`consolidation proposal interrupted; candidates already queued: ${proposed.join(", ") || "none"}; no curated notes changed${refusal ? `; ${refusal.message}` : ""}`);
|
|
25
|
+
this.name = "ConsolidationProposalError";
|
|
26
|
+
}
|
|
24
27
|
}
|
|
25
28
|
const modelAnswer = (row: View) => row.peer === "amem-model" || row.text.startsWith("Model answer:");
|
|
26
29
|
/** On-demand candidate writes only. Every source is screened once under one policy and source revision. */
|
|
@@ -67,12 +70,12 @@ export async function proposeConsolidation(store: MemoryStore, candidates: Candi
|
|
|
67
70
|
// put() is INSERT OR IGNORE: a candidate the person already accepted or rejected keeps its status, so it is neither written again nor reported as queued.
|
|
68
71
|
for (const input of prepared) {
|
|
69
72
|
check();
|
|
70
|
-
const known = candidates.get(
|
|
71
|
-
if (known && known.status !== "pending") { report.alreadyReviewed++; continue; }
|
|
72
|
-
|
|
73
|
+
const id = candidateInputId(input), known = candidates.get(id, project);
|
|
74
|
+
if (known && known.status !== "pending" || candidates.reviewedConsolidation(id, project)) { report.alreadyReviewed++; continue; }
|
|
75
|
+
candidates.put(input, "public");
|
|
73
76
|
report.candidates.push({ id, action: input.action, sources: input.targets.map(target => `mem:${target.id}`) });
|
|
74
77
|
}
|
|
75
78
|
if (options.model !== undefined) report.model = await proposeModelConsolidations(store, candidates, synced, options.model, { project, actor: options.actor, check, policy });
|
|
76
|
-
} catch { throw new ConsolidationProposalError(report.candidates.map(row => row.id)); }
|
|
79
|
+
} catch (error) { throw new ConsolidationProposalError(report.candidates.map(row => row.id), error instanceof ConsolidationReviewLimit ? error : undefined); }
|
|
77
80
|
return report;
|
|
78
81
|
}
|
|
@@ -8,9 +8,14 @@ export interface ConsolidationCluster { action: ConsolidationAction; sources: Vi
|
|
|
8
8
|
export const CONSOLIDATION_SOURCE_LIMIT = 256, CONSOLIDATION_PROPOSAL_LIMIT = 32;
|
|
9
9
|
const words = (row: View) => new Set((`${row.title ?? ""} ${row.text}`.toLowerCase().match(/[\p{L}\p{N}_]{3,}/gu) ?? []).filter(word => !["the", "and", "for", "with", "from", "this", "that", "approach", "symptom", "cause", "evidence", "retry", "when"].includes(word)).slice(0, 128));
|
|
10
10
|
/** A note written by accepting a consolidation candidate. Proposing over one would be a summary of summaries. */
|
|
11
|
-
export
|
|
12
|
-
|
|
13
|
-
const
|
|
11
|
+
export function isAcceptedConsolidation(row: Pick<View, "text">): boolean {
|
|
12
|
+
const end = row.text.indexOf("\n\n"); if (end < 0) return false;
|
|
13
|
+
const lines = row.text.slice(0, end).split("\n");
|
|
14
|
+
return lines.length === 3 && lines[0]!.startsWith(ACCEPTED_CONSOLIDATION_PREFIX)
|
|
15
|
+
&& /^[a-f0-9]{64}$/.test(lines[0]!.slice(ACCEPTED_CONSOLIDATION_PREFIX.length))
|
|
16
|
+
&& /^Action: (?:merge|supersession|recurring-fix)$/.test(lines[1]!)
|
|
17
|
+
&& /^Sources: mem:[a-f0-9]{32}(?:, mem:[a-f0-9]{32}){0,7}$/.test(lines[2]!);
|
|
18
|
+
}
|
|
14
19
|
const paths = (row: View) => row.about.filter(node => node.startsWith("path:"));
|
|
15
20
|
const sharedPath = (a: View, b: View) => paths(a).some(node => paths(b).includes(node));
|
|
16
21
|
const overlap = (a: Set<string>, b: Set<string>) => [...a].filter(word => b.has(word)).length / Math.max(1, Math.min(a.size, b.size));
|
|
@@ -18,7 +23,7 @@ const overlap = (a: Set<string>, b: Set<string>) => [...a].filter(word => b.has(
|
|
|
18
23
|
/** Bounded deterministic proposals over already-screened rows. Never mutates a note or runs a model. */
|
|
19
24
|
export function consolidationClusters(input: readonly View[], project: string, now: number): { clusters: ConsolidationCluster[]; truncated: boolean; engine: "fts5" | "substring" } {
|
|
20
25
|
if (!project.trim() || !Number.isFinite(now)) throw Error("consolidation context unavailable");
|
|
21
|
-
const eligible = input.filter(row => row.project === project && recallViewIsLive(row, now))
|
|
26
|
+
const eligible = input.filter(row => row.project === project && recallViewIsLive(row, now) && !isAcceptedConsolidation(row))
|
|
22
27
|
.sort((a, b) => b.at.localeCompare(a.at) || a.id.localeCompare(b.id));
|
|
23
28
|
const rows = [...new Map(eligible.map(row => [row.id, row])).values()].slice(0, CONSOLIDATION_SOURCE_LIMIT);
|
|
24
29
|
const index = new SearchIndex(":memory:"), clusters: ConsolidationCluster[] = [], seen = new Set<string>();
|
|
@@ -36,15 +41,15 @@ export function consolidationClusters(input: readonly View[], project: string, n
|
|
|
36
41
|
for (let i = 0; i < rows.length; i++) {
|
|
37
42
|
const row = rows[i]!, candidates = index.available ? new Set(index.query([...terms.get(row.id)!].slice(0, 32).join(" "), rows.length, project).map(hit => hit.ref)) : undefined;
|
|
38
43
|
for (const peer of rows.slice(i + 1)) {
|
|
39
|
-
if (row.kind !== peer.kind
|
|
44
|
+
if (row.kind !== peer.kind) continue;
|
|
40
45
|
const lexical = overlap(terms.get(row.id)!, terms.get(peer.id)!);
|
|
41
46
|
if (row.text === peer.text) add("supersession", [row, peer], "Identical text; review one replacement for both originals.");
|
|
42
47
|
else if ((!candidates || candidates.has(peer.id)) && lexical >= 0.5 && sharedPath(row, peer)) add("merge", [row, peer], "Shared path and overlapping indexed terms; no equivalence is assumed.");
|
|
43
48
|
}
|
|
44
49
|
// A retained supersession chain is context, never a reason to resurrect an old body.
|
|
45
|
-
if (row.supersedes.length >= 2
|
|
50
|
+
if (row.supersedes.length >= 2) add("merge", [row], "Current head already replaces multiple originals; review a chain summary.");
|
|
46
51
|
}
|
|
47
|
-
const failRows = rows.filter(row => row.kind === "fail"
|
|
52
|
+
const failRows = rows.filter(row => row.kind === "fail"), decisions = rows.filter(row => row.kind === "decision" || row.kind === "contract");
|
|
48
53
|
for (let i = 0; i < failRows.length; i++) for (const peer of failRows.slice(i + 1)) {
|
|
49
54
|
const fail = failRows[i]!;
|
|
50
55
|
if (!sharedPath(fail, peer) || overlap(terms.get(fail.id)!, terms.get(peer.id)!) < 0.5) continue;
|
|
@@ -79,18 +79,22 @@ export function evalSeeded(set: EvalSet, ks?: readonly number[], rank?: RankFn):
|
|
|
79
79
|
const base = join(dir, "memory");
|
|
80
80
|
mkdirSync(base, { recursive: true });
|
|
81
81
|
// One entry per day, so the recency weight (a 30-day scale) is exercised, long-tier entries first and short-tier ones last: a short-term note leaves the index 14 days after it is
|
|
82
|
-
// written, and the clock stops a day after the last entry, so the short ones written in that window are live when the set is searched (a set with more short entries than that
|
|
82
|
+
// written, and the undated clock stops a day after the last entry, so the short ones written in that window are live when the set is searched (a set with more short entries than that
|
|
83
83
|
// window holds loses its oldest, and a question naming one is then reported as not in the store, like any other missing entry).
|
|
84
|
+
// Explicit dates override only their entry; undated entries retain their legacy file-order day.
|
|
85
|
+
// Search a day after the latest actual seed time unless the set supplies its own clock.
|
|
84
86
|
let current = EPOCH;
|
|
85
87
|
store = new MemoryStore({ memoryDir: base, now: () => current, screen: () => "keep" });
|
|
86
88
|
const ids = new Map<string, string>();
|
|
87
89
|
const ordered = [...set.entries.filter(entry => entry.tier !== "short"), ...set.entries.filter(entry => entry.tier === "short")];
|
|
90
|
+
let latest = -Infinity;
|
|
88
91
|
for (const [index, entry] of ordered.entries()) {
|
|
89
|
-
current = EPOCH + index * DAY;
|
|
92
|
+
current = entry.at === undefined ? EPOCH + index * DAY : Date.parse(entry.at);
|
|
93
|
+
latest = Math.max(latest, current);
|
|
90
94
|
ids.set(entry.key, store.note({ op: "note", project: PROJECT, scope: "project", tier: entry.tier ?? "long", kind: entry.kind ?? "finding", peer: "eval", source: "person",
|
|
91
95
|
...(entry.title ? { title: entry.title } : {}), text: entry.text, about: entry.about ?? [] }, entry.tier === "short"));
|
|
92
96
|
}
|
|
93
|
-
current =
|
|
97
|
+
current = set.now === undefined ? latest + DAY : Date.parse(set.now);
|
|
94
98
|
const window = Math.max(...(ks ?? [10]));
|
|
95
99
|
const search = rank ? { rank: (question: string) => rank(store!, question, window), engine: () => "custom" } : ranker(store, PROJECT, window);
|
|
96
100
|
// Resolved through `store.get`, so a note that has already expired is absent, not silently present: coverage stays separate from ranking.
|
package/src/memory/eval.ts
CHANGED
|
@@ -12,10 +12,10 @@ export const MAX_QUESTIONS = 500, MAX_ENTRIES = 200, MAX_QUESTION_CHARS = 500, M
|
|
|
12
12
|
export const DEFAULT_KS: readonly number[] = [1, 5, 10];
|
|
13
13
|
|
|
14
14
|
export const EVAL_KINDS = ["decision", "finding", "contract", "fail"] as const;
|
|
15
|
-
export interface EvalEntry { key: string; title?: string; text: string; kind?: (typeof EVAL_KINDS)[number]; tier?: "short" | "long"; about?: string[] }
|
|
15
|
+
export interface EvalEntry { key: string; title?: string; text: string; kind?: (typeof EVAL_KINDS)[number]; tier?: "short" | "long"; about?: string[]; /** Explicit seed date; absent keeps the legacy positional day. */ at?: string }
|
|
16
16
|
/** `tag` is an optional label for what a question probes (a test counts them); it never changes the scoring. */
|
|
17
17
|
export interface EvalQuestion { q: string; expect: string[]; tag?: string }
|
|
18
|
-
export interface EvalSet { schema: typeof EVAL_SCHEMA; entries?: EvalEntry[]; questions: EvalQuestion[] }
|
|
18
|
+
export interface EvalSet { schema: typeof EVAL_SCHEMA; entries?: EvalEntry[]; questions: EvalQuestion[]; /** Search clock for a self-contained set; defaults to one day after its latest seed. */ now?: string }
|
|
19
19
|
|
|
20
20
|
const fail = (message: string): never => { throw new Error(`invalid eval set: ${message}`); };
|
|
21
21
|
const object = (value: unknown, what: string): Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : fail(`${what} must be an object`);
|
|
@@ -24,17 +24,25 @@ function exact(row: Record<string, unknown>, allowed: readonly string[], what: s
|
|
|
24
24
|
}
|
|
25
25
|
const text = (value: unknown, what: string, max: number): string => typeof value === "string" && value.trim() !== "" && value.length <= max && !/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/.test(value) ? value : fail(`${what} must be a non-empty string of at most ${max} characters`);
|
|
26
26
|
|
|
27
|
+
/** ISO calendar date or canonical UTC timestamp; reject Date.parse's rollover and locale-dependent forms. */
|
|
28
|
+
function date(value: unknown, what: string): string {
|
|
29
|
+
if (typeof value !== "string" || !/^[0-9]{4}-[0-9]{2}-[0-9]{2}(?:T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:[.][0-9]{3})?Z)?$/.test(value)) return fail(`${what} must be an ISO date or UTC timestamp`);
|
|
30
|
+
const ms = Date.parse(value), normalized = value.length === 10 ? value + "T00:00:00.000Z" : value.length === 20 ? value.slice(0, -1) + ".000Z" : value;
|
|
31
|
+
if (!Number.isFinite(ms) || new Date(ms).toISOString() !== normalized) return fail(`${what} must be a valid ISO date or UTC timestamp`);
|
|
32
|
+
return value;
|
|
33
|
+
}
|
|
34
|
+
|
|
27
35
|
/** Strict, bounded parse of untrusted JSON. Unknown keys are errors, so a typo cannot silently change what is measured. */
|
|
28
36
|
export function parseEvalSet(raw: unknown): EvalSet {
|
|
29
37
|
const root = object(raw, "the set");
|
|
30
|
-
exact(root, ["schema", "entries", "questions"], "the set");
|
|
38
|
+
exact(root, ["schema", "entries", "questions", "now"], "the set");
|
|
31
39
|
if (root.schema !== EVAL_SCHEMA) fail(`schema must be "${EVAL_SCHEMA}"`);
|
|
32
40
|
const entries: EvalEntry[] = [], keys = new Set<string>();
|
|
33
41
|
if (root.entries !== undefined) {
|
|
34
42
|
if (!Array.isArray(root.entries) || !root.entries.length || root.entries.length > MAX_ENTRIES) fail(`entries must hold 1 to ${MAX_ENTRIES} items when present`);
|
|
35
43
|
for (const [index, item] of (root.entries as unknown[]).entries()) {
|
|
36
44
|
const row = object(item, `entries[${index}]`);
|
|
37
|
-
exact(row, ["key", "title", "text", "kind", "tier", "about"], `entries[${index}]`);
|
|
45
|
+
exact(row, ["key", "title", "text", "kind", "tier", "about", "at"], `entries[${index}]`);
|
|
38
46
|
const key = text(row.key, `entries[${index}].key`, 80);
|
|
39
47
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(key)) fail(`entries[${index}].key may use letters, digits, ".", "_" and "-"`);
|
|
40
48
|
if (keys.has(key)) fail(`entries[${index}].key "${key}" repeats`);
|
|
@@ -42,9 +50,10 @@ export function parseEvalSet(raw: unknown): EvalSet {
|
|
|
42
50
|
const kind = row.kind === undefined ? undefined : (EVAL_KINDS as readonly unknown[]).includes(row.kind) ? row.kind as EvalEntry["kind"] : fail(`entries[${index}].kind must be one of ${EVAL_KINDS.join(", ")}`);
|
|
43
51
|
const tier = row.tier === undefined ? undefined : row.tier === "short" || row.tier === "long" ? row.tier : fail(`entries[${index}].tier must be short or long`);
|
|
44
52
|
const about = row.about === undefined ? undefined : Array.isArray(row.about) && row.about.length <= 20 ? (row.about as unknown[]).map((value, at) => text(value, `entries[${index}].about[${at}]`, 200)) : fail(`entries[${index}].about must be a list of at most 20 strings`);
|
|
45
|
-
entries.push({ key, text: text(row.text, `entries[${index}].text`, MAX_ENTRY_CHARS), ...(row.title === undefined ? {} : { title: text(row.title, `entries[${index}].title`, 200) }), ...(kind ? { kind } : {}), ...(tier ? { tier } : {}), ...(about ? { about } : {}) });
|
|
53
|
+
entries.push({ key, text: text(row.text, `entries[${index}].text`, MAX_ENTRY_CHARS), ...(row.title === undefined ? {} : { title: text(row.title, `entries[${index}].title`, 200) }), ...(kind ? { kind } : {}), ...(tier ? { tier } : {}), ...(about ? { about } : {}), ...(row.at === undefined ? {} : { at: date(row.at, `entries[${index}].at`) }) });
|
|
46
54
|
}
|
|
47
55
|
}
|
|
56
|
+
if (root.now !== undefined && !entries.length) fail("now requires self-contained entries");
|
|
48
57
|
if (!Array.isArray(root.questions) || !root.questions.length || root.questions.length > MAX_QUESTIONS) fail(`questions must hold 1 to ${MAX_QUESTIONS} items`);
|
|
49
58
|
const questions = (root.questions as unknown[]).map((item, index): EvalQuestion => {
|
|
50
59
|
const row = object(item, `questions[${index}]`);
|
|
@@ -58,7 +67,7 @@ export function parseEvalSet(raw: unknown): EvalSet {
|
|
|
58
67
|
}
|
|
59
68
|
return { q: text(row.q, `questions[${index}].q`, MAX_QUESTION_CHARS), expect, ...(tag ? { tag } : {}) };
|
|
60
69
|
});
|
|
61
|
-
return { schema: EVAL_SCHEMA, ...(entries.length ? { entries } : {}), questions };
|
|
70
|
+
return { schema: EVAL_SCHEMA, ...(entries.length ? { entries } : {}), questions, ...(root.now === undefined ? {} : { now: date(root.now, "now") }) };
|
|
62
71
|
}
|
|
63
72
|
|
|
64
73
|
/** `--k 1,5,10`: ascending, unique, 1 to MAX_K. */
|
|
@@ -206,7 +206,14 @@ function title(source: ExternalMemorySource, file: SourceFile, record: RawRecord
|
|
|
206
206
|
}
|
|
207
207
|
|
|
208
208
|
function portable(path: string): string { return path.split(sep).join("/"); }
|
|
209
|
-
|
|
209
|
+
/** Claude Code 2.1.311's project-path encoding, including its long-name suffix. */
|
|
210
|
+
function claudeProjectDirectory(cwd: string): string {
|
|
211
|
+
const path = realPath(cwd), encoded = path.replace(/[^a-zA-Z0-9]/g, "-");
|
|
212
|
+
if (encoded.length <= 200) return encoded;
|
|
213
|
+
let hash = 0;
|
|
214
|
+
for (let index = 0; index < path.length; index++) hash = (Math.imul(hash, 31) + path.charCodeAt(index)) | 0;
|
|
215
|
+
return `${encoded.slice(0, 200)}-${Math.abs(hash).toString(36)}`;
|
|
216
|
+
}
|
|
210
217
|
|
|
211
218
|
export async function importExternalMemory(opts: {
|
|
212
219
|
base: string;
|
package/src/memory/platform.ts
CHANGED
|
@@ -103,18 +103,18 @@ export function memoryGitEnv(env: NodeJS.ProcessEnv = process.env, options: { re
|
|
|
103
103
|
/** spawnSync form of the same call: the env option is merged with memoryGitEnv, everything else passes through. */
|
|
104
104
|
export function memoryGitSync(args: string[], options: SpawnSyncOptionsWithStringEncoding): SpawnSyncReturns<string> {
|
|
105
105
|
const argv = memoryGitArgv(args);
|
|
106
|
-
|
|
106
|
+
const timeout = gitTimeoutContext?.getStore();
|
|
107
|
+
return childProcess().spawnSync(argv[0]!, argv.slice(1), { ...(timeout !== undefined ? { timeout, killSignal: "SIGKILL" as const } : {}), ...options, env: memoryGitEnv(options.env ?? process.env) });
|
|
107
108
|
}
|
|
108
|
-
let
|
|
109
|
+
let gitTimeoutContext: import("node:async_hooks").AsyncLocalStorage<number> | undefined;
|
|
109
110
|
/**
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
* ponytail: one global instead of a parameter on every caller between doctor and its git calls (project key, repository root, visibility). Upgrade: thread an options argument through them.
|
|
111
|
+
* Bound `memoryGitSync` calls inside this async operation unless a call supplies its own timeout.
|
|
112
|
+
* Nested and concurrent operations keep independent contexts; unrelated host calls retain their own options.
|
|
113
|
+
* Load async_hooks only when a caller requests a deadline, keeping path-only hook imports unchanged.
|
|
114
114
|
*/
|
|
115
115
|
export async function withGitTimeout<T>(ms: number, work: () => Promise<T>): Promise<T> {
|
|
116
|
-
|
|
117
|
-
|
|
116
|
+
gitTimeoutContext ??= new (require("node:async_hooks").AsyncLocalStorage)();
|
|
117
|
+
return gitTimeoutContext!.run(ms, work);
|
|
118
118
|
}
|
|
119
119
|
|
|
120
120
|
/**
|
|
@@ -15,7 +15,7 @@ const NOTE_KEYS = new Set(["schema", "id", "at", "machine", "op", "project", "sc
|
|
|
15
15
|
const OPERATIONS_WARNING = "Supersede, demote, settle and forget operations are not exported; re-import restores authored note rows only.";
|
|
16
16
|
|
|
17
17
|
type ImportedNote = { id: string; at: string; machine: string; note: NoteInput };
|
|
18
|
-
type Provenance = { source: Record<string, number>; peer: Record<string, number>; scope: Record<string, number> };
|
|
18
|
+
type Provenance = { source: Record<string, number>; peer: Record<string, number>; scope: Record<string, number>; machine: Record<string, number> };
|
|
19
19
|
export interface ProjectImportResult {
|
|
20
20
|
applied: boolean;
|
|
21
21
|
project: string;
|
|
@@ -46,13 +46,18 @@ function increment(counts: Map<string, number>, value: string | undefined): void
|
|
|
46
46
|
}
|
|
47
47
|
|
|
48
48
|
function provenance(notes: Iterable<ImportedNote>): Provenance {
|
|
49
|
-
const source = new Map<string, number>(), peer = new Map<string, number>(), scope = new Map<string, number>();
|
|
50
|
-
for (const { note } of notes) {
|
|
49
|
+
const source = new Map<string, number>(), peer = new Map<string, number>(), scope = new Map<string, number>(), machine = new Map<string, number>();
|
|
50
|
+
for (const { note, machine: origin } of notes) {
|
|
51
51
|
increment(source, note.source);
|
|
52
52
|
increment(peer, note.peer);
|
|
53
53
|
increment(scope, note.scope);
|
|
54
|
+
increment(machine, origin);
|
|
54
55
|
}
|
|
55
|
-
return { source: Object.fromEntries(source), peer: Object.fromEntries(peer), scope: Object.fromEntries(scope) };
|
|
56
|
+
return { source: Object.fromEntries(source), peer: Object.fromEntries(peer), scope: Object.fromEntries(scope), machine: Object.fromEntries(machine) };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function relationTargets(imported: ImportedNote): string[] {
|
|
60
|
+
return [imported.note.supersedes, imported.note.promotedFrom].filter((target): target is string => typeof target === "string");
|
|
56
61
|
}
|
|
57
62
|
|
|
58
63
|
function unchangedAfterNormalization(note: NoteInput): boolean {
|
|
@@ -71,10 +76,10 @@ export async function importProject(store: MemoryStore, opts: {
|
|
|
71
76
|
const fd = openSync(opts.from, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
72
77
|
const stream = createReadStream(opts.from, { fd, encoding: "utf8" });
|
|
73
78
|
const result: ProjectImportResult = { applied: opts.confirm === true, project: opts.project, read: 0, added: 0, wouldAdd: 0,
|
|
74
|
-
skipped: { invalid: 0, policy: 0, other: 0 }, provenance: { source: {}, peer: {}, scope: {} }, warnings: [OPERATIONS_WARNING] };
|
|
79
|
+
skipped: { invalid: 0, policy: 0, other: 0 }, provenance: { source: {}, peer: {}, scope: {}, machine: {} }, warnings: [OPERATIONS_WARNING] };
|
|
75
80
|
let header = false, bytes = 0;
|
|
76
81
|
let aliases: string[] = [];
|
|
77
|
-
const pending = new Map<string, ImportedNote>(), accepted
|
|
82
|
+
const pending = new Map<string, ImportedNote>(), accepted = new Map<string, ImportedNote>();
|
|
78
83
|
try {
|
|
79
84
|
const stat = fstatSync(fd);
|
|
80
85
|
if (!stat.isFile() || stat.size > PROJECT_IMPORT_MAX_BYTES) throw new Error("project import needs a regular JSONL file of at most 64 MiB");
|
|
@@ -121,22 +126,34 @@ export async function importProject(store: MemoryStore, opts: {
|
|
|
121
126
|
if (!header) throw new Error("project import has no header");
|
|
122
127
|
|
|
123
128
|
for (const imported of pending.values()) {
|
|
124
|
-
if (
|
|
125
|
-
const target = imported.note[field as "supersedes" | "promotedFrom"] as string | null | undefined;
|
|
126
|
-
return target !== undefined && target !== null && !pending.has(target);
|
|
127
|
-
})) {
|
|
129
|
+
if (relationTargets(imported).some(target => !pending.has(target))) {
|
|
128
130
|
result.skipped.invalid++;
|
|
129
131
|
continue;
|
|
130
132
|
}
|
|
131
133
|
const screened = stripPrivateDeep(imported.note);
|
|
132
134
|
if (looksSecret(JSON.stringify(screened)) || !await opts.policy(screened)) { result.skipped.policy++; continue; }
|
|
133
|
-
accepted.
|
|
135
|
+
accepted.set(imported.id, imported);
|
|
136
|
+
}
|
|
137
|
+
// A policy-refused target cannot authorize its dependents. Propagate removals before any append,
|
|
138
|
+
// regardless of file order, in linear time even when a long relation chain is refused.
|
|
139
|
+
const dependents = new Map<string, string[]>(), refused: string[] = [];
|
|
140
|
+
for (const imported of accepted.values()) for (const target of relationTargets(imported)) {
|
|
141
|
+
const rows = dependents.get(target) ?? [];
|
|
142
|
+
rows.push(imported.id); dependents.set(target, rows);
|
|
143
|
+
if (!accepted.has(target)) refused.push(imported.id);
|
|
144
|
+
}
|
|
145
|
+
for (let at = 0; at < refused.length; at++) {
|
|
146
|
+
const id = refused[at]!;
|
|
147
|
+
if (!accepted.delete(id)) continue;
|
|
148
|
+
result.skipped.invalid++;
|
|
149
|
+
for (const dependent of dependents.get(id) ?? []) refused.push(dependent);
|
|
134
150
|
}
|
|
135
|
-
result.provenance = provenance(accepted);
|
|
151
|
+
result.provenance = provenance(accepted.values());
|
|
136
152
|
const existing = new Set(store.portableNotes(opts.project).map(note => note.id));
|
|
137
|
-
result.wouldAdd = accepted.filter(
|
|
153
|
+
result.wouldAdd = [...accepted.keys()].filter(id => !existing.has(id)).length;
|
|
138
154
|
if (opts.confirm === true) {
|
|
139
|
-
|
|
155
|
+
store.assertImportable([...accepted.values()].map(imported => imported.note));
|
|
156
|
+
for (const imported of accepted.values()) {
|
|
140
157
|
const written = store.importNote(imported.note, imported.at, { machine: imported.machine, expectedId: imported.id });
|
|
141
158
|
if (written.added) result.added++;
|
|
142
159
|
}
|
package/src/memory/search-fts.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { rankedScoreSql, rankedScoreSqlDetails, type ScoreFactors } from "./search-score.ts";
|
|
1
2
|
import { Database } from "bun:sqlite";
|
|
2
3
|
import { chmodSync, existsSync } from "node:fs";
|
|
3
4
|
import { join } from "node:path";
|
|
@@ -187,6 +188,8 @@ export class SearchIndex {
|
|
|
187
188
|
close(): void { this.db?.close(); }
|
|
188
189
|
}
|
|
189
190
|
|
|
191
|
+
const SCREENED_SCORE = rankedScoreSql(), SCREENED_SUBSTRING_SCORE = rankedScoreSql("-1.0");
|
|
192
|
+
|
|
190
193
|
/** Screened query-local corpus in SQLite-owned TEMP storage, removed even when the process exits. */
|
|
191
194
|
export class ScreenedSearchIndex<T> {
|
|
192
195
|
readonly available = fts5Available();
|
|
@@ -222,15 +225,21 @@ export class ScreenedSearchIndex<T> {
|
|
|
222
225
|
})();
|
|
223
226
|
}
|
|
224
227
|
|
|
225
|
-
best(query: string, limit: number, now: number, useFts = this.available): { value: T; score: number }[] {
|
|
228
|
+
best(query: string, limit: number, now: number, useFts = this.available, explain = false): { value: T; score: number; scoreDetails?: ScoreFactors }[] {
|
|
226
229
|
if (useFts && !ftsTerms(query)) return [];
|
|
227
|
-
const
|
|
230
|
+
const score = useFts ? SCREENED_SCORE : SCREENED_SUBSTRING_SCORE;
|
|
228
231
|
const instant = new Date(now).toISOString();
|
|
229
232
|
const where = "WHERE (expires IS NULL OR expires > ?)" + (useFts ? " AND docs MATCH ?" : "");
|
|
230
|
-
const
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
233
|
+
const factors = explain ? rankedScoreSqlDetails(useFts ? "bm25(docs)" : "-1.0", useFts) : undefined;
|
|
234
|
+
const columns = factors ? `, ${factors.rawBm25Rank} AS rawBm25Rank, ${factors.tierFactor} AS tierFactor, ${factors.recencyFactor} AS recencyFactor` : "";
|
|
235
|
+
const statement = this.db.query(`SELECT payload, ${score} AS score${columns} FROM docs ${where} ORDER BY score DESC, ref LIMIT ?`);
|
|
236
|
+
const rows = (explain ? statement.all(instant, instant, now, ...(useFts ? [ftsTerms(query)] : []), limit)
|
|
237
|
+
: statement.all(instant, now, ...(useFts ? [ftsTerms(query)] : []), limit)) as { payload: string; score: number; rawBm25Rank?: number | null; tierFactor?: number | null; recencyFactor?: number | null }[];
|
|
238
|
+
return rows.map(row => {
|
|
239
|
+
const hit: { value: T; score: number; scoreDetails?: ScoreFactors } = { value: JSON.parse(row.payload) as T, score: row.score };
|
|
240
|
+
if (explain) hit.scoreDetails = { rawBm25Rank: row.rawBm25Rank ?? null, tierFactor: row.tierFactor ?? null, recencyFactor: row.recencyFactor ?? null, finalScore: row.score, matchMode: useFts ? "fts5" : "substring" };
|
|
241
|
+
return hit;
|
|
242
|
+
});
|
|
234
243
|
}
|
|
235
244
|
|
|
236
245
|
close(): void {
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/** One fixed score recipe for numeric library ranking and native screened SQL. */
|
|
2
|
+
export interface ScoreFactors {
|
|
3
|
+
rawBm25Rank: number | null;
|
|
4
|
+
tierFactor: number | null;
|
|
5
|
+
recencyFactor: number | null;
|
|
6
|
+
finalScore: number | null;
|
|
7
|
+
matchMode: "fts5" | "substring";
|
|
8
|
+
}
|
|
9
|
+
export interface SearchScoreDetails extends ScoreFactors { source: "curated" | "capture" | "repo" }
|
|
10
|
+
|
|
11
|
+
type ScoreExpr = { op: "rank" | "tier" | "age" } | { op: "constant"; value: number }
|
|
12
|
+
| { op: "negative" | "reciprocal"; value: ScoreExpr }
|
|
13
|
+
| { op: "add" | "multiply" | "max"; left: ScoreExpr; right: ScoreExpr };
|
|
14
|
+
const MONTH_DAYS = 30;
|
|
15
|
+
const TIER_WEIGHTS: Readonly<Record<string, number>> = { long: 1, capture: 0.5 };
|
|
16
|
+
const DEFAULT_TIER_WEIGHT = 0.9;
|
|
17
|
+
const TIER_FACTOR: ScoreExpr = { op: "tier" };
|
|
18
|
+
const DENOMINATOR: ScoreExpr = { op: "add", left: { op: "constant", value: 1 }, right: { op: "max", left: { op: "constant", value: 0 }, right: { op: "age" } } };
|
|
19
|
+
const RECENCY_FACTOR: ScoreExpr = { op: "reciprocal", value: DENOMINATOR };
|
|
20
|
+
const SCORE: ScoreExpr = { op: "multiply", left: { op: "multiply", left: { op: "negative", value: { op: "rank" } }, right: TIER_FACTOR }, right: RECENCY_FACTOR };
|
|
21
|
+
// ponytail: this represents only the fixed rank/tier/age recipe, not arbitrary expressions;
|
|
22
|
+
// extend its operations with numeric/SQL parity tests when the ranking contract changes.
|
|
23
|
+
function numeric(expr: ScoreExpr, rank: number, tier: string, at: string, now: number): number {
|
|
24
|
+
switch (expr.op) {
|
|
25
|
+
case "rank": return rank;
|
|
26
|
+
case "tier": return Object.hasOwn(TIER_WEIGHTS, tier) ? TIER_WEIGHTS[tier]! : DEFAULT_TIER_WEIGHT;
|
|
27
|
+
case "age": return (now - Date.parse(at)) / (MONTH_DAYS * 24 * 3600_000);
|
|
28
|
+
case "constant": return expr.value;
|
|
29
|
+
case "negative": return -numeric(expr.value, rank, tier, at, now);
|
|
30
|
+
case "reciprocal": return 1 / numeric(expr.value, rank, tier, at, now);
|
|
31
|
+
case "add": return numeric(expr.left, rank, tier, at, now) + numeric(expr.right, rank, tier, at, now);
|
|
32
|
+
case "multiply": return numeric(expr.left, rank, tier, at, now) * numeric(expr.right, rank, tier, at, now);
|
|
33
|
+
case "max": return Math.max(numeric(expr.left, rank, tier, at, now), numeric(expr.right, rank, tier, at, now));
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
const real = (value: number): string => Number.isInteger(value) ? `${value}.0` : String(value);
|
|
37
|
+
function sql(expr: ScoreExpr, rank: string): string {
|
|
38
|
+
const get = (part: ScoreExpr): string => sql(part, rank);
|
|
39
|
+
switch (expr.op) {
|
|
40
|
+
case "rank": return rank;
|
|
41
|
+
case "tier": return `CASE tier ${Object.entries(TIER_WEIGHTS).map(([name, value]) => `WHEN '${name}' THEN ${real(value)}`).join(" ")} ELSE ${real(DEFAULT_TIER_WEIGHT)} END`;
|
|
42
|
+
case "age": return `((julianday(?) - julianday(at)) / ${real(MONTH_DAYS)})`;
|
|
43
|
+
case "constant": return real(expr.value);
|
|
44
|
+
case "negative": return `(-(${get(expr.value)}))`;
|
|
45
|
+
case "reciprocal": return `(1.0 / ${get(expr.value)})`;
|
|
46
|
+
case "add": return `(${get(expr.left)} + ${get(expr.right)})`;
|
|
47
|
+
// Preserve the existing SQL quotient's floating-point order; numeric ranking
|
|
48
|
+
// retains its original reciprocal product. Both interpret the same recipe.
|
|
49
|
+
case "multiply": return expr.right.op === "reciprocal" ? `(${get(expr.left)} / ${get(expr.right.value)})` : `(${get(expr.left)} * ${get(expr.right)})`;
|
|
50
|
+
case "max": return `max(${get(expr.left)}, ${get(expr.right)})`;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
export function rankedScore(rank: number, tier: string, at: string, now: number): number {
|
|
54
|
+
return numeric(SCORE, rank, tier, at, now);
|
|
55
|
+
}
|
|
56
|
+
/** SQL uses fixed internal identifiers and bound clock values, never interpolated row/query input. */
|
|
57
|
+
export function rankedScoreSql(rank = "bm25(docs)"): string { return sql(SCORE, rank); }
|
|
58
|
+
export function rankedScoreSqlDetails(rank = "bm25(docs)", useFts = true): { rawBm25Rank: string; tierFactor: string; recencyFactor: string } {
|
|
59
|
+
return { rawBm25Rank: useFts ? rank : "NULL", tierFactor: sql(TIER_FACTOR, rank), recencyFactor: sql(RECENCY_FACTOR, rank) };
|
|
60
|
+
}
|
|
61
|
+
export function rankedScoreDetails(source: SearchScoreDetails["source"], rank: number, tier: string, at: string, now: number, finalScore: number): SearchScoreDetails {
|
|
62
|
+
return { source, rawBm25Rank: rank, tierFactor: numeric(TIER_FACTOR, rank, tier, at, now), recencyFactor: numeric(RECENCY_FACTOR, rank, tier, at, now), finalScore, matchMode: "fts5" };
|
|
63
|
+
}
|
|
64
|
+
export function unrankedScoreDetails(source: SearchScoreDetails["source"], finalScore: number | null): SearchScoreDetails {
|
|
65
|
+
return { source, rawBm25Rank: null, tierFactor: null, recencyFactor: null, finalScore, matchMode: "substring" };
|
|
66
|
+
}
|
package/src/memory/store.ts
CHANGED
|
@@ -10,6 +10,8 @@ import { memoryHome } from "./home.ts";
|
|
|
10
10
|
import { machineId, type Machine } from "./machine.ts";
|
|
11
11
|
import { assertProjectActive } from "./lifecycle.ts";
|
|
12
12
|
export { machineId, type Machine } from "./machine.ts";
|
|
13
|
+
import { rankedScore, rankedScoreDetails, unrankedScoreDetails, type SearchScoreDetails } from "./search-score.ts";
|
|
14
|
+
export type { SearchScoreDetails } from "./search-score.ts";
|
|
13
15
|
import { readWindowSql, readRef, type ReadWindow, type ReadRef, SearchIndex, captureSearch } from "./search-fts.ts";
|
|
14
16
|
import { assertCuratedShardPaths, ensureShardDescriptor, readShardDescriptor, readShardedSelection, aliasedKeys, resolveAlias, resolveShardProfile, shardDir, shardKeyHash, shardMachineDir, userShardDir } from "./shard.ts";
|
|
15
17
|
import { readCaptureConfig, type CaptureRow } from "./capture.ts";
|
|
@@ -168,10 +170,12 @@ export interface Filter { project?: string; tier?: Tier; kind?: Kind; peer?: str
|
|
|
168
170
|
/** #13 search: `Filter` plus the capture-tier switch (on by default; capture is recall, never curated evidence). */
|
|
169
171
|
export interface SearchFilter extends Filter {
|
|
170
172
|
includeCapture?: boolean;
|
|
173
|
+
/** Attach score factors only when explicitly requested. */
|
|
174
|
+
explain?: boolean;
|
|
171
175
|
/** A host that writes capture with a `retentionDays` override passes the same value here; files decide otherwise. */
|
|
172
176
|
captureRetentionDays?: number;
|
|
173
177
|
}
|
|
174
|
-
export interface SearchHit { source: "curated" | "capture"; score: number; view?: View; capture?: CaptureRow }
|
|
178
|
+
export interface SearchHit { source: "curated" | "capture"; score: number; scoreDetails?: SearchScoreDetails; view?: View; capture?: CaptureRow }
|
|
175
179
|
export interface SearchResult { engine: "fts5" | "substring"; hits: SearchHit[] }
|
|
176
180
|
export interface Graph { nodes: { id: Node; type: string; label: string; kind?: string; tier?: Tier }[]; edges: { from: Node; to: Node; rel: string }[] }
|
|
177
181
|
|
|
@@ -490,18 +494,30 @@ export class MemoryStore {
|
|
|
490
494
|
// `ownClock`: a note asked for short-term on purpose expires on its own clock, open task or not, and is never promoted.
|
|
491
495
|
note(n: NoteInput, ownClock = false): string { return this.append(n.tier === "short" && !n.expires && (ownClock || !n.task || this.settled(n.task)) ? { ...n, expires: this.until() } : n, n.tier === "short"); }
|
|
492
496
|
|
|
493
|
-
/**
|
|
494
|
-
|
|
495
|
-
* the same id, so a second import writes nothing. Returns the id and whether it was new.
|
|
496
|
-
*/
|
|
497
|
-
importNote(n: NoteInput, at: string, identity: { machine?: string; expectedId?: string } = {}): { id: string; added: boolean } {
|
|
497
|
+
/** The already-opened owner of an imported row, shared by preflight and append. */
|
|
498
|
+
private importOwner(n: NoteInput): MemoryStore {
|
|
498
499
|
if (this.bootstrapOnly) throw new Error("migration bootstrap cannot import memory");
|
|
499
|
-
if (this.userOverlay && n.scope === "shared") return this.userOverlay.
|
|
500
|
+
if (this.userOverlay && n.scope === "shared") return this.userOverlay.importOwner(n);
|
|
500
501
|
if (this.profile.sharded && !this.profile.shardUser && n.scope === "project" && n.project !== this.profile.shardKey) {
|
|
501
502
|
const owner = this.aliasStores.find(store => store.profile.shardKey === n.project);
|
|
502
|
-
if (owner) return owner.
|
|
503
|
+
if (owner) return owner.importOwner(n);
|
|
503
504
|
}
|
|
504
505
|
if (this.profile.sharded && (this.profile.shardUser ? n.scope !== "shared" : n.scope !== "project" || n.project !== this.profile.shardKey)) throw new Error("entry does not belong to this curated shard");
|
|
506
|
+
return this;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/** Check every import route before a multi-row caller appends; never creates a missing alias shard. */
|
|
510
|
+
assertImportable(notes: Iterable<NoteInput>): void {
|
|
511
|
+
for (const note of notes) this.importOwner(note);
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* An entry from another store with its own time (`ahub memory import claude-mem`): the same source row always gives
|
|
516
|
+
* the same id, so a second import writes nothing. Returns the id and whether it was new.
|
|
517
|
+
*/
|
|
518
|
+
importNote(n: NoteInput, at: string, identity: { machine?: string; expectedId?: string } = {}): { id: string; added: boolean } {
|
|
519
|
+
const owner = this.importOwner(n);
|
|
520
|
+
if (owner !== this) return owner.importNote(n, at, identity);
|
|
505
521
|
assertSharedV1Op(n.op);
|
|
506
522
|
// A fixed machine: the same observation imported on two machines is one entry once they sync. A project import preserves the exported identity instead.
|
|
507
523
|
const body = { schema: MEMORY_SCHEMA, at, machine: identity.machine ?? "claude-mem", ...n, text: stripPrivateSpans(n.text).slice(0, MAX_TEXT), ...(n.title ? { title: stripPrivateSpans(n.title).slice(0, MAX_TITLE) } : {}) } as Omit<Entry, "id">;
|
|
@@ -1057,12 +1073,15 @@ export class MemoryStore {
|
|
|
1057
1073
|
*/
|
|
1058
1074
|
search(q: string, f: SearchFilter = {}): SearchResult {
|
|
1059
1075
|
const limit = f.limit ?? 200;
|
|
1060
|
-
if (!this.searchIndex.available) return { engine: "substring", hits: this.list({ ...f, q }).map(
|
|
1076
|
+
if (!this.searchIndex.available) return { engine: "substring", hits: this.list({ ...f, q }).map(view => {
|
|
1077
|
+
const hit: SearchHit = { source: "curated", score: 0, view };
|
|
1078
|
+
if (f.explain) hit.scoreDetails = unrankedScoreDetails("curated", hit.score);
|
|
1079
|
+
return hit;
|
|
1080
|
+
}) };
|
|
1061
1081
|
const visible = new Map(this.live(f).filter((v) => (!f.project || this.inScope(v.project, f.project) || v.scope === "shared")
|
|
1062
1082
|
&& (!f.since || Date.parse(v.at) >= Date.parse(f.since)) && (f.includeSuperseded || !v.supersededBy)).map((v) => [v.id, v]));
|
|
1063
1083
|
for (const store of this.readStores()) store.syncSearchIndex();
|
|
1064
1084
|
const now = this.now();
|
|
1065
|
-
const recency = (at: string): number => 1 / (1 + Math.max(0, now - Date.parse(at)) / (30 * 24 * 3600_000));
|
|
1066
1085
|
const hits: SearchHit[] = [];
|
|
1067
1086
|
// The project filter runs inside the FTS query (its rows plus shared ones), so a busy sibling project can never
|
|
1068
1087
|
// starve this project's LIMIT.
|
|
@@ -1082,7 +1101,9 @@ export class MemoryStore {
|
|
|
1082
1101
|
if (!view || seen.has(hit.ref)) continue;
|
|
1083
1102
|
seen.add(hit.ref);
|
|
1084
1103
|
found++;
|
|
1085
|
-
|
|
1104
|
+
const selected: SearchHit = { source: "curated", score: rankedScore(hit.rank, view.tier, hit.at, now), view };
|
|
1105
|
+
if (f.explain) selected.scoreDetails = rankedScoreDetails("curated", hit.rank, view.tier, hit.at, now, selected.score);
|
|
1106
|
+
hits.push(selected);
|
|
1086
1107
|
}
|
|
1087
1108
|
if (ranked.length < page) break;
|
|
1088
1109
|
}
|
|
@@ -1097,12 +1118,18 @@ export class MemoryStore {
|
|
|
1097
1118
|
const retained = Number.isFinite(cutoff.getTime()) ? cutoff.toISOString() : undefined;
|
|
1098
1119
|
const since = f.since && Number.isFinite(Date.parse(f.since)) ? new Date(Date.parse(f.since)).toISOString() : undefined;
|
|
1099
1120
|
for (const { row, rank } of captureSearch(shard, q, Math.max(limit * 4, 32), since && (!retained || since > retained) ? since : retained)) {
|
|
1100
|
-
|
|
1121
|
+
const selected: SearchHit = { source: "capture", score: rankedScore(rank, "capture", row.at, now), capture: row };
|
|
1122
|
+
if (f.explain) selected.scoreDetails = rankedScoreDetails("capture", rank, "capture", row.at, now, selected.score);
|
|
1123
|
+
hits.push(selected);
|
|
1101
1124
|
}
|
|
1102
1125
|
}
|
|
1103
1126
|
// Where FTS misses entirely (a stopword-only query, a tokenization edge), the substring filter is the safety net.
|
|
1104
1127
|
if (!hits.length && q.trim()) {
|
|
1105
|
-
for (const view of this.list({ ...f, q }))
|
|
1128
|
+
for (const view of this.list({ ...f, q })) {
|
|
1129
|
+
const hit: SearchHit = { source: "curated", score: 0, view };
|
|
1130
|
+
if (f.explain) hit.scoreDetails = unrankedScoreDetails("curated", hit.score);
|
|
1131
|
+
hits.push(hit);
|
|
1132
|
+
}
|
|
1106
1133
|
}
|
|
1107
1134
|
const ref = (h: SearchHit): string => h.view?.id ?? String(h.capture?.id ?? 0).padStart(12, "0");
|
|
1108
1135
|
hits.sort((a, b) => b.score - a.score || ref(a).localeCompare(ref(b)));
|