@sema-agent/core 7.64.0 → 7.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +317 -0
- package/dist/agents/peer-held-queue.js +1 -1
- package/dist/agents/peer-idle.js +1 -1
- package/dist/agents/peer-session-drain.js +3 -3
- package/dist/agents/subagent.js +5 -2
- package/dist/core/auto-promote.d.ts +3 -2
- package/dist/core/background-agent-store.d.ts +7 -0
- package/dist/core/checkpoint-store.d.ts +2 -3
- package/dist/core/context-settings.d.ts +1 -2
- package/dist/core/context-settings.js +1 -1
- package/dist/core/engine-attestation.d.ts +2 -2
- package/dist/core/engine-notice.d.ts +13 -1
- package/dist/core/engine-notice.js +3 -1
- package/dist/core/file-history-store.d.ts +2 -2
- package/dist/core/file-history-store.js +0 -4
- package/dist/core/gate-fold.js +3 -1
- package/dist/core/governance-codes.d.ts +6 -4
- package/dist/core/governance-codes.js +4 -3
- package/dist/core/leg-identity.d.ts +18 -2
- package/dist/core/leg-identity.js +5 -7
- package/dist/core/mailbox-store.d.ts +1 -1
- package/dist/core/memory-engine/consolidation.d.ts +10 -4
- package/dist/core/memory-engine/consolidation.js +11 -1
- package/dist/core/memory-engine/control-objects.d.ts +36 -3
- package/dist/core/memory-engine/control-objects.js +134 -12
- package/dist/core/memory-engine/control-plane-adoption.d.ts +5 -2
- package/dist/core/memory-engine/control-plane-adoption.js +5 -3
- package/dist/core/memory-engine/delegation-settlement.d.ts +59 -42
- package/dist/core/memory-engine/delegation-settlement.js +175 -202
- package/dist/core/memory-engine/dual-root.d.ts +3 -2
- package/dist/core/memory-engine/dual-root.js +5 -1
- package/dist/core/memory-engine/engine.d.ts +16 -10
- package/dist/core/memory-engine/engine.js +138 -145
- package/dist/core/memory-engine/file-backend.d.ts +17 -11
- package/dist/core/memory-engine/file-backend.js +53 -103
- package/dist/core/memory-engine/layout.d.ts +129 -49
- package/dist/core/memory-engine/layout.js +202 -82
- package/dist/core/memory-engine/memory-age.d.ts +12 -0
- package/dist/core/memory-engine/memory-age.js +17 -0
- package/dist/core/memory-engine/memory-backend-contract.js +5 -4
- package/dist/core/memory-engine/memory-deployment.d.ts +1 -1
- package/dist/core/memory-engine/origin-clearance.js +1 -5
- package/dist/core/memory-engine/tools.d.ts +2 -2
- package/dist/core/memory-engine/tools.js +7 -7
- package/dist/core/memory-engine/types.d.ts +16 -10
- package/dist/core/memory-vector.d.ts +6 -14
- package/dist/core/memory.d.ts +0 -462
- package/dist/core/memory.js +0 -333
- package/dist/core/permission-rule-consent.d.ts +49 -18
- package/dist/core/permission-rule-consent.js +59 -38
- package/dist/core/permission-rule-store.d.ts +24 -14
- package/dist/core/retention.d.ts +2 -7
- package/dist/core/retention.js +0 -1
- package/dist/core/runner/classifier-ladder.d.ts +3 -1
- package/dist/core/runner/classifier-ladder.js +2 -2
- package/dist/core/runner/contracts.d.ts +9 -9
- package/dist/core/runner/delegation-owner.d.ts +0 -8
- package/dist/core/runner/delegation-owner.js +1 -3
- package/dist/core/runner/prepare-gate-stations.d.ts +8 -5
- package/dist/core/runner/prepare-gate-stations.js +13 -5
- package/dist/core/runner/prepare-memory-engine-session.js +1 -1
- package/dist/core/runner/prepare-memory.js +9 -9
- package/dist/core/runner/prepare-policy-chain.js +11 -3
- package/dist/core/runner/prepare-task.d.ts +5 -1
- package/dist/core/runner/prepare-task.js +3 -3
- package/dist/core/runner/run-notification-lane.js +1 -1
- package/dist/core/runner/sub-agent-deny-lists.d.ts +19 -0
- package/dist/core/runner/sub-agent-deny-lists.js +25 -0
- package/dist/core/secret-env.d.ts +2 -2
- package/dist/core/secret-env.js +6 -20
- package/dist/core/session-store.d.ts +6 -6
- package/dist/core/session-store.js +3 -3
- package/dist/core/store-contracts/background-agent-store-contract.js +13 -0
- package/dist/core/task-notification.d.ts +35 -8
- package/dist/core/task-notification.js +31 -13
- package/dist/core/task-registry-agent.js +1 -0
- package/dist/core/task-registry-shared.d.ts +4 -0
- package/dist/core/tool-errors.d.ts +3 -3
- package/dist/core/tool-spec.d.ts +7 -7
- package/dist/core/write-protect.d.ts +16 -0
- package/dist/core/write-protect.js +10 -0
- package/dist/engine/session/repo-utils.d.ts +14 -4
- package/dist/engine/session/repo-utils.js +5 -4
- package/dist/index.d.ts +6 -6
- package/dist/index.js +6 -6
- package/dist/prompts/default.d.ts +1 -2
- package/dist/scenarios/full-body.d.ts +1 -1
- package/dist/stores/file/adoption/adopt.d.ts +1 -1
- package/dist/stores/file/index.d.ts +1 -15
- package/dist/stores/file/index.js +1 -15
- package/dist/stores/file/permission-rule-store.d.ts +12 -1
- package/dist/stores/file/permission-rule-store.js +31 -4
- package/dist/stores/file/session-store.js +1 -1
- package/package.json +2 -2
- package/test/export-surface.snapshot.json +9 -73
- package/wire-changes/core-stores-sql@7.65.0.json +30 -0
- package/wire-changes/core@7.65.0.json +1650 -0
- package/wire-changes/mods-kernel@7.65.0.json +30 -0
- package/wire-changes/primitives@7.65.0.json +337 -0
- package/dist/core/memory-recall.d.ts +0 -208
- package/dist/core/memory-recall.js +0 -136
- package/dist/stores/file/memory-store.d.ts +0 -124
- package/dist/stores/file/memory-store.js +0 -353
|
@@ -1,136 +0,0 @@
|
|
|
1
|
-
import { MEMORY_INJECTION_TAGS } from "./memory.js";
|
|
2
|
-
import { sanitizeUntrustedText } from "./untrusted-text.js";
|
|
3
|
-
export const DEFAULT_MAX_SELECTED = 5;
|
|
4
|
-
export const DEFAULT_MAX_LINKED = 3;
|
|
5
|
-
export const DEFAULT_SELECT_TIMEOUT_MS = 5_000;
|
|
6
|
-
const MANIFEST_MAX_LINES = 200;
|
|
7
|
-
const MANIFEST_MAX_BYTES = 25_000;
|
|
8
|
-
const SELECTED_BODY_MAX_BYTES = 50_000;
|
|
9
|
-
const ONE_DAY_MS = 24 * 60 * 60 * 1_000;
|
|
10
|
-
const ONE_WEEK_MS = 7 * ONE_DAY_MS;
|
|
11
|
-
const ONE_MONTH_MS = 30.44 * ONE_DAY_MS;
|
|
12
|
-
export function formatMemoryAge(ageMs) {
|
|
13
|
-
if (ageMs < ONE_DAY_MS)
|
|
14
|
-
return "today";
|
|
15
|
-
if (ageMs < 2 * ONE_DAY_MS)
|
|
16
|
-
return "yesterday";
|
|
17
|
-
if (ageMs < ONE_WEEK_MS)
|
|
18
|
-
return `${Math.floor(ageMs / ONE_DAY_MS)} days ago`;
|
|
19
|
-
if (ageMs < ONE_MONTH_MS) {
|
|
20
|
-
const weeks = Math.floor(ageMs / ONE_WEEK_MS);
|
|
21
|
-
return `${weeks} ${weeks === 1 ? "week" : "weeks"} ago`;
|
|
22
|
-
}
|
|
23
|
-
const months = Math.floor(ageMs / ONE_MONTH_MS);
|
|
24
|
-
return `${months} ${months === 1 ? "month" : "months"} ago`;
|
|
25
|
-
}
|
|
26
|
-
export const RECALL_CAVEAT = "These notes were recalled as relevant, but they are point-in-time observations from when they were " +
|
|
27
|
-
"written — they may now be stale. Trust current tool output over memory, and update or remove a note you " +
|
|
28
|
-
"find is wrong.\n\n" +
|
|
29
|
-
"## Before recommending from memory\n" +
|
|
30
|
-
'- "The memory says X exists" is not the same as "X exists now" — a note is not proof the thing still exists.\n' +
|
|
31
|
-
"- If the user is about to act on your recommendation (not just asking about history), verify first.\n" +
|
|
32
|
-
"- If a note names a file, check the file exists; if it names a function or symbol, grep for it; if it names a flag or value, read the current source.\n" +
|
|
33
|
-
"- Memory is a snapshot from when it was written, not a live view — prefer `git log` or reading the code over recalling the snapshot.";
|
|
34
|
-
function isNewerNote(candidate, incumbent) {
|
|
35
|
-
if (candidate.timestampMissing || incumbent.timestampMissing)
|
|
36
|
-
return false;
|
|
37
|
-
return candidate.mtimeMs > incumbent.mtimeMs;
|
|
38
|
-
}
|
|
39
|
-
export function resolveLinkedIds(headers, selected, max) {
|
|
40
|
-
if (max <= 0 || selected.length === 0)
|
|
41
|
-
return [];
|
|
42
|
-
const byName = new Map();
|
|
43
|
-
for (const h of headers) {
|
|
44
|
-
if (!h.name)
|
|
45
|
-
continue;
|
|
46
|
-
const prev = byName.get(h.name);
|
|
47
|
-
if (!prev || isNewerNote(h, prev))
|
|
48
|
-
byName.set(h.name, h);
|
|
49
|
-
}
|
|
50
|
-
const selectedIds = new Set(selected.map((r) => r.id));
|
|
51
|
-
const out = [];
|
|
52
|
-
const seen = new Set();
|
|
53
|
-
for (const r of selected) {
|
|
54
|
-
for (const m of r.text.matchAll(/\[\[([^[\]\n]{1,128})\]\]/g)) {
|
|
55
|
-
const target = byName.get(m[1].trim());
|
|
56
|
-
if (!target || selectedIds.has(target.id) || seen.has(target.id))
|
|
57
|
-
continue;
|
|
58
|
-
seen.add(target.id);
|
|
59
|
-
out.push(target.id);
|
|
60
|
-
if (out.length >= max)
|
|
61
|
-
return out;
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
return out;
|
|
65
|
-
}
|
|
66
|
-
function manifestField(s, cap) {
|
|
67
|
-
const t = s.replace(/\s+/g, " ").trim();
|
|
68
|
-
return t.length > cap ? `${t.slice(0, cap)}…` : t;
|
|
69
|
-
}
|
|
70
|
-
export function buildManifestText(headers) {
|
|
71
|
-
const lines = [];
|
|
72
|
-
let bytes = 0;
|
|
73
|
-
let truncated;
|
|
74
|
-
for (const h of headers) {
|
|
75
|
-
if (lines.length >= MANIFEST_MAX_LINES) {
|
|
76
|
-
truncated = "lines";
|
|
77
|
-
break;
|
|
78
|
-
}
|
|
79
|
-
const tag = h.type ? `[${manifestField(h.type, 32)}] ` : "";
|
|
80
|
-
const line = `- ${h.id}: ${tag}${manifestField(h.description, 300)}`;
|
|
81
|
-
const lb = Buffer.byteLength(line, "utf8") + 1;
|
|
82
|
-
if (bytes + lb > MANIFEST_MAX_BYTES) {
|
|
83
|
-
truncated = "bytes";
|
|
84
|
-
break;
|
|
85
|
-
}
|
|
86
|
-
lines.push(line);
|
|
87
|
-
bytes += lb;
|
|
88
|
-
}
|
|
89
|
-
const note = truncated ? `\n<manifest truncated: ${truncated} cap reached; ${headers.length - lines.length} notes omitted — consolidation should shrink memory>` : "";
|
|
90
|
-
return lines.join("\n") + note;
|
|
91
|
-
}
|
|
92
|
-
export function validateSelectedIds(headers, ids, max) {
|
|
93
|
-
const valid = new Set(headers.map((h) => h.id));
|
|
94
|
-
const out = [];
|
|
95
|
-
const seen = new Set();
|
|
96
|
-
for (const id of ids) {
|
|
97
|
-
if (out.length >= max)
|
|
98
|
-
break;
|
|
99
|
-
if (valid.has(id) && !seen.has(id)) {
|
|
100
|
-
seen.add(id);
|
|
101
|
-
out.push(id);
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
return out;
|
|
105
|
-
}
|
|
106
|
-
export function composeSelectiveBody(manifestText, selected, nowMs, linked = [], recallable = true, recallToolName) {
|
|
107
|
-
const renderNote = (r, label) => {
|
|
108
|
-
const ageMs = nowMs - r.mtimeMs;
|
|
109
|
-
const verify = r.timestampMissing || ageMs > ONE_DAY_MS ? " — verify it's still current" : "";
|
|
110
|
-
const stale = r.timestampMissing ? ` (write time unknown${verify})` : ` (written ${formatMemoryAge(ageMs)}${verify})`;
|
|
111
|
-
const prefix = label ? `${sanitizeUntrustedText(label, [...MEMORY_INJECTION_TAGS])} ` : "";
|
|
112
|
-
return `- ${prefix}${sanitizeUntrustedText(r.text, [...MEMORY_INJECTION_TAGS])}${stale}`;
|
|
113
|
-
};
|
|
114
|
-
let body = selected.map((r) => renderNote(r)).join("\n");
|
|
115
|
-
if (linked.length > 0) {
|
|
116
|
-
body +=
|
|
117
|
-
`\n\nLinked notes (followed one hop from [[name]] references above — same staleness caveats apply):\n` +
|
|
118
|
-
linked.map((r) => renderNote(r, r.name ? `[[${r.name}]]` : "")).join("\n");
|
|
119
|
-
}
|
|
120
|
-
if (Buffer.byteLength(body, "utf8") > SELECTED_BODY_MAX_BYTES) {
|
|
121
|
-
const buf = Buffer.from(body, "utf8");
|
|
122
|
-
let cut = SELECTED_BODY_MAX_BYTES;
|
|
123
|
-
while (cut > 0 && (buf[cut] & 0xc0) === 0x80)
|
|
124
|
-
cut--;
|
|
125
|
-
body = `${buf.subarray(0, cut).toString("utf8")}\n<selected notes truncated to ${SELECTED_BODY_MAX_BYTES} bytes>`;
|
|
126
|
-
}
|
|
127
|
-
const selectedSection = selected.length > 0 ? `\n\nRelevant notes:\n${body}` : "\n\n(no notes selected as relevant to this task)";
|
|
128
|
-
const recallCall = recallToolName !== undefined ? `call ${recallToolName}` : `search the memory-recall tool bound to this scope`;
|
|
129
|
-
const manifestSection = recallable
|
|
130
|
-
? `Memory index — to load a note in full, ${recallCall} with keywords from its description:\n${sanitizeUntrustedText(manifestText, [...MEMORY_INJECTION_TAGS])}`
|
|
131
|
-
: "";
|
|
132
|
-
return `${manifestSection}${selectedSection}`;
|
|
133
|
-
}
|
|
134
|
-
export function encodeSurfacedKey(scope, id) {
|
|
135
|
-
return `${scope.length}:${scope}:${id}`;
|
|
136
|
-
}
|
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
import { type Embedder, type MemoryNoteHeader, type MemoryNoteRecord, type MemoryStore, type MemoryVectorMode, type ScoredMemory, type StructuredNoteInput } from "../../core/memory.js";
|
|
2
|
-
export declare class FileMemoryStore implements MemoryStore {
|
|
3
|
-
private readonly root;
|
|
4
|
-
/** Optional injected embedder (design/81 Slice 5). When present, note bodies are embedded (in the background,
|
|
5
|
-
* after the durable note write) into a rebuildable sidecar index and `searchScored` ranks by cosine distance
|
|
6
|
-
* (portable / in-process, mirrors PgMemoryStore's jsonb mode). Absent ⇒ lexical, byte-identical to before. */
|
|
7
|
-
private readonly embedder?;
|
|
8
|
-
/** ms budget for a background embed; a slower/stuck endpoint yields no vector (row stays lexical), never
|
|
9
|
-
* blocking the durable note write (which already completed). */
|
|
10
|
-
private readonly embedTimeoutMs;
|
|
11
|
-
/** In-flight background embeds (so tests can flush + `close` can drain). */
|
|
12
|
-
private readonly pending;
|
|
13
|
-
/**
|
|
14
|
-
* RB-146 (2026-07-25, 按面收口): scope state is shared per PHYSICAL directory, like every other file
|
|
15
|
-
* backend. This was the last store on this surface still caching replayed state on the INSTANCE: a second
|
|
16
|
-
* `FileMemoryStore` over one directory replayed its own view on first touch and then served it forever,
|
|
17
|
-
* so a note durably committed through one instance was invisible to the other — the same shape as RB-73
|
|
18
|
-
* (session), RB-55 (agent), RB-59 (run) and RB-134 (checkpoint). Refcounted, and the last holder closes
|
|
19
|
-
* the logs; the `closed` latch below already existed for the fd-recycle guard.
|
|
20
|
-
*/
|
|
21
|
-
private get scopes();
|
|
22
|
-
private readonly sharedScopes;
|
|
23
|
-
private readonly scopeKey;
|
|
24
|
-
/** Set by {@link close}: a background embed that resolves AFTER close must NOT writeSync to the (now closed)
|
|
25
|
-
* vlog fd — a stray write on a recycled fd could corrupt an unrelated file. */
|
|
26
|
-
private closed;
|
|
27
|
-
constructor(root: string, options?: {
|
|
28
|
-
embedder?: Embedder;
|
|
29
|
-
embedTimeoutMs?: number;
|
|
30
|
-
});
|
|
31
|
-
/** design/81 Slice 5 rung: `portable` (in-process cosine over the sidecar) when an embedder is injected,
|
|
32
|
-
* else `lexical`. File never reaches `native` (no backend index). */
|
|
33
|
-
get vectorMode(): MemoryVectorMode;
|
|
34
|
-
/** True when an embedder was injected. Exposed so a consumer/test can confirm it was accepted. */
|
|
35
|
-
get hasEmbedder(): boolean;
|
|
36
|
-
/** Await all in-flight background embeds (tests / `close`). */
|
|
37
|
-
flushEmbeds(): Promise<void>;
|
|
38
|
-
/** Open (replay) a scope's notes + sidecar vectors once and cache it. */
|
|
39
|
-
private scopeState;
|
|
40
|
-
/**
|
|
41
|
-
* RB-267 — record that `scope` holds live descriptors, and evict the coldest scopes past
|
|
42
|
-
* {@link MAX_OPEN_MEMORY_SCOPES}. Eviction releases the handles ONLY (`closeForSwap`, the RB-167 lazy
|
|
43
|
-
* reopen); the replayed `ScopeState` stays in the map, so nothing about the authority — the entry
|
|
44
|
-
* list, the cursor, the vector cache — is rebuilt or lost, and a later write to an evicted scope
|
|
45
|
-
* reopens its log transparently.
|
|
46
|
-
*/
|
|
47
|
-
private touchOpenLogs;
|
|
48
|
-
/** RB-267 — the notes log for a WRITE. Every durable append goes through here so the LRU sees the
|
|
49
|
-
* access: a write is also the only thing that can (re)open a descriptor (`AppendLog.append` reopens
|
|
50
|
-
* a swap-closed log lazily), so touching here is what keeps "has an open handle ⇒ tracked" true.
|
|
51
|
-
* Reads never open anything and deliberately do not touch. */
|
|
52
|
-
private writeLog;
|
|
53
|
-
/** RB-267 — the sidecar vector log, opened on FIRST WRITE. Only {@link queueEmbed} writes vectors, so
|
|
54
|
-
* a store with no embedder never opens this file at all (it used to take a descriptor per scope for
|
|
55
|
-
* a file it would never touch). Replay reads the path directly and needs no handle. */
|
|
56
|
-
private vectorLog;
|
|
57
|
-
/** Queue a background embed of `body` → sidecar vector for `id` (best-effort, timeout-bounded, non-blocking).
|
|
58
|
-
* Only when an embedder is injected; a failure/timeout leaves the row lexical (fail-open). The durable note
|
|
59
|
-
* write has ALREADY happened before this is called, so this never gates durability. */
|
|
60
|
-
private queueEmbed;
|
|
61
|
-
/** Rewrite the derived `MEMORY.md` projection atomically (CC markdown parity), STAGED IN THE TARGET'S
|
|
62
|
-
* OWN DIRECTORY: `rename(2)` is defined only within one filesystem, so staging beside the target makes
|
|
63
|
-
* the two share a mount by construction where a fixed scratch dir elsewhere made it deployment luck (a
|
|
64
|
-
* memory tree on its own device took `EXDEV` on every note). EVERY caller is POST-DURABLE, so a failure
|
|
65
|
-
* here cannot mean the note was lost: it is {@link MemoryProjectionError}, that phase's one name. */
|
|
66
|
-
private writeProjection;
|
|
67
|
-
read(scope: string): string | null;
|
|
68
|
-
append(scope: string, note: string): string | void;
|
|
69
|
-
appendStructured(scope: string, note: StructuredNoteInput): string;
|
|
70
|
-
/** Append + fsync the note event BEFORE updating the in-memory list, then refresh the projection. */
|
|
71
|
-
private commit;
|
|
72
|
-
clear(scope: string): void;
|
|
73
|
-
/** design/84 Seam B: read the cached periodic-consolidation cursor (replayed from the last `op:cursor`). */
|
|
74
|
-
getConsolidationCursor(scope: string): string | undefined;
|
|
75
|
-
/** design/84 Seam B: persist the periodic-consolidation cursor via a durable `op:cursor` append-log event,
|
|
76
|
-
* then update the cached state (durable FIRST, like `commit`). */
|
|
77
|
-
setConsolidationCursor(scope: string, cursor: string): void;
|
|
78
|
-
/**
|
|
79
|
-
* Keyword search backing the `recall` tool. design/81 Slice 2b — uses the SHARED synonym/stem-aware
|
|
80
|
-
* {@link import("../../core/memory.js").lexicalSearchMatch} (the SAME matcher as
|
|
81
|
-
* {@link import("../../core/memory.js").InMemoryMemoryStore.search}, ZERO model/native/embedder dep) so
|
|
82
|
-
* "auth token" matches a note saying "credentials" / "bearer", and the `recall` tool returns IDENTICAL
|
|
83
|
-
* results across the File and InMemory backends. A bullet matches a query term iff it contains ANY of that
|
|
84
|
-
* term's expansions, and must match EVERY query term (AND across terms, OR within a term's synonyms). With
|
|
85
|
-
* no synonym/stem hit this is byte-identical to the plain substring filter.
|
|
86
|
-
*/
|
|
87
|
-
search(scope: string, query: string, limit?: number): string[];
|
|
88
|
-
/** Manifest (design/65 §2.2): a header per note, body excluded — in-memory parity. */
|
|
89
|
-
listStructuredNotes(scope: string): MemoryNoteHeader[];
|
|
90
|
-
getByIds(scope: string, ids: string[]): MemoryNoteRecord[];
|
|
91
|
-
/**
|
|
92
|
-
* Id-addressable similarity retrieval (design/81 Slice 2 — the consolidation candidate source). The
|
|
93
|
-
* `score` is a cosine **DISTANCE** ∈ [0,2], **0 = identical** ({@link ScoredMemory} contract): it is the
|
|
94
|
-
* lexical `1 - Jaccard(terms)` ∈ [0,1] ⊂ [0,2] stand-in (byte-identical to
|
|
95
|
-
* {@link InMemoryMemoryStore.searchScored} / the lexical {@link import("../pg.js").PgMemoryStore} fallback),
|
|
96
|
-
* so the `{0.05, 0.3}` consolidation band needs no retune. An injected embedder is ACCEPTED but vector
|
|
97
|
-
* ranking is the DEFERRED Slice 5 — so this method ALWAYS uses the lexical distance for now (an injected
|
|
98
|
-
* embedder is harmless, NOT a throw). Excludes the query's own id (a note must not self-match — see below)
|
|
99
|
-
* and zero-overlap entries; returns the `limit` nearest, ascending by distance.
|
|
100
|
-
*
|
|
101
|
-
* **Self-id exclusion:** byte-identical to {@link InMemoryMemoryStore.searchScored} / the lexical
|
|
102
|
-
* {@link import("../pg.js").PgMemoryStore} fallback, the SOLE consumer (consolidation) excludes the
|
|
103
|
-
* query note's own id (and its sibling this-task ids) by filtering the returned `ScoredMemory[]` on its
|
|
104
|
-
* `thisTaskIds` set — keeping the store's ranking pure and the cross-backend contract identical.
|
|
105
|
-
*/
|
|
106
|
-
searchScored(scope: string, query: string, limit?: number): ScoredMemory[] | Promise<ScoredMemory[]>;
|
|
107
|
-
/**
|
|
108
|
-
* Consolidation UPDATE (design/81 Slice 2): replace a note's body by id, keeping the id stable. Appends a
|
|
109
|
-
* durable `update` event FIRST, then mutates the in-memory replay + refreshes the projection — same
|
|
110
|
-
* durable-before-visible order as {@link commit}. An auto-derived description is re-derived from the new
|
|
111
|
-
* body (so the manifest doesn't drift); an explicit one is preserved (design/65 [79], in-memory parity).
|
|
112
|
-
* Unknown id → no-op (defensive). NB: no `sanitizeUntrustedText` here — the fence lives at the
|
|
113
|
-
* inlets/recall, NOT in `update` (design/81 §1; in-memory `update` only trims).
|
|
114
|
-
*/
|
|
115
|
-
update(scope: string, id: string, text: string): void;
|
|
116
|
-
/**
|
|
117
|
-
* Consolidation DELETE (design/81 Slice 2): remove a note by id (id never reused afterward). Appends a
|
|
118
|
-
* durable `delete` tombstone FIRST, then drops it from the in-memory replay + refreshes the projection.
|
|
119
|
-
* Unknown id → no-op.
|
|
120
|
-
*/
|
|
121
|
-
delete(scope: string, id: string): void;
|
|
122
|
-
/** Release all open scope handles (best-effort). */
|
|
123
|
-
close(): void;
|
|
124
|
-
}
|
|
@@ -1,353 +0,0 @@
|
|
|
1
|
-
import { join } from "node:path";
|
|
2
|
-
import { uuidv7 } from "../../internal/harness.js";
|
|
3
|
-
import { firstSentence, lexicalSearchMatch, parseNoteTimestamp, MemoryProjectionError, } from "../../core/memory.js";
|
|
4
|
-
import { cosineDistance, jaccardDistance, termSet } from "../../core/memory-vector.js";
|
|
5
|
-
import { canonicalStoreKey, AppendLog, atomicWriteFile, ensureDir, readJsonlRecords, sanitizeScope, } from "./fs-atomic.js";
|
|
6
|
-
import { assertAdoptionBootGate } from "./adoption/marker.js";
|
|
7
|
-
import { optionKeys, receiveOptions } from "../../core/own-property-receipt.js";
|
|
8
|
-
const MAX_OPEN_MEMORY_SCOPES = 64;
|
|
9
|
-
const sharedMemoryDirs = new Map();
|
|
10
|
-
const FILE_MEMORY_STORE_OPTION_KEYS = optionKeys()({ embedder: 0, embedTimeoutMs: 0 });
|
|
11
|
-
export class FileMemoryStore {
|
|
12
|
-
root;
|
|
13
|
-
embedder;
|
|
14
|
-
embedTimeoutMs;
|
|
15
|
-
pending = new Set();
|
|
16
|
-
get scopes() {
|
|
17
|
-
return this.sharedScopes.scopes;
|
|
18
|
-
}
|
|
19
|
-
sharedScopes;
|
|
20
|
-
scopeKey;
|
|
21
|
-
closed = false;
|
|
22
|
-
constructor(root, options) {
|
|
23
|
-
const opts = receiveOptions(options, "FileMemoryStore", FILE_MEMORY_STORE_OPTION_KEYS);
|
|
24
|
-
assertAdoptionBootGate(root, "FileMemoryStore");
|
|
25
|
-
this.root = join(root, "memory");
|
|
26
|
-
this.scopeKey = canonicalStoreKey(this.root);
|
|
27
|
-
const live = sharedMemoryDirs.get(this.scopeKey);
|
|
28
|
-
if (live !== undefined) {
|
|
29
|
-
live.refs += 1;
|
|
30
|
-
this.sharedScopes = live;
|
|
31
|
-
}
|
|
32
|
-
else {
|
|
33
|
-
this.sharedScopes = { scopes: new Map(), refs: 1, openLogs: new Set() };
|
|
34
|
-
sharedMemoryDirs.set(this.scopeKey, this.sharedScopes);
|
|
35
|
-
}
|
|
36
|
-
this.embedder = opts.embedder;
|
|
37
|
-
this.embedTimeoutMs = Math.max(0, opts.embedTimeoutMs ?? 10_000);
|
|
38
|
-
ensureDir(this.root);
|
|
39
|
-
}
|
|
40
|
-
get vectorMode() {
|
|
41
|
-
return this.embedder ? "portable" : "lexical";
|
|
42
|
-
}
|
|
43
|
-
get hasEmbedder() {
|
|
44
|
-
return this.embedder !== undefined;
|
|
45
|
-
}
|
|
46
|
-
async flushEmbeds() {
|
|
47
|
-
await Promise.all([...this.pending]);
|
|
48
|
-
}
|
|
49
|
-
scopeState(scope) {
|
|
50
|
-
const existing = this.scopes.get(scope);
|
|
51
|
-
if (existing)
|
|
52
|
-
return existing;
|
|
53
|
-
const dir = join(this.root, sanitizeScope(scope));
|
|
54
|
-
ensureDir(dir);
|
|
55
|
-
const notesPath = join(dir, "notes.jsonl");
|
|
56
|
-
const entries = [];
|
|
57
|
-
let cursor;
|
|
58
|
-
for (const ev of readJsonlRecords(notesPath)) {
|
|
59
|
-
switch (ev.op) {
|
|
60
|
-
case "append":
|
|
61
|
-
entries.push(ev.entry);
|
|
62
|
-
break;
|
|
63
|
-
case "clear":
|
|
64
|
-
entries.length = 0;
|
|
65
|
-
cursor = undefined;
|
|
66
|
-
break;
|
|
67
|
-
case "cursor":
|
|
68
|
-
cursor = ev.cursor;
|
|
69
|
-
break;
|
|
70
|
-
case "update": {
|
|
71
|
-
const target = entries.find((e) => e.id === ev.id);
|
|
72
|
-
if (target) {
|
|
73
|
-
target.text = ev.text;
|
|
74
|
-
target.ts = ev.ts;
|
|
75
|
-
if (ev.description !== undefined)
|
|
76
|
-
target.description = ev.description;
|
|
77
|
-
}
|
|
78
|
-
break;
|
|
79
|
-
}
|
|
80
|
-
case "delete": {
|
|
81
|
-
const i = entries.findIndex((e) => e.id === ev.id);
|
|
82
|
-
if (i !== -1)
|
|
83
|
-
entries.splice(i, 1);
|
|
84
|
-
break;
|
|
85
|
-
}
|
|
86
|
-
default:
|
|
87
|
-
break;
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
const vectors = new Map();
|
|
91
|
-
const vpath = join(dir, "vectors.jsonl");
|
|
92
|
-
for (const ve of readJsonlRecords(vpath)) {
|
|
93
|
-
if (ve && typeof ve.id === "string" && Array.isArray(ve.embedding)) {
|
|
94
|
-
vectors.set(ve.id, { embedding: ve.embedding, h: typeof ve.h === "string" ? ve.h : "" });
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
const state = { entries, log: new AppendLog(notesPath), dir, cursor, vectors, vpath };
|
|
98
|
-
this.scopes.set(scope, state);
|
|
99
|
-
this.touchOpenLogs(scope);
|
|
100
|
-
return state;
|
|
101
|
-
}
|
|
102
|
-
touchOpenLogs(scope) {
|
|
103
|
-
const open = this.sharedScopes.openLogs;
|
|
104
|
-
open.delete(scope);
|
|
105
|
-
open.add(scope);
|
|
106
|
-
while (open.size > MAX_OPEN_MEMORY_SCOPES) {
|
|
107
|
-
const coldest = open.values().next().value;
|
|
108
|
-
if (coldest === undefined)
|
|
109
|
-
break;
|
|
110
|
-
open.delete(coldest);
|
|
111
|
-
const st = this.scopes.get(coldest);
|
|
112
|
-
st?.log.closeForSwap();
|
|
113
|
-
st?.vlog?.closeForSwap();
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
writeLog(scope, state) {
|
|
117
|
-
this.touchOpenLogs(scope);
|
|
118
|
-
return state.log;
|
|
119
|
-
}
|
|
120
|
-
vectorLog(scope, state) {
|
|
121
|
-
if (state.vlog === undefined)
|
|
122
|
-
state.vlog = new AppendLog(state.vpath);
|
|
123
|
-
this.touchOpenLogs(scope);
|
|
124
|
-
return state.vlog;
|
|
125
|
-
}
|
|
126
|
-
queueEmbed(scope, state, id, body) {
|
|
127
|
-
const emb = this.embedder;
|
|
128
|
-
if (!emb)
|
|
129
|
-
return;
|
|
130
|
-
const h = hashBody(body);
|
|
131
|
-
const job = Promise.race([
|
|
132
|
-
Promise.resolve().then(() => emb.embed(body)),
|
|
133
|
-
new Promise((r) => setTimeout(() => r(null), this.embedTimeoutMs).unref?.()),
|
|
134
|
-
])
|
|
135
|
-
.then((vec) => {
|
|
136
|
-
if (this.closed)
|
|
137
|
-
return;
|
|
138
|
-
if (!Array.isArray(vec) || vec.length !== emb.dimensions)
|
|
139
|
-
return;
|
|
140
|
-
const cur = state.entries.find((e) => e.id === id);
|
|
141
|
-
if (!cur || hashBody(cur.text) !== h)
|
|
142
|
-
return;
|
|
143
|
-
this.vectorLog(scope, state).append({ id, embedding: vec, h }, false);
|
|
144
|
-
state.vectors.set(id, { embedding: vec, h });
|
|
145
|
-
})
|
|
146
|
-
.catch(() => { });
|
|
147
|
-
this.pending.add(job);
|
|
148
|
-
void job.finally(() => this.pending.delete(job));
|
|
149
|
-
}
|
|
150
|
-
writeProjection(state) {
|
|
151
|
-
const body = state.entries.map(renderBullet).join("\n");
|
|
152
|
-
const target = join(state.dir, "MEMORY.md");
|
|
153
|
-
try {
|
|
154
|
-
atomicWriteFile(state.dir, target, body ? `${body}\n` : "");
|
|
155
|
-
}
|
|
156
|
-
catch (err) {
|
|
157
|
-
throw new MemoryProjectionError(target, err);
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
read(scope) {
|
|
161
|
-
const { entries } = this.scopeState(scope);
|
|
162
|
-
if (entries.length === 0)
|
|
163
|
-
return null;
|
|
164
|
-
return entries.map(renderBullet).join("\n");
|
|
165
|
-
}
|
|
166
|
-
append(scope, note) {
|
|
167
|
-
const text = note.trim().replace(/^#+\s*/, "");
|
|
168
|
-
if (!text)
|
|
169
|
-
return;
|
|
170
|
-
const entry = { id: uuidv7(), text, ts: nowTs() };
|
|
171
|
-
this.commit(scope, entry);
|
|
172
|
-
return entry.id;
|
|
173
|
-
}
|
|
174
|
-
appendStructured(scope, note) {
|
|
175
|
-
const body = note.body.trim().replace(/^#+\s*/, "");
|
|
176
|
-
if (!body)
|
|
177
|
-
return "";
|
|
178
|
-
const entry = {
|
|
179
|
-
id: uuidv7(),
|
|
180
|
-
text: body,
|
|
181
|
-
ts: nowTs(),
|
|
182
|
-
name: note.name,
|
|
183
|
-
type: note.type ?? "project",
|
|
184
|
-
description: note.description?.trim() || firstSentence(body),
|
|
185
|
-
links: note.links,
|
|
186
|
-
...(note.consolidationGenerated ? { consolidationGenerated: true } : {}),
|
|
187
|
-
};
|
|
188
|
-
this.commit(scope, entry);
|
|
189
|
-
return entry.id;
|
|
190
|
-
}
|
|
191
|
-
commit(scope, entry) {
|
|
192
|
-
const state = this.scopeState(scope);
|
|
193
|
-
this.writeLog(scope, state).append({ op: "append", entry }, true);
|
|
194
|
-
state.entries.push(entry);
|
|
195
|
-
this.writeProjection(state);
|
|
196
|
-
this.queueEmbed(scope, state, entry.id, entry.text);
|
|
197
|
-
}
|
|
198
|
-
clear(scope) {
|
|
199
|
-
const state = this.scopeState(scope);
|
|
200
|
-
this.writeLog(scope, state).append({ op: "clear" }, true);
|
|
201
|
-
state.entries.length = 0;
|
|
202
|
-
state.cursor = undefined;
|
|
203
|
-
state.vectors.clear();
|
|
204
|
-
this.writeProjection(state);
|
|
205
|
-
}
|
|
206
|
-
getConsolidationCursor(scope) {
|
|
207
|
-
return this.scopeState(scope).cursor;
|
|
208
|
-
}
|
|
209
|
-
setConsolidationCursor(scope, cursor) {
|
|
210
|
-
const state = this.scopeState(scope);
|
|
211
|
-
this.writeLog(scope, state).append({ op: "cursor", cursor }, true);
|
|
212
|
-
state.cursor = cursor;
|
|
213
|
-
}
|
|
214
|
-
search(scope, query, limit = 10) {
|
|
215
|
-
const { entries } = this.scopeState(scope);
|
|
216
|
-
if (entries.length === 0)
|
|
217
|
-
return [];
|
|
218
|
-
if (query.trim() === "")
|
|
219
|
-
return [];
|
|
220
|
-
return entries
|
|
221
|
-
.map(renderBullet)
|
|
222
|
-
.filter((b) => lexicalSearchMatch(query, b))
|
|
223
|
-
.slice(-limit);
|
|
224
|
-
}
|
|
225
|
-
listStructuredNotes(scope) {
|
|
226
|
-
const { entries } = this.scopeState(scope);
|
|
227
|
-
return entries.map((e) => ({
|
|
228
|
-
id: e.id,
|
|
229
|
-
description: e.description ?? firstSentence(e.text),
|
|
230
|
-
...parseNoteTimestamp(e.ts),
|
|
231
|
-
...(e.name ? { name: e.name } : {}),
|
|
232
|
-
...(e.type ? { type: e.type } : {}),
|
|
233
|
-
...(e.consolidationGenerated ? { consolidationGenerated: true } : {}),
|
|
234
|
-
}));
|
|
235
|
-
}
|
|
236
|
-
getByIds(scope, ids) {
|
|
237
|
-
const { entries } = this.scopeState(scope);
|
|
238
|
-
const want = new Set(ids);
|
|
239
|
-
return entries
|
|
240
|
-
.filter((e) => want.has(e.id))
|
|
241
|
-
.map((e) => ({
|
|
242
|
-
id: e.id,
|
|
243
|
-
text: e.text,
|
|
244
|
-
description: e.description ?? firstSentence(e.text),
|
|
245
|
-
...parseNoteTimestamp(e.ts),
|
|
246
|
-
...(e.name ? { name: e.name } : {}),
|
|
247
|
-
...(e.type ? { type: e.type } : {}),
|
|
248
|
-
...(e.consolidationGenerated ? { consolidationGenerated: true } : {}),
|
|
249
|
-
}));
|
|
250
|
-
}
|
|
251
|
-
searchScored(scope, query, limit = 20) {
|
|
252
|
-
const state = this.scopeState(scope);
|
|
253
|
-
if (state.entries.length === 0)
|
|
254
|
-
return [];
|
|
255
|
-
if (!this.embedder)
|
|
256
|
-
return lexicalScored(state.entries, query, limit);
|
|
257
|
-
const emb = this.embedder;
|
|
258
|
-
return Promise.race([
|
|
259
|
-
Promise.resolve().then(() => emb.embed(query)),
|
|
260
|
-
new Promise((r) => setTimeout(() => r(null), this.embedTimeoutMs).unref?.()),
|
|
261
|
-
]).then((qv) => (Array.isArray(qv) && qv.length === emb.dimensions ? vectorScored(state, qv, query, emb.dimensions, limit) : lexicalScored(state.entries, query, limit)), () => lexicalScored(state.entries, query, limit));
|
|
262
|
-
}
|
|
263
|
-
update(scope, id, text) {
|
|
264
|
-
const clean = text.trim();
|
|
265
|
-
if (!clean)
|
|
266
|
-
return;
|
|
267
|
-
const state = this.scopeState(scope);
|
|
268
|
-
const entry = state.entries.find((e) => e.id === id);
|
|
269
|
-
if (!entry)
|
|
270
|
-
return;
|
|
271
|
-
const newBody = clean.replace(/^#+\s*/, "");
|
|
272
|
-
const newDesc = entry.description !== undefined && entry.description === firstSentence(entry.text)
|
|
273
|
-
? firstSentence(newBody)
|
|
274
|
-
: entry.description;
|
|
275
|
-
const ts = nowTs();
|
|
276
|
-
const ev = { op: "update", id, text: newBody, ts };
|
|
277
|
-
if (newDesc !== undefined)
|
|
278
|
-
ev.description = newDesc;
|
|
279
|
-
this.writeLog(scope, state).append(ev, true);
|
|
280
|
-
entry.text = newBody;
|
|
281
|
-
entry.ts = ts;
|
|
282
|
-
entry.description = newDesc;
|
|
283
|
-
this.writeProjection(state);
|
|
284
|
-
state.vectors.delete(id);
|
|
285
|
-
this.queueEmbed(scope, state, id, newBody);
|
|
286
|
-
}
|
|
287
|
-
delete(scope, id) {
|
|
288
|
-
const state = this.scopeState(scope);
|
|
289
|
-
const i = state.entries.findIndex((e) => e.id === id);
|
|
290
|
-
if (i === -1)
|
|
291
|
-
return;
|
|
292
|
-
this.writeLog(scope, state).append({ op: "delete", id }, true);
|
|
293
|
-
state.entries.splice(i, 1);
|
|
294
|
-
this.writeProjection(state);
|
|
295
|
-
}
|
|
296
|
-
close() {
|
|
297
|
-
if (this.closed)
|
|
298
|
-
return;
|
|
299
|
-
this.closed = true;
|
|
300
|
-
this.sharedScopes.refs -= 1;
|
|
301
|
-
if (this.sharedScopes.refs > 0)
|
|
302
|
-
return;
|
|
303
|
-
if (sharedMemoryDirs.get(this.scopeKey) === this.sharedScopes)
|
|
304
|
-
sharedMemoryDirs.delete(this.scopeKey);
|
|
305
|
-
for (const s of this.scopes.values()) {
|
|
306
|
-
s.log.close();
|
|
307
|
-
s.vlog?.close();
|
|
308
|
-
}
|
|
309
|
-
this.sharedScopes.openLogs.clear();
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
function renderBullet(e) {
|
|
313
|
-
return `- (${e.ts} UTC) ${e.text}`;
|
|
314
|
-
}
|
|
315
|
-
function lexicalScored(entries, query, limit) {
|
|
316
|
-
const q = termSet(query);
|
|
317
|
-
if (q.size === 0)
|
|
318
|
-
return [];
|
|
319
|
-
const scored = [];
|
|
320
|
-
for (const e of entries) {
|
|
321
|
-
const d = jaccardDistance(q, e.text);
|
|
322
|
-
if (d === null)
|
|
323
|
-
continue;
|
|
324
|
-
scored.push({ id: e.id, text: e.text, score: d, ...(e.consolidationGenerated ? { consolidationGenerated: true } : {}) });
|
|
325
|
-
}
|
|
326
|
-
scored.sort((a, b) => a.score - b.score);
|
|
327
|
-
return scored.slice(0, limit);
|
|
328
|
-
}
|
|
329
|
-
function vectorScored(state, qv, query, dims, limit) {
|
|
330
|
-
const qTerms = termSet(query);
|
|
331
|
-
const scored = [];
|
|
332
|
-
for (const e of state.entries) {
|
|
333
|
-
const ev = state.vectors.get(e.id);
|
|
334
|
-
const usable = ev && ev.embedding.length === dims && ev.h === hashBody(e.text);
|
|
335
|
-
const score = usable ? cosineDistance(qv, ev.embedding) : jaccardDistance(qTerms, e.text);
|
|
336
|
-
if (score === null)
|
|
337
|
-
continue;
|
|
338
|
-
scored.push({ id: e.id, text: e.text, score, ...(e.consolidationGenerated ? { consolidationGenerated: true } : {}) });
|
|
339
|
-
}
|
|
340
|
-
scored.sort((a, b) => a.score - b.score);
|
|
341
|
-
return scored.slice(0, limit);
|
|
342
|
-
}
|
|
343
|
-
function hashBody(s) {
|
|
344
|
-
let h = 0x811c9dc5;
|
|
345
|
-
for (let i = 0; i < s.length; i++) {
|
|
346
|
-
h ^= s.charCodeAt(i);
|
|
347
|
-
h = Math.imul(h, 0x01000193);
|
|
348
|
-
}
|
|
349
|
-
return (h >>> 0).toString(16);
|
|
350
|
-
}
|
|
351
|
-
function nowTs() {
|
|
352
|
-
return new Date().toISOString().replace("T", " ").slice(0, 16);
|
|
353
|
-
}
|