@staix/agent-mem 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -6
- package/package.json +4 -3
- package/src/cli.ts +116 -15
- package/src/commands-table.ts +15 -2
- package/src/index.ts +13 -0
- package/src/mcp-memory.ts +151 -0
- package/src/mcp.ts +122 -0
- package/src/memory/candidates.ts +12 -8
- package/src/memory/capacity.ts +179 -0
- package/src/memory/capture.ts +460 -0
- package/src/memory/catalog.ts +136 -0
- package/src/memory/commands.ts +7 -4
- package/src/memory/generation-index.ts +54 -11
- package/src/memory/generation.ts +41 -2
- package/src/memory/home.ts +5 -0
- package/src/memory/import.ts +60 -0
- package/src/memory/migrate.ts +8 -0
- package/src/memory/mutation.ts +13 -1
- package/src/memory/ops-contract.ts +19 -2
- package/src/memory/policy.ts +120 -0
- package/src/memory/project.ts +22 -1
- package/src/memory/reader-lease.ts +136 -0
- package/src/memory/screen.ts +8 -0
- package/src/memory/search-fts.ts +203 -0
- package/src/memory/shard-bridge.ts +19 -0
- package/src/memory/shard-migration.ts +562 -0
- package/src/memory/shard.ts +179 -0
- package/src/memory/store.ts +298 -60
- package/src/memory/vault.ts +40 -0
- package/src/memory/work-lists.ts +243 -0
package/src/memory/store.ts
CHANGED
|
@@ -1,16 +1,22 @@
|
|
|
1
1
|
import { Database } from "bun:sqlite";
|
|
2
2
|
import { createHash, randomBytes } from "node:crypto";
|
|
3
|
-
import { appendFileSync, chmodSync, closeSync, existsSync, fstatSync, linkSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { appendFileSync, chmodSync, closeSync, constants, existsSync, fstatSync, fsyncSync, linkSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { isAbsolute, join, normalize } from "node:path";
|
|
5
5
|
import { isDenied } from "./platform.ts";
|
|
6
|
+
import { looksSecret } from "./screen.ts";
|
|
6
7
|
import { mutationDatabase } from "./mutation.ts";
|
|
7
8
|
import { memoryHome } from "./home.ts";
|
|
9
|
+
import { SearchIndex, captureSearch } from "./search-fts.ts";
|
|
10
|
+
import { assertCuratedShardPaths, ensureShardDescriptor, readShardDescriptor, readShardedSelection, resolveAlias, resolveShardProfile, shardDir, shardKeyHash, shardMachineDir, userShardDir } from "./shard.ts";
|
|
11
|
+
import { readCaptureConfig, type CaptureRow } from "./capture.ts";
|
|
8
12
|
import { assertSharedV1Op } from "./ops-contract.ts";
|
|
9
|
-
import { generationDescriptor, generationDir, canonicalJSON, readProfileJSON, resolveProfile, type BoundProfile, type ProfileInput } from "./generation.ts";
|
|
13
|
+
import { generationDescriptor, generationDir, fsyncDirectory, canonicalJSON, hashJSON, readProfileJSON, resolveProfile, writeProfileJSON, type BoundProfile, type ProfileInput, validateVaultLinkRow, vaultLinkId, type VaultLink, type VaultLinkRow } from "./generation.ts";
|
|
10
14
|
import { GenerationIndex, explicitExpirySnapshot, GENERATION_TABLES } from "./generation-index.ts";
|
|
11
15
|
import { legacyEntries, type LegacyCheckpoint } from "./bridge.ts";
|
|
16
|
+
import { readMigrationSources } from "./migrate.ts";
|
|
12
17
|
import type { PruneStorePort } from "./prune.ts";
|
|
13
18
|
import { isDeepStrictEqual } from "node:util";
|
|
19
|
+
import { registerReaderLease } from "./reader-lease.ts";
|
|
14
20
|
|
|
15
21
|
/**
|
|
16
22
|
* #258: the hub's own memory. Append-only JSONL logs are the truth: `log/<machine>.jsonl` holds long-term entries and
|
|
@@ -79,10 +85,8 @@ function canonical(value: unknown): string {
|
|
|
79
85
|
}
|
|
80
86
|
export const entryId = (e: Omit<Entry, "id">): string => createHash("sha256").update(canonical(e)).digest("hex").slice(0, 32);
|
|
81
87
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
/** A secret-looking string: refused before anything is written, since long-term entries leave the machine. */
|
|
85
|
-
export const looksSecret = (text: string): boolean => SECRETS.some((re) => re.test(text));
|
|
88
|
+
// Single-sourced in screen.ts so the capture tier (#13) screens excerpts without importing this store; the export stays.
|
|
89
|
+
export { looksSecret };
|
|
86
90
|
|
|
87
91
|
/** `task`: the board task's id and proposal time; its node carries both, so a task id reused after `ahub reset --all` is another node. */
|
|
88
92
|
export interface Extract { cwd: string; project: string; repo: string; deny?: string[]; paths?: string[]; symbols?: string[]; task?: { id: number; created: number }; checkExists?: boolean }
|
|
@@ -162,6 +166,7 @@ export interface View {
|
|
|
162
166
|
/** Written as short-term, so it lives in this machine's local segment and never syncs. */
|
|
163
167
|
local: boolean;
|
|
164
168
|
}
|
|
169
|
+
export interface VaultEntry { view: View; /** Full authored hash in generation profiles; empty for read-only shared-v1. */ rowHash: string }
|
|
165
170
|
export interface GoalView {
|
|
166
171
|
id: string; at: string; machine: string; project: string; scope: Scope; title: string; milestones: string[]; dependsOn: string[];
|
|
167
172
|
state: GoalState; links: { task: string; at: string; machine: string }[];
|
|
@@ -169,6 +174,14 @@ export interface GoalView {
|
|
|
169
174
|
lease: { holder: string | null; until: string | null; conflict: string[]; expired: boolean; lastHolder?: string | null };
|
|
170
175
|
}
|
|
171
176
|
export interface Filter { project?: string; tier?: Tier; kind?: Kind; peer?: string; machine?: string; since?: string; q?: string; limit?: number; includeSuperseded?: boolean }
|
|
177
|
+
/** #13 search: `Filter` plus the capture-tier switch (on by default; capture is recall, never curated evidence). */
|
|
178
|
+
export interface SearchFilter extends Filter {
|
|
179
|
+
includeCapture?: boolean;
|
|
180
|
+
/** A host that writes capture with a `retentionDays` override passes the same value here; files decide otherwise. */
|
|
181
|
+
captureRetentionDays?: number;
|
|
182
|
+
}
|
|
183
|
+
export interface SearchHit { source: "curated" | "capture"; score: number; view?: View; capture?: CaptureRow }
|
|
184
|
+
export interface SearchResult { engine: "fts5" | "substring"; hits: SearchHit[] }
|
|
172
185
|
export interface Graph { nodes: { id: Node; type: string; label: string; kind?: string; tier?: Tier }[]; edges: { from: Node; to: Node; rel: string }[] }
|
|
173
186
|
|
|
174
187
|
/**
|
|
@@ -180,11 +193,11 @@ export type PromotionScreen = (v: View) => "promote" | "keep" | "skip";
|
|
|
180
193
|
* `screen`: applied to every promotion. Without one, automatic promotions (a settlement, a held note whose replacement
|
|
181
194
|
* went) wait for a process that has one; a person's explicit promote goes through.
|
|
182
195
|
*/
|
|
183
|
-
export interface StoreOptions { home?: string; ttlMs?: number; now?: () => number; log?: (line: string) => void; screen?: PromotionScreen; deferPromotions?: boolean; profile?: ProfileInput;
|
|
196
|
+
export interface StoreOptions { home?: string; memoryDir?: string; ttlMs?: number; now?: () => number; log?: (line: string) => void; screen?: PromotionScreen; deferPromotions?: boolean; profile?: ProfileInput;
|
|
184
197
|
}
|
|
185
198
|
|
|
186
199
|
const MIGRATION_BOOTSTRAP = Symbol("migration index bootstrap");
|
|
187
|
-
interface BootstrapBinding { token: typeof MIGRATION_BOOTSTRAP; profile: BoundProfile; limits
|
|
200
|
+
interface BootstrapBinding { token: typeof MIGRATION_BOOTSTRAP; profile: BoundProfile; limits?: readonly LegacyCheckpoint[]; commit?: string; authoring?: boolean; userOverlay?: MemoryStore }
|
|
188
201
|
export class MemoryStore {
|
|
189
202
|
readonly dir: string;
|
|
190
203
|
readonly profile: BoundProfile;
|
|
@@ -197,49 +210,73 @@ export class MemoryStore {
|
|
|
197
210
|
private readonly now: () => number;
|
|
198
211
|
private readonly log: (line: string) => void;
|
|
199
212
|
private readonly deferPromotions: boolean;
|
|
213
|
+
/** #13: the derived FTS5 copy of the curated rows, and the memory home base for capture shard resolution (#15 T1). */
|
|
214
|
+
private readonly searchIndex: SearchIndex;
|
|
215
|
+
private readonly memoryBaseDir: string;
|
|
216
|
+
private releaseReaderLease?: () => void;
|
|
217
|
+
private readonly userOverlay?: MemoryStore;
|
|
218
|
+
/** A full rebuild refills FTS in one resync instead of one derived transaction per entry. */
|
|
219
|
+
private rebuildingSearch = false;
|
|
200
220
|
|
|
201
221
|
constructor(opts: StoreOptions = {}, bootstrap?: BootstrapBinding) {
|
|
202
222
|
if (bootstrap && bootstrap.token !== MIGRATION_BOOTSTRAP) throw new Error("invalid migration bootstrap");
|
|
203
|
-
this.bootstrapOnly = bootstrap !==
|
|
204
|
-
|
|
223
|
+
this.bootstrapOnly = bootstrap ? bootstrap.authoring !== true : false;
|
|
224
|
+
this.userOverlay = bootstrap?.userOverlay;
|
|
225
|
+
if (opts.memoryDir !== undefined && (opts.home !== undefined || typeof opts.memoryDir !== "string" || !opts.memoryDir.trim())) throw new Error("choose a memory home or one explicit memory directory");
|
|
226
|
+
const home = bootstrap?.profile.home ?? opts.memoryDir ?? (opts.home === undefined ? memoryHome() : join(opts.home, "memory"));
|
|
227
|
+
if (!bootstrap && lstatSync(join(home, "shards.json"), { throwIfNoEntry: false })) throw new Error("memory home has a curated split; hosts must use openShardStore({ project }) or openShardStore({ user: true }); finish an incomplete split with amem migrate --shards --resume");
|
|
205
228
|
this.profile = bootstrap?.profile ?? resolveProfile(home, opts.profile);
|
|
206
229
|
if (bootstrap && (this.profile.kind !== "generation" || !this.profile.generation
|
|
207
|
-
|| this.profile.dir !== generationDir(this.profile.home, this.profile.generation)
|
|
230
|
+
|| (!this.profile.sharded && this.profile.dir !== generationDir(this.profile.home, this.profile.generation))
|
|
208
231
|
|| canonicalJSON(readProfileJSON(join(this.profile.dir, "generation.json"))) !== canonicalJSON(generationDescriptor(this.profile.generation)))) throw new Error("invalid migration profile binding");
|
|
209
232
|
// Shared-v1 preserves its historical lexical dir API; profile.home is still canonical for selector identity.
|
|
210
233
|
this.dir = this.profile.kind === "shared-v1" ? home : this.profile.dir;
|
|
211
234
|
mkdirSync(join(this.dir, "log"), { recursive: true, mode: 0o700 });
|
|
212
235
|
mkdirSync(join(this.dir, "local"), { recursive: true, mode: 0o700 });
|
|
213
236
|
chmodSync(this.dir, 0o700); // mkdir's mode applies only when it creates the directory
|
|
214
|
-
this.machine = machineId(this.dir);
|
|
237
|
+
this.machine = machineId(this.profile.sharded ? this.profile.home : this.dir);
|
|
215
238
|
this.ttl = opts.ttlMs ?? DEFAULT_TTL_MS;
|
|
216
239
|
this.now = opts.now ?? Date.now;
|
|
217
240
|
this.log = opts.log ?? (() => {});
|
|
218
241
|
this.screen = opts.screen;
|
|
219
242
|
this.deferPromotions = opts.deferPromotions ?? false;
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
db.run(`PRAGMA
|
|
243
|
+
// Announce before opening derived handles or reading rows; failed admission leaks no database handles.
|
|
244
|
+
if (!bootstrap) this.releaseReaderLease = registerReaderLease(this.profile.home);
|
|
245
|
+
let openedDb: Database | undefined, openedMutation: Database | undefined, openedSearch: SearchIndex | undefined;
|
|
246
|
+
try {
|
|
247
|
+
const db = openedDb = new Database(join(this.dir, "index.db"), { create: true });
|
|
248
|
+
db.run(`PRAGMA busy_timeout = ${MEMORY_DB_BUSY_TIMEOUT_MS}`);
|
|
249
|
+
db.run("PRAGMA journal_mode = WAL");
|
|
250
|
+
if ((db.query("PRAGMA user_version").get() as { user_version: number }).user_version !== INDEX_VERSION) {
|
|
251
|
+
for (const t of ["segments", "heads", "entries", "ops", "edges", "goals"]) db.run(`DROP TABLE IF EXISTS ${t}`);
|
|
252
|
+
db.run(`PRAGMA user_version = ${INDEX_VERSION}`);
|
|
253
|
+
}
|
|
254
|
+
db.run("CREATE TABLE IF NOT EXISTS segments (file TEXT PRIMARY KEY, offset INTEGER NOT NULL)");
|
|
255
|
+
// The hash of each segment's whole indexed prefix: a segment rewritten in place (same size or larger) is caught.
|
|
256
|
+
db.run("CREATE TABLE IF NOT EXISTS heads (file TEXT PRIMARY KEY, hash TEXT NOT NULL)");
|
|
257
|
+
db.run("CREATE TABLE IF NOT EXISTS goals (id TEXT PRIMARY KEY, at TEXT NOT NULL, machine TEXT NOT NULL, project TEXT NOT NULL, scope TEXT NOT NULL, title TEXT NOT NULL, milestones TEXT NOT NULL, depends TEXT NOT NULL, actor TEXT NOT NULL)");
|
|
258
|
+
db.run(`CREATE TABLE IF NOT EXISTS entries (id TEXT PRIMARY KEY, at TEXT NOT NULL, machine TEXT NOT NULL, project TEXT NOT NULL, scope TEXT NOT NULL, tier TEXT NOT NULL,
|
|
259
|
+
kind TEXT NOT NULL, peer TEXT NOT NULL, title TEXT, text TEXT NOT NULL, task TEXT, about TEXT NOT NULL, supersedes TEXT, promoted_from TEXT, source TEXT NOT NULL, expires TEXT)`);
|
|
260
|
+
db.run("CREATE TABLE IF NOT EXISTS ops (id TEXT PRIMARY KEY, op TEXT NOT NULL, at TEXT NOT NULL, machine TEXT NOT NULL, target TEXT NOT NULL, by TEXT, actor TEXT NOT NULL)");
|
|
261
|
+
db.run("CREATE TABLE IF NOT EXISTS edges (src TEXT NOT NULL, dst TEXT NOT NULL, PRIMARY KEY (src, dst))");
|
|
262
|
+
db.run("CREATE INDEX IF NOT EXISTS edges_dst ON edges (dst)");
|
|
263
|
+
this.db = db;
|
|
264
|
+
this.mutationDb = openedMutation = mutationDatabase(this.dir);
|
|
265
|
+
// One derived FTS copy per profile: a generation handle and a shared-v1 handle on one home never resync-fight.
|
|
266
|
+
this.searchIndex = openedSearch = new SearchIndex(this.profile.sharded ? join(this.dir, "search.db") : join(home, this.profile.kind === "generation" ? `search-${this.profile.generation}.db` : "search.db"));
|
|
267
|
+
this.memoryBaseDir = home;
|
|
268
|
+
try { chmodSync(join(this.dir, "index.db"), 0o600); } catch { /* best effort */ }
|
|
269
|
+
if (this.profile.kind === "generation") this.generationIndex = new GenerationIndex({ profile: this.profile, db, now: this.now,
|
|
270
|
+
apply: (line, source) => this.apply(line, source), withMutation: work => this.withMutation(work),
|
|
271
|
+
...(bootstrap ? { limits: bootstrap.limits } : {}), ...(bootstrap?.commit ? { commit: bootstrap.commit } : {}) });
|
|
272
|
+
this.refresh();
|
|
273
|
+
} catch (error) {
|
|
274
|
+
for (const handle of [openedSearch, openedMutation, openedDb]) { try { handle?.close(); } catch { /* preserve the opening failure */ } }
|
|
275
|
+
const release = this.releaseReaderLease;
|
|
276
|
+
this.releaseReaderLease = undefined;
|
|
277
|
+
release?.();
|
|
278
|
+
throw error;
|
|
226
279
|
}
|
|
227
|
-
db.run("CREATE TABLE IF NOT EXISTS segments (file TEXT PRIMARY KEY, offset INTEGER NOT NULL)");
|
|
228
|
-
// The hash of each segment's whole indexed prefix: a segment rewritten in place (same size or larger) is caught.
|
|
229
|
-
db.run("CREATE TABLE IF NOT EXISTS heads (file TEXT PRIMARY KEY, hash TEXT NOT NULL)");
|
|
230
|
-
db.run("CREATE TABLE IF NOT EXISTS goals (id TEXT PRIMARY KEY, at TEXT NOT NULL, machine TEXT NOT NULL, project TEXT NOT NULL, scope TEXT NOT NULL, title TEXT NOT NULL, milestones TEXT NOT NULL, depends TEXT NOT NULL, actor TEXT NOT NULL)");
|
|
231
|
-
db.run(`CREATE TABLE IF NOT EXISTS entries (id TEXT PRIMARY KEY, at TEXT NOT NULL, machine TEXT NOT NULL, project TEXT NOT NULL, scope TEXT NOT NULL, tier TEXT NOT NULL,
|
|
232
|
-
kind TEXT NOT NULL, peer TEXT NOT NULL, title TEXT, text TEXT NOT NULL, task TEXT, about TEXT NOT NULL, supersedes TEXT, promoted_from TEXT, source TEXT NOT NULL, expires TEXT)`);
|
|
233
|
-
db.run("CREATE TABLE IF NOT EXISTS ops (id TEXT PRIMARY KEY, op TEXT NOT NULL, at TEXT NOT NULL, machine TEXT NOT NULL, target TEXT NOT NULL, by TEXT, actor TEXT NOT NULL)");
|
|
234
|
-
db.run("CREATE TABLE IF NOT EXISTS edges (src TEXT NOT NULL, dst TEXT NOT NULL, PRIMARY KEY (src, dst))");
|
|
235
|
-
db.run("CREATE INDEX IF NOT EXISTS edges_dst ON edges (dst)");
|
|
236
|
-
this.db = db;
|
|
237
|
-
this.mutationDb = mutationDatabase(this.dir);
|
|
238
|
-
try { chmodSync(join(this.dir, "index.db"), 0o600); } catch { /* best effort */ }
|
|
239
|
-
if (this.profile.kind === "generation") this.generationIndex = new GenerationIndex({ profile: this.profile, db, now: this.now,
|
|
240
|
-
apply: (line, source) => this.apply(line, source), withMutation: work => this.withMutation(work),
|
|
241
|
-
...(bootstrap ? { limits: bootstrap.limits } : {}), ...(bootstrap?.commit ? { commit: bootstrap.commit } : {}) });
|
|
242
|
-
this.refresh();
|
|
243
280
|
}
|
|
244
281
|
|
|
245
282
|
/** The host must authenticate person intent before authorizing plan/apply through this port. */
|
|
@@ -248,13 +285,68 @@ export class MemoryStore {
|
|
|
248
285
|
return this.generationIndex.port(this.machine.id);
|
|
249
286
|
}
|
|
250
287
|
|
|
288
|
+
/** Read-only project views, bounded before bodies or hashes; never promotes or authors a prune. */
|
|
289
|
+
vaultEntries(project: string): VaultEntry[] {
|
|
290
|
+
if (!project.trim() || this.profile.sharded && !this.profile.shardUser && this.profile.shardKey !== project) throw new Error("vault project does not match the bound store");
|
|
291
|
+
this.withDeferredPromotions(() => this.ensureFresh());
|
|
292
|
+
const scanLimit = 5000;
|
|
293
|
+
const own = this.db.query("SELECT id FROM entries WHERE project=? LIMIT ?").all(project, scanLimit + 1) as { id: string }[];
|
|
294
|
+
if (own.length > scanLimit) throw new Error("vault scan budget exceeded; refine the store before retrying");
|
|
295
|
+
const overlay = new Map(this.userOverlay?.vaultEntries(project).map(entry => [entry.view.id, entry.rowHash]));
|
|
296
|
+
if (new Set([...own.map(row => row.id), ...overlay.keys()]).size > scanLimit) throw new Error("vault scan budget exceeded; refine the store before retrying");
|
|
297
|
+
const authored = this.generationIndex?.vaultWitnesses(own.map(row => row.id));
|
|
298
|
+
return this.views(false, { project }).filter(view => view.project === project).map(view => {
|
|
299
|
+
// A shared-v1 read cannot record links and needs no authored witness or new index field.
|
|
300
|
+
const rowHash = this.generationIndex ? authored?.get(view.id) ?? overlay.get(view.id) : "";
|
|
301
|
+
if (rowHash === undefined) throw new Error("authored vault entry witness unavailable");
|
|
302
|
+
return { view, rowHash };
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
vaultLinks(project: string): VaultLink[] {
|
|
306
|
+
this.ensureFresh();
|
|
307
|
+
if (!project.trim() || this.profile.sharded && !this.profile.shardUser && this.profile.shardKey !== project) throw new Error("vault project does not match the bound store");
|
|
308
|
+
return [...(this.generationIndex?.vaultLinks(project) ?? []), ...(this.userOverlay?.vaultLinks(project) ?? [])];
|
|
309
|
+
}
|
|
310
|
+
/** Exact id/hash recheck under the existing mutation fence; never follows promoted/superseded aliases. */
|
|
311
|
+
recordVaultLink(id: string, path: string, project: string, rowHash: string): string {
|
|
312
|
+
if (this.bootstrapOnly || !this.generationIndex) throw new Error("vault links require an active generation handle");
|
|
313
|
+
if (this.userOverlay?.vaultEntries(project).some(entry => entry.view.id === id)) return this.userOverlay.recordVaultLink(id, path, project, rowHash);
|
|
314
|
+
return this.withMutation(() => {
|
|
315
|
+
this.refresh();
|
|
316
|
+
const entry = this.vaultEntries(project).find(row => row.view.id === id && row.rowHash === rowHash);
|
|
317
|
+
if (!entry || entry.view.tier !== "long" || entry.view.local || entry.view.supersededBy) throw new Error("exact vault entry changed or unavailable");
|
|
318
|
+
const prior = this.vaultLinks(project).find(row => row.target === id && row.rowHash === rowHash && row.path === path);
|
|
319
|
+
if (prior) return prior.id;
|
|
320
|
+
const body = { schema: "agent-mem.vault-link/v1", op: "vault_link", generation: this.profile.generation!,
|
|
321
|
+
at: new Date(this.now()).toISOString(), machine: this.machine.id, actor: "person", placement: "local", target: id, rowHash, project, path } as const;
|
|
322
|
+
const row: VaultLinkRow = { ...body, id: vaultLinkId(body) };
|
|
323
|
+
validateVaultLinkRow(row, this.profile.generation, "local");
|
|
324
|
+
this.generationIndex!.assertAppendable(true, this.machine.id);
|
|
325
|
+
const file = join(this.dir, "local", `${this.machine.id}.jsonl`), existed = existsSync(file);
|
|
326
|
+
const fd = openSync(file, constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW, 0o600);
|
|
327
|
+
try {
|
|
328
|
+
const stat = fstatSync(fd);
|
|
329
|
+
if (!stat.isFile() || stat.nlink !== 1) throw new Error("vault link segment is not an owned regular file");
|
|
330
|
+
appendFileSync(fd, canonicalJSON(row) + "\n"); fsyncSync(fd);
|
|
331
|
+
} finally { closeSync(fd); }
|
|
332
|
+
if (!existed) fsyncDirectory(join(this.dir, "local"));
|
|
333
|
+
try { this.refresh(); } catch (error) { this.stale = true; throw error; }
|
|
334
|
+
return row.id;
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
|
|
251
338
|
/** Full derived receipt for golden replay; raw debug data, never model evidence. */
|
|
252
339
|
derivedTables(): Record<string, unknown[]> {
|
|
253
340
|
const tables = this.db.query("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name").all() as { name: string }[];
|
|
254
341
|
return Object.fromEntries(tables.map(({ name }) => [name, (this.db.query(`SELECT * FROM "${name.replaceAll('"', '""')}"`).all() as unknown[]).sort((a, b) => canonicalJSON(a).localeCompare(canonicalJSON(b)))]));
|
|
255
342
|
}
|
|
256
343
|
|
|
257
|
-
close(): void {
|
|
344
|
+
close(): void {
|
|
345
|
+
const release = this.releaseReaderLease;
|
|
346
|
+
this.releaseReaderLease = undefined;
|
|
347
|
+
try { this.db.close(); this.mutationDb.close(); this.searchIndex.close(); }
|
|
348
|
+
finally { try { this.userOverlay?.close(); } finally { release?.(); } }
|
|
349
|
+
}
|
|
258
350
|
|
|
259
351
|
/** A synchronous host-authorized mutation whose reads must not implicitly promote unrelated notes. */
|
|
260
352
|
withDeferredPromotions<T>(work: () => T): T {
|
|
@@ -269,6 +361,7 @@ export class MemoryStore {
|
|
|
269
361
|
|
|
270
362
|
/** Serialize current-store writers, screen one normalized body, and expose a one-shot append of that body. */
|
|
271
363
|
withAcceptedNote<T>(note: NoteInput, work: (append: () => string) => T): T {
|
|
364
|
+
if (this.userOverlay && note.scope === "shared") return this.userOverlay.withAcceptedNote(note, work);
|
|
272
365
|
try { return this.withMutation(() => {
|
|
273
366
|
// Ordinary writes may defer indexing, but accepting an exact target revision must have a current index.
|
|
274
367
|
this.db.transaction(() => this.refresh()).immediate();
|
|
@@ -307,6 +400,14 @@ export class MemoryStore {
|
|
|
307
400
|
|
|
308
401
|
private appendBody(body: Omit<Entry, "id">, local: boolean): string {
|
|
309
402
|
if (this.bootstrapOnly) throw new Error("migration bootstrap cannot author memory");
|
|
403
|
+
if (this.userOverlay) {
|
|
404
|
+
if ((body.op === "note" || body.op === "goal") && "scope" in body && body.scope === "shared") return this.userOverlay.appendBody(body, local);
|
|
405
|
+
if ("target" in body && body.op !== "settle" && this.userOverlay.ownsTarget(body.target as string)) return this.userOverlay.appendBody(body, local);
|
|
406
|
+
}
|
|
407
|
+
if (this.profile.sharded && (body.op === "note" || body.op === "goal") && "scope" in body && "project" in body) {
|
|
408
|
+
if (this.profile.shardUser ? body.scope !== "shared" : body.scope !== "project" || body.project !== this.profile.shardKey) throw new Error("entry does not belong to this curated shard");
|
|
409
|
+
}
|
|
410
|
+
if (this.profile.sharded && body.op === "settle" && "target" in body && (typeof body.target !== "string" || (!this.profile.shardUser && !body.target.startsWith(`${this.profile.shardKey}#`)))) throw new Error("settlement does not belong to this curated shard");
|
|
310
411
|
assertSharedV1Op(body.op);
|
|
311
412
|
// Lock the authoritative log independently: a busy derived index must never prevent an ordinary append.
|
|
312
413
|
return this.withMutation(() => {
|
|
@@ -330,6 +431,8 @@ export class MemoryStore {
|
|
|
330
431
|
*/
|
|
331
432
|
importNote(n: NoteInput, at: string): { id: string; added: boolean } {
|
|
332
433
|
if (this.bootstrapOnly) throw new Error("migration bootstrap cannot import memory");
|
|
434
|
+
if (this.userOverlay && n.scope === "shared") return this.userOverlay.importNote(n, at);
|
|
435
|
+
if (this.profile.sharded && (this.profile.shardUser ? n.scope !== "shared" : n.scope !== "project" || n.project !== this.profile.shardKey)) throw new Error("entry does not belong to this curated shard");
|
|
333
436
|
assertSharedV1Op(n.op);
|
|
334
437
|
// A fixed machine: the same observation imported on two machines is one entry once they sync.
|
|
335
438
|
const body = { schema: MEMORY_SCHEMA, at, machine: "claude-mem", ...n, text: n.text.slice(0, MAX_TEXT), ...(n.title ? { title: n.title.slice(0, MAX_TITLE) } : {}) } as Omit<Entry, "id">;
|
|
@@ -419,6 +522,7 @@ export class MemoryStore {
|
|
|
419
522
|
*/
|
|
420
523
|
promote(id: string, actor: string, project?: string, auto = false): string | undefined {
|
|
421
524
|
const v = this.get(id, project);
|
|
525
|
+
if (v && this.userOverlay?.ownsTarget(v.id)) return this.userOverlay.promote(v.id, actor, project, auto);
|
|
422
526
|
if (v && this.generationIndex && !this.generationIndex.own(v.id)) return undefined;
|
|
423
527
|
if (!v || v.tier !== "short") return v?.id;
|
|
424
528
|
if (v.supersededBy) return undefined;
|
|
@@ -442,8 +546,11 @@ export class MemoryStore {
|
|
|
442
546
|
settle(task: string, actor: string): number {
|
|
443
547
|
this.promoting = true; // promoted here, in the settling actor's name, not after each read
|
|
444
548
|
try {
|
|
445
|
-
|
|
446
|
-
|
|
549
|
+
const entry = { op: "settle", target: task, actor, by: this.until() } as const;
|
|
550
|
+
const at = new Date(this.now()).toISOString();
|
|
551
|
+
this.append(entry, true, at);
|
|
552
|
+
if (this.userOverlay) this.userOverlay.append(entry, true, at, this.machine.id);
|
|
553
|
+
return this.promoteSettled(task, actor) + (this.userOverlay?.promoteSettled(task, actor) ?? 0);
|
|
447
554
|
} finally { this.promoting = false; }
|
|
448
555
|
}
|
|
449
556
|
private promoteSettled(task: string, actor: string): number {
|
|
@@ -451,14 +558,14 @@ export class MemoryStore {
|
|
|
451
558
|
const moving = this.views().filter((v) => (!this.generationIndex || this.generationIndex.own(v.id)) && v.task === task && v.tier === "short" && v.local && !v.supersededBy && v.expiresAt === null && PROMOTED_KINDS.includes(v.kind));
|
|
452
559
|
return moving.filter((v) => this.promote(v.id, actor, undefined, true) !== undefined).length;
|
|
453
560
|
}
|
|
454
|
-
/** Whether a task was
|
|
455
|
-
settled(task: string): boolean {
|
|
456
|
-
return
|
|
561
|
+
/** Whether a task was settled in this profile. Reconciliation may opt into bridged legacy settlements. */
|
|
562
|
+
settled(task: string, options: { includeLegacy?: boolean } = {}): boolean {
|
|
563
|
+
return this.readRows(this.generationIndex && !options.includeLegacy
|
|
457
564
|
? "SELECT 1 FROM ops JOIN generation_rows ON generation_rows.id=ops.id WHERE op='settle' AND target=? AND own=1"
|
|
458
|
-
: "SELECT 1 FROM ops WHERE op = 'settle' AND target = ?").
|
|
565
|
+
: "SELECT 1 FROM ops WHERE op = 'settle' AND target = ?", task).length > 0;
|
|
459
566
|
}
|
|
460
567
|
/** Whether any entry was saved against the task. */
|
|
461
|
-
hasTask(task: string): boolean { return
|
|
568
|
+
hasTask(task: string): boolean { this.ensureFresh(); return this.readRows("SELECT 1 FROM entries WHERE task = ?", task).length > 0; }
|
|
462
569
|
private promoting = false;
|
|
463
570
|
private readonly screen?: PromotionScreen;
|
|
464
571
|
/**
|
|
@@ -546,9 +653,14 @@ export class MemoryStore {
|
|
|
546
653
|
|
|
547
654
|
/** Drop the index and read every segment again: the same rows as the incremental index. */
|
|
548
655
|
rebuild(): void {
|
|
656
|
+
this.searchIndex.reset();
|
|
549
657
|
if (this.generationIndex) { this.generationIndex.rebuild(); return; }
|
|
550
658
|
// One transaction: no reader sees the index empty, and no other process commits an offset in between.
|
|
551
|
-
this.
|
|
659
|
+
this.rebuildingSearch = true;
|
|
660
|
+
try {
|
|
661
|
+
this.db.transaction(() => { for (const t of ["segments", "heads", "entries", "ops", "edges", "goals"]) this.db.run(`DELETE FROM ${t}`); this.refresh(); }).immediate();
|
|
662
|
+
} finally { this.rebuildingSearch = false; }
|
|
663
|
+
this.syncSearchIndex(); // one batched derived write, not one search.db transaction per rebuilt entry
|
|
552
664
|
}
|
|
553
665
|
|
|
554
666
|
private segments(sub: "log" | "local"): string[] {
|
|
@@ -572,9 +684,13 @@ export class MemoryStore {
|
|
|
572
684
|
if (e.op === "note") {
|
|
573
685
|
if (!KINDS.includes(e.kind) || typeof e.text !== "string" || !Array.isArray(e.about) || !str(e.project, 400) || !str(e.peer, 64)
|
|
574
686
|
|| !opt(e.title) || !opt(e.task) || !opt(e.supersedes) || !opt(e.promotedFrom) || !opt(e.expires) || (typeof e.expires === "string" && Number.isNaN(Date.parse(e.expires))) || !opt(e.source) || !["project", "shared"].includes(e.scope as string) || !["short", "long"].includes(e.tier as string)) { this.log(`memory segment ${segment}: a malformed entry was skipped`); return; }
|
|
575
|
-
this.db.query(`INSERT OR IGNORE INTO entries (id, at, machine, project, scope, tier, kind, peer, title, text, task, about, supersedes, promoted_from, source, expires)
|
|
687
|
+
const inserted = this.db.query(`INSERT OR IGNORE INTO entries (id, at, machine, project, scope, tier, kind, peer, title, text, task, about, supersedes, promoted_from, source, expires)
|
|
576
688
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, e.at, e.machine, e.project, e.scope === "shared" ? "shared" : "project", e.tier === "long" ? "long" : "short",
|
|
577
|
-
e.kind, e.peer, e.title ?? null, e.text, e.task ?? null, JSON.stringify(e.about.map(String)), e.supersedes ?? null, e.promotedFrom ?? null, e.source ?? "hub", typeof e.expires === "string" ? e.expires : null);
|
|
689
|
+
e.kind, e.peer, e.title ?? null, e.text, e.task ?? null, JSON.stringify(e.about.map(String)), e.supersedes ?? null, e.promotedFrom ?? null, e.source ?? "hub", typeof e.expires === "string" ? e.expires : null).changes;
|
|
690
|
+
// The FTS copy follows the committed row. `changes` is only a hint here (bun:sqlite misreports it for ignored
|
|
691
|
+
// inserts inside a transaction): addCurated is idempotent by entry id, and a missed hint becomes a count drift
|
|
692
|
+
// the next search resyncs.
|
|
693
|
+
if (inserted && !this.rebuildingSearch) this.searchIndex.addCurated({ id, at: e.at, project: e.project, tier: e.tier === "long" ? "long" : "short", scope: e.scope === "shared" ? "shared" : "project", body: `${e.title ?? ""}\n${e.text}\n${e.about.map(String).join(" ")}` });
|
|
578
694
|
for (const node of e.about) this.db.query("INSERT OR IGNORE INTO edges (src, dst) VALUES (?, ?)").run(id, String(node));
|
|
579
695
|
} else if (e.op === "goal") {
|
|
580
696
|
if (!str(e.title, 400) || !Array.isArray(e.milestones) || !Array.isArray(e.dependsOn) || !str(e.project, 400) || !opt(e.actor) || !opt(e.scope)) { this.log(`memory segment ${segment}: a malformed entry was skipped`); return; }
|
|
@@ -588,6 +704,21 @@ export class MemoryStore {
|
|
|
588
704
|
}
|
|
589
705
|
}
|
|
590
706
|
|
|
707
|
+
/** Only the bound project and user overlay participate; never scan another project's private shard. */
|
|
708
|
+
private readStores(): MemoryStore[] { return this.userOverlay ? [this, this.userOverlay] : [this]; }
|
|
709
|
+
|
|
710
|
+
private readRows(sql: string, ...bind: string[]): Record<string, unknown>[] {
|
|
711
|
+
const rows = this.readStores().flatMap(store => store.db.query(sql).all(...bind) as Record<string, unknown>[]);
|
|
712
|
+
const unique = new Map<string, Record<string, unknown>>();
|
|
713
|
+
for (const row of rows) unique.set(typeof row.id === "string" ? row.id : canonicalJSON(row), row);
|
|
714
|
+
return [...unique.values()].sort((a, b) => String(a.at ?? "").localeCompare(String(b.at ?? "")) || String(a.id ?? "").localeCompare(String(b.id ?? "")));
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
private ownsTarget(id: string): boolean {
|
|
718
|
+
this.ensureFresh();
|
|
719
|
+
return !!this.db.query("SELECT id FROM entries WHERE id=? UNION SELECT id FROM goals WHERE id=? UNION SELECT id FROM ops WHERE id=?").get(id, id, id);
|
|
720
|
+
}
|
|
721
|
+
|
|
591
722
|
/**
|
|
592
723
|
* Every live note with its derived state; an expired one is left out as soon as its deadline passes, before any
|
|
593
724
|
* prune. Lifecycle relations still fold globally; SQL selects full rows before View/JSON materialization.
|
|
@@ -595,19 +726,20 @@ export class MemoryStore {
|
|
|
595
726
|
*/
|
|
596
727
|
private views(expired = false, selection?: Filter, expiredOnly = false): View[] {
|
|
597
728
|
// Metadata, ops and candidate bodies must describe one WAL snapshot, even if another process indexes a copy.
|
|
598
|
-
|
|
729
|
+
const read = () => this.viewRows(expired, selection, expiredOnly);
|
|
730
|
+
return this.db.transaction(() => this.userOverlay ? this.userOverlay.db.transaction(read).deferred() : read()).deferred();
|
|
599
731
|
}
|
|
600
732
|
|
|
601
733
|
private viewRows(expired: boolean, selection: Filter | undefined, expiredOnly: boolean): View[] {
|
|
602
734
|
const now = this.now(), selecting = selection !== undefined || expiredOnly;
|
|
603
|
-
const pruned = this.generationIndex ? new Set((this.
|
|
604
|
-
if (this.generationIndex
|
|
735
|
+
const pruned = this.generationIndex ? new Set((this.readRows("SELECT id FROM generation_targets") as { id: string }[]).map(row => row.id)) : undefined;
|
|
736
|
+
if (this.generationIndex) this.generationIndex.pruned = new Set((this.db.query("SELECT id FROM generation_targets").all() as { id: string }[]).map(row => row.id));
|
|
605
737
|
// Relations may cross projects and promoted copies: fold their small metadata globally, never filtered bodies.
|
|
606
|
-
const rows = this.
|
|
738
|
+
const rows = this.readRows(selecting
|
|
607
739
|
? "SELECT id, at, tier, kind, task, expires, supersedes, promoted_from FROM entries"
|
|
608
|
-
: "SELECT * FROM entries")
|
|
609
|
-
const ops = this.
|
|
610
|
-
const realms = this.generationIndex ? new Map((this.
|
|
740
|
+
: "SELECT * FROM entries") as Record<string, string | null>[];
|
|
741
|
+
const ops = this.readRows("SELECT id, op, at, target, by FROM ops ORDER BY at, id") as { id: string; op: string; at: string; target: string; by: string | null }[];
|
|
742
|
+
const realms = this.generationIndex ? new Map((this.readRows("SELECT id,own,legacy FROM generation_rows") as { id: string; own: number; legacy: number }[]).map(row => [row.id, row])) : undefined;
|
|
611
743
|
const born = realms ? new Set([...realms.values()].filter(row => row.own).map(row => row.id)) : undefined;
|
|
612
744
|
const forgotten = new Set(ops.filter((o) => o.op === "forget").map((o) => o.target));
|
|
613
745
|
// Deadlines were fixed when the entries were written, so every process (whatever its own TTL) prunes the same rows.
|
|
@@ -703,7 +835,7 @@ export class MemoryStore {
|
|
|
703
835
|
}
|
|
704
836
|
// No SQL LIMIT: a host may screen the complete matching set before its own caps or visible counts.
|
|
705
837
|
// Date parsing, Unicode query matching, locale order and slice semantics stay in JavaScript.
|
|
706
|
-
candidates = this.
|
|
838
|
+
candidates = this.readRows(`SELECT * FROM entries${where.length ? ` WHERE ${where.join(" AND ")}` : ""}`, ...bind) as Record<string, string | null>[];
|
|
707
839
|
}
|
|
708
840
|
const out: View[] = [];
|
|
709
841
|
for (const r of candidates) {
|
|
@@ -734,11 +866,13 @@ export class MemoryStore {
|
|
|
734
866
|
for (const v of expired) for (let id: string | undefined = v.id; id && !gone.has(id); id = parent.get(id)) gone.add(id);
|
|
735
867
|
if (!gone.size) return 0;
|
|
736
868
|
this.db.transaction(() => { for (const id of gone) { this.db.query("DELETE FROM entries WHERE id = ?").run(id); this.db.query("DELETE FROM edges WHERE src = ?").run(id); } }).immediate();
|
|
869
|
+
this.searchIndex.markDirty(); // FTS cannot subtract rows incrementally: the next search resyncs it
|
|
737
870
|
return gone.size;
|
|
738
871
|
}
|
|
739
872
|
|
|
740
873
|
/** Index deferred writes before any note, goal or suggestion read. */
|
|
741
874
|
private ensureFresh(): void {
|
|
875
|
+
this.userOverlay?.ensureFresh();
|
|
742
876
|
if (this.generationIndex) { this.generationIndex.refresh(); return; }
|
|
743
877
|
// Another process (a CLI, another project's hub, a sync) may have written since this one last read the segments.
|
|
744
878
|
if (this.stale || this.shape() !== this.seen) try { this.stale = false; this.refresh(); } catch { this.stale = true; /* still busy: the next read tries again */ }
|
|
@@ -750,7 +884,7 @@ export class MemoryStore {
|
|
|
750
884
|
*/
|
|
751
885
|
private live(selection?: Filter): View[] {
|
|
752
886
|
this.ensureFresh();
|
|
753
|
-
this.prune(); this.promoteHeld(); return this.views(false, selection);
|
|
887
|
+
this.prune(); this.userOverlay?.promoteHeld(); this.promoteHeld(); return this.views(false, selection);
|
|
754
888
|
}
|
|
755
889
|
/** A write whose index update failed: the next read refreshes first. */
|
|
756
890
|
private stale = false;
|
|
@@ -764,6 +898,76 @@ export class MemoryStore {
|
|
|
764
898
|
.sort((a, b) => b.at.localeCompare(a.at) || a.id.localeCompare(b.id)).slice(0, f.limit ?? 200);
|
|
765
899
|
}
|
|
766
900
|
|
|
901
|
+
/**
|
|
902
|
+
* #13 R5: natural-language search over the curated tier and this machine's capture tier, ranked by BM25 with
|
|
903
|
+
* tier and recency weights (curated outranks capture; a newer row beats an older one at equal text). Visibility
|
|
904
|
+
* (forgotten, superseded, expired, pruned) is decided by the same live views `list` uses. Without FTS5 the engine
|
|
905
|
+
* falls back to the substring filter and says so; capture recall needs FTS5 and is skipped in the fallback (the
|
|
906
|
+
* R3 cost of the fallback is the pre-#13 behavior: same scan, same memory).
|
|
907
|
+
*/
|
|
908
|
+
search(q: string, f: SearchFilter = {}): SearchResult {
|
|
909
|
+
const limit = f.limit ?? 200;
|
|
910
|
+
if (!this.searchIndex.available) return { engine: "substring", hits: this.list({ ...f, q }).map((view) => ({ source: "curated" as const, score: 0, view })) };
|
|
911
|
+
const visible = new Map(this.live(f).filter((v) => (!f.project || v.project === f.project || v.scope === "shared")
|
|
912
|
+
&& (!f.since || Date.parse(v.at) >= Date.parse(f.since)) && (f.includeSuperseded || !v.supersededBy)).map((v) => [v.id, v]));
|
|
913
|
+
for (const store of this.readStores()) store.syncSearchIndex();
|
|
914
|
+
const now = this.now();
|
|
915
|
+
const recency = (at: string): number => 1 / (1 + Math.max(0, now - Date.parse(at)) / (30 * 24 * 3600_000));
|
|
916
|
+
const hits: SearchHit[] = [];
|
|
917
|
+
// The project filter runs inside the FTS query (its rows plus shared ones), so a busy sibling project can never
|
|
918
|
+
// starve this project's LIMIT.
|
|
919
|
+
// Superseded, forgotten, expired and `since` rows are filtered after FTS, so a fixed candidate cap could be filled
|
|
920
|
+
// by invisible rows and hide a visible match: widen the cap until `limit` visible hits or the index is exhausted.
|
|
921
|
+
// Pages of fixed size, so memory stays one page plus the hits (#13 R3) however many invisible rows rank first.
|
|
922
|
+
// ponytail: with almost nothing visible (a selective `since`, many superseded or forgotten rows) the pages walk the
|
|
923
|
+
// store's whole match set; push `since` into SearchIndex.query (docs carries `at`) if that time shows up.
|
|
924
|
+
const page = Math.max(limit * 4, 32);
|
|
925
|
+
for (const store of visible.size ? this.readStores() : []) {
|
|
926
|
+
let found = 0;
|
|
927
|
+
const seen = new Set<string>(); // another process's sync between pages can shift offsets and repeat a ref
|
|
928
|
+
for (let offset = 0; found < limit; offset += page) {
|
|
929
|
+
const ranked = store.searchIndex.query(q, page, f.project, offset);
|
|
930
|
+
for (const hit of ranked) {
|
|
931
|
+
const view = visible.get(hit.ref);
|
|
932
|
+
if (!view || seen.has(hit.ref)) continue;
|
|
933
|
+
seen.add(hit.ref);
|
|
934
|
+
found++;
|
|
935
|
+
hits.push({ source: "curated", score: -hit.rank * (view.tier === "long" ? 1 : 0.9) * recency(hit.at), view });
|
|
936
|
+
}
|
|
937
|
+
if (ranked.length < page) break;
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
// #15 D4/T1: the capture tier is project-scoped; without a project key there is no shard to recall from.
|
|
941
|
+
if (f.includeCapture !== false && f.project) {
|
|
942
|
+
const shard = shardMachineDir(this.memoryBaseDir, resolveAlias(this.memoryBaseDir, f.project), this.machine.id);
|
|
943
|
+
// The retention cutoff and `since` run in the capture query, so neither expired nor old rows take a candidate slot.
|
|
944
|
+
// An out-of-range retention_days gives no cutoff rather than failing the whole search (eviction keeps its own check).
|
|
945
|
+
const override = f.captureRetentionDays, days = typeof override === "number" && Number.isFinite(override) && override > 0 ? override : readCaptureConfig(shard, this.memoryBaseDir).retentionDays;
|
|
946
|
+
const cutoff = new Date(now - days * 24 * 3600_000);
|
|
947
|
+
const retained = Number.isFinite(cutoff.getTime()) ? cutoff.toISOString() : undefined;
|
|
948
|
+
const since = f.since && Number.isFinite(Date.parse(f.since)) ? new Date(Date.parse(f.since)).toISOString() : undefined;
|
|
949
|
+
for (const { row, rank } of captureSearch(shard, q, Math.max(limit * 4, 32), since && (!retained || since > retained) ? since : retained)) {
|
|
950
|
+
hits.push({ source: "capture", score: -rank * 0.5 * recency(row.at), capture: row });
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
// Where FTS misses entirely (a stopword-only query, a tokenization edge), the substring filter is the safety net.
|
|
954
|
+
if (!hits.length && q.trim()) {
|
|
955
|
+
for (const view of this.list({ ...f, q })) hits.push({ source: "curated", score: 0, view });
|
|
956
|
+
}
|
|
957
|
+
const ref = (h: SearchHit): string => h.view?.id ?? String(h.capture?.id ?? 0).padStart(12, "0");
|
|
958
|
+
hits.sort((a, b) => b.score - a.score || ref(a).localeCompare(ref(b)));
|
|
959
|
+
return { engine: "fts5", hits: hits.slice(0, limit) };
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
/** Bring the derived FTS copy level with the curated rows when a prune, a crash, a lock or a rebuild drifted them apart. */
|
|
963
|
+
private syncSearchIndex(): void {
|
|
964
|
+
if (!this.searchIndex.available) return;
|
|
965
|
+
const curated = (this.db.query("SELECT COUNT(*) AS n FROM entries").get() as { n: number }).n;
|
|
966
|
+
if (!this.searchIndex.needsResync(curated)) return;
|
|
967
|
+
this.searchIndex.resync(() => (this.db.query("SELECT id, at, project, tier, scope, title, text, about FROM entries").all() as { id: string; at: string; project: string; tier: string; scope: string; title: string | null; text: string; about: string }[])
|
|
968
|
+
.map((r) => ({ id: r.id, at: r.at, project: r.project, tier: r.tier, scope: r.scope, body: `${r.title ?? ""}\n${r.text}\n${(JSON.parse(r.about) as string[]).join(" ")}` })));
|
|
969
|
+
}
|
|
970
|
+
|
|
767
971
|
/** One entry by id or unique id prefix; with `project`, only one that project may read (its own or shared). */
|
|
768
972
|
get(id: string, project?: string): View | undefined {
|
|
769
973
|
id = id.replace(/^mem:/, ""); // briefs and recall print `mem:<id>`
|
|
@@ -774,8 +978,8 @@ export class MemoryStore {
|
|
|
774
978
|
if (found) return found;
|
|
775
979
|
// A short-term id an agent kept after its task was approved names the promoted copy now, here or on another machine
|
|
776
980
|
// that has only the synced copy (which names its source).
|
|
777
|
-
let at = pick(this.
|
|
778
|
-
for (let copy = at; copy; copy = (this.
|
|
981
|
+
let at = pick(this.readRows("SELECT id FROM entries UNION SELECT promoted_from AS id FROM entries WHERE promoted_from IS NOT NULL") as { id: string }[])?.id;
|
|
982
|
+
for (let copy = at; copy; copy = (this.readRows("SELECT id FROM entries WHERE promoted_from = ?", copy)[0] as { id: string } | undefined)?.id) { if (this.generationIndex?.pruned.has(copy)) return undefined; at = copy; }
|
|
779
983
|
return at ? all.find((v) => v.id === at) : undefined;
|
|
780
984
|
}
|
|
781
985
|
|
|
@@ -793,7 +997,7 @@ export class MemoryStore {
|
|
|
793
997
|
const near = new Map<string, 1 | 2>();
|
|
794
998
|
const about = (ns: Iterable<Node>) => {
|
|
795
999
|
const ids = new Set<string>();
|
|
796
|
-
for (const n of ns) for (const r of this.
|
|
1000
|
+
for (const n of ns) for (const r of this.readRows("SELECT src FROM edges WHERE dst = ?", n) as { src: string }[]) if (live.has(r.src)) ids.add(r.src);
|
|
797
1001
|
return ids;
|
|
798
1002
|
};
|
|
799
1003
|
const first = about(nodes);
|
|
@@ -893,8 +1097,8 @@ export class MemoryStore {
|
|
|
893
1097
|
goals(project?: string): GoalView[] {
|
|
894
1098
|
this.ensureFresh();
|
|
895
1099
|
const now = this.now();
|
|
896
|
-
const rows = this.
|
|
897
|
-
const ops = this.
|
|
1100
|
+
const rows = this.readRows("SELECT * FROM goals") as { id: string; at: string; machine: string; project: string; scope: string; title: string; milestones: string; depends: string }[];
|
|
1101
|
+
const ops = this.readRows("SELECT id, op, at, machine, target, by FROM ops WHERE op IN ('goal_state', 'goal_link', 'lease', 'lease_force') ORDER BY at, id") as { op: string; at: string; machine: string; target: string; by: string | null }[];
|
|
898
1102
|
return rows.filter((r) => !project || r.project === project || r.scope === "shared").map((r): GoalView => {
|
|
899
1103
|
const mine = ops.filter((o) => o.target === r.id);
|
|
900
1104
|
const states = mine.filter((o) => o.op === "goal_state" && (GOAL_STATES as readonly string[]).includes(o.by ?? ""));
|
|
@@ -943,7 +1147,7 @@ export class MemoryStore {
|
|
|
943
1147
|
suggestions(project?: string): Suggestion[] {
|
|
944
1148
|
this.ensureFresh();
|
|
945
1149
|
const visible = project ? new Set(this.list({ project, limit: Number.MAX_SAFE_INTEGER, includeSuperseded: true }).map((v) => v.id)) : undefined;
|
|
946
|
-
const ops = this.
|
|
1150
|
+
const ops = this.readRows("SELECT id, op, at, target, by FROM ops WHERE op IN ('suggest', 'confirm', 'reject') ORDER BY at, id") as { id: string; op: string; at: string; target: string; by: string | null }[];
|
|
947
1151
|
const verdict = new Map<string, "confirmed" | "rejected">();
|
|
948
1152
|
for (const o of ops) if (o.op !== "suggest") verdict.set(o.target, o.op === "confirm" ? "confirmed" : "rejected");
|
|
949
1153
|
return ops.filter((o) => o.op === "suggest" && o.by).flatMap((o) => {
|
|
@@ -980,6 +1184,40 @@ export function buildGenerationIndex(profile: BoundProfile, sources: readonly Le
|
|
|
980
1184
|
finally { store.close(); }
|
|
981
1185
|
}
|
|
982
1186
|
|
|
1187
|
+
function initializeShard(home: string, opts: { project?: string; user?: boolean }): void {
|
|
1188
|
+
const selection = readShardedSelection(home), generation = selection.generation;
|
|
1189
|
+
const dir = opts.user === true ? userShardDir(home) : shardDir(home, opts.project!), machine = machineId(home);
|
|
1190
|
+
if (opts.project !== undefined) ensureShardDescriptor(home, opts.project);
|
|
1191
|
+
for (const folder of ["log", "local", "legacy/log", "legacy/local", "legacy-main/log", "legacy-main/local"]) mkdirSync(join(dir, folder), { recursive: true, mode: 0o700 });
|
|
1192
|
+
for (const [path, value] of [["generation.json", generationDescriptor(generation)], ["machine.json", machine], ["legacy/machine.json", machine], ["legacy-main/machine.json", machine]] as const) {
|
|
1193
|
+
const file = join(dir, path), prior = readProfileJSON(file);
|
|
1194
|
+
if (prior === undefined) writeProfileJSON(file, value);
|
|
1195
|
+
else if (canonicalJSON(prior) !== canonicalJSON(value)) throw new Error("curated shard metadata changed during initialization");
|
|
1196
|
+
}
|
|
1197
|
+
const legacyHome = join(dir, "legacy"), sources = readMigrationSources(legacyHome);
|
|
1198
|
+
writeProfileJSON(join(dir, "migration-sources.json"), { schema: "agent-mem.migration-sources/v1", sources });
|
|
1199
|
+
const profile: BoundProfile = { kind: "generation", home, dir, generation, selectorRevision: selection.selectorRevision,
|
|
1200
|
+
legacyHome, legacyMainHome: join(dir, "legacy-main"), sharded: true, ...(opts.user === true ? { shardUser: true } : { shardKey: opts.project! }) };
|
|
1201
|
+
buildGenerationIndex(profile, sources);
|
|
1202
|
+
writeProfileJSON(join(dir, "generation-seal.json"), { schema: "agent-mem.generation-seal/v1", generation,
|
|
1203
|
+
descriptorHash: hashJSON(generationDescriptor(generation)), sourceDigest: hashJSON({ sources, legacyCommit: undefined }) });
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
/** Open one sealed project or user shard directly; no legacy-home scan, catalog walk or old-layout reader lease. */
|
|
1207
|
+
export function openShardStore(opts: StoreOptions & { project?: string; user?: boolean; memoryDir?: string }): MemoryStore {
|
|
1208
|
+
if (opts.user === true && opts.project !== undefined || opts.user !== true && !opts.project) throw new Error("choose exactly one project or the user shard");
|
|
1209
|
+
if (opts.memoryDir !== undefined && (opts.home !== undefined || typeof opts.memoryDir !== "string" || !opts.memoryDir.trim())) throw new Error("choose a memory home or one explicit memory directory");
|
|
1210
|
+
const home = opts.memoryDir ?? (opts.home === undefined ? memoryHome() : join(opts.home, "memory"));
|
|
1211
|
+
assertCuratedShardPaths(home, opts);
|
|
1212
|
+
const dir = opts.user === true ? userShardDir(home) : shardDir(home, opts.project!);
|
|
1213
|
+
const missing = !existsSync(join(dir, "generation.json"));
|
|
1214
|
+
if (missing) initializeShard(home, opts);
|
|
1215
|
+
const profile = resolveShardProfile(home, opts);
|
|
1216
|
+
const userOverlay = opts.user === true ? undefined : openShardStore({ ...opts, project: undefined, user: true });
|
|
1217
|
+
try { return new MemoryStore(opts, { token: MIGRATION_BOOTSTRAP, profile, authoring: true, userOverlay }); }
|
|
1218
|
+
catch (error) { userOverlay?.close(); throw error; }
|
|
1219
|
+
}
|
|
1220
|
+
|
|
983
1221
|
/** Person-only migration receipt; this inspector reads raw rows without opening a store or writing any index. */
|
|
984
1222
|
export function summarizeRawExpiry(rawLines: string[], now: number, tailBytes: number): import("./migrate.ts").MigrationCounts {
|
|
985
1223
|
const bytes = Buffer.from(rawLines.filter(Boolean).join("\n") + (rawLines.length ? "\n" : ""));
|