@staix/agent-mem 0.8.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +5 -3
- package/package.json +1 -1
- package/src/cli.ts +15 -7
- package/src/commands-table.ts +2 -2
- package/src/memory/capture.ts +1 -1
- package/src/memory/commands.ts +2 -2
- package/src/memory/import.ts +91 -0
- package/src/memory/platform.ts +5 -1
- package/src/memory/policy.ts +29 -9
- package/src/memory/shard-sync-format.ts +3 -4
- package/src/memory/shard-sync.ts +135 -17
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Curated memory with append-only JSONL logs, a derived SQLite index, short- and long-term tiers, a typed graph, goals with leases, and git sync between machines. Extracted from [agent-hub](https://github.com/STAIxBWLB/agent-hub), with compatible storage and entry IDs.
|
|
4
4
|
|
|
5
|
-
Version: 0.
|
|
5
|
+
Version: 0.9.1. Requires Bun 1.4.2 or later; no runtime dependencies.
|
|
6
6
|
|
|
7
7
|
## Quick start
|
|
8
8
|
|
|
@@ -284,6 +284,8 @@ Successful native project identities have a private, bounded derived cache (128
|
|
|
284
284
|
|
|
285
285
|
Intake runs at the end of each Codex, Kimi and Pi session once `amem setup <cli> --apply` has registered it. Sync is off unless you ask for it: amem registers no sync hook and runs nothing between sessions. To sync on your own schedule, call `amem sync --if-due` from your shell (for example an EXIT trap in `~/.zshrc`; it needs a terminal). It syncs at most hourly, backs off 15 minutes doubling to a day after failures, stops after 30 seconds, and does nothing without a remote set by `amem sync --remote <private git url>`. `--force` skips the schedule and the backoff.
|
|
286
286
|
|
|
287
|
+
Split-home sync durably holds verified foreign-generation prunes with a "held, not applied" warning while unrelated sync continues; targets can remain visible, and full cross-generation removal remains a separate ceiling (see [the sync contract](docs/contracts.md#per-shard-private-sync-15-t3)).
|
|
288
|
+
|
|
287
289
|
## Native transcript intake
|
|
288
290
|
|
|
289
291
|
`amem ingest` reads existing Codex, Kimi or Pi transcripts and writes screened deterministic capture rows, never curated notes. Run it from the project to bind intake to that working directory; the CLI exposes no project/home override or policy bypass:
|
|
@@ -367,7 +369,7 @@ Real migration remains blocked until the separately reviewed generation-capable
|
|
|
367
369
|
|
|
368
370
|
## Curated shard migration (#15 T2)
|
|
369
371
|
|
|
370
|
-
`amem migrate --shards` moves the curated single home into `projects/<key-hash>/` and `user/` through a resumable generation journal. The original single-home log bytes are never rewritten. A populated selected generation is supported and keeps its generation ID. Each shard keeps private legacy inputs in read-only `legacy/`, public-main inputs in read-only `legacy-main/`, and generation-born rows in its own `log/` and `local/`. It has its own `index.db`, `search.db` and mutation lock; original IDs, hashes, placement and own/local/public provenance are preserved. The source generation bytes stay unchanged. Immutable prune proofs are validated before effects; a multi-shard prune is translated into one deterministic row per shard, with a receipt naming the original row. Existing capture and `work/` directories are not migration inputs and are left untouched. `catalog.db` remains derived and rebuildable, now including curated entry counts and last use. Split curated stores sync the opened canonical project and user overlay to separate private-remote refs (`amem/p/<key-hash>` and `amem/u`) with `amem sync --policy-free`; unopened projects and machine data are not fetched. Original IDs and attribution survive independent generations.
|
|
372
|
+
`amem migrate --shards` moves the curated single home into `projects/<key-hash>/` and `user/` through a resumable generation journal. The original single-home log bytes are never rewritten. A populated selected generation is supported and keeps its generation ID. Each shard keeps private legacy inputs in read-only `legacy/`, public-main inputs in read-only `legacy-main/`, and generation-born rows in its own `log/` and `local/`. It has its own `index.db`, `search.db` and mutation lock; original IDs, hashes, placement and own/local/public provenance are preserved. The source generation bytes stay unchanged. Immutable prune proofs are validated before effects; a multi-shard prune is translated into one deterministic row per shard, with a receipt naming the original row. Existing capture and `work/` directories are not migration inputs and are left untouched. `catalog.db` remains derived and rebuildable, now including curated entry counts and last use. Split curated stores sync the opened canonical project and user overlay to separate private-remote refs (`amem/p/<key-hash>` and `amem/u`) with `amem sync --policy-free`; unopened projects and machine data are not fetched. Original IDs and attribution survive independent generations. See `docs/contracts.md`, "Per-shard private sync", for prune transport and application boundaries.
|
|
371
373
|
|
|
372
374
|
Before splitting a live home, every installed host must open it through `openShardStore({ project })` or the explicit user shard. Plain `new MemoryStore()` refuses any split marker, including an interrupted split, so it cannot silently keep writing the old layout. No host capability registry currently exists; migration preflight output states that limitation. Do not release T2 for agent-hub use until its memory factory is integrated with `openShardStore` and that host version is installed. Existing unannounced older builds remain the documented unchanged-source ceiling.
|
|
373
375
|
|
|
@@ -413,7 +415,7 @@ The frozen nine-tool contract for dotfiles#255 and maru#458:
|
|
|
413
415
|
|
|
414
416
|
Search/show/timeline label capture as recall with `evidence: false`; capture never enters the curated graph. Policy screens complete rows before selection, limits or graph construction. The standalone secret screen, private flags, PII patterns and deny paths compose with a host's stricter policy through `createMemoryMcp({cwd, base, policy})`. Screening errors fail closed. `base` is the exact memory root; historical library `home` options still name its parent. Scope and profile changes refuse further access instead of silently reopening another store. Graph reads retain the 5,000-row curated ceiling; work-list reads stream the complete ordered history without an operation-count ceiling, applying policy before latest-value reduction. The working state scales with the current result, not historical payloads. The graph ceiling is an operational bound, not an exposed count of unscreened data. Graphs cover curated-note relationships, not goal/suggestion tables.
|
|
415
417
|
|
|
416
|
-
`WorkListStore` writes `projects/<key-hash>/work/` with its own append log, derived index and lock. Scalar fields reduce by latest value, null clears, list status `done` closes, and item status is `todo|doing|done|dropped`. Work lists are outside generation selection and capture eviction. `amem import claude-mem` additionally imports `work_state_entries` only through persisted person-reviewed D2 aliases, preserving source DB bytes and stable replay receipts; JSON keeps observation fields and adds `work` counts/cursor.
|
|
418
|
+
`WorkListStore` writes `projects/<key-hash>/work/` with its own append log, derived index and lock. Scalar fields reduce by latest value, null clears, list status `done` closes, and item status is `todo|doing|done|dropped`. Work lists are outside generation selection and capture eviction. `amem import claude-mem` additionally imports `work_state_entries` only through persisted person-reviewed D2 aliases, preserving source DB bytes and stable replay receipts; JSON keeps observation fields and adds `work` counts/cursor. `--capture-days N` also imports observations and session summaries from the last N days into machine-local capture under the same aliases, with read/added/spooled/skipped counts, stable replay receipts and the configured capture cap and retention. Widening N after an earlier import requires `--restart` to revisit rows below the saved cursor. Chroma vectors are never read.
|
|
417
419
|
|
|
418
420
|
`amem vault-candidates --json` and the MCP read tool select durable long-term notes under current policy, excluding task/review/PR/CI progress and already-linked entries. The deterministic status filter covers English status phrases and can miss paraphrased or other-language status text; candidates require person review. No vault is opened, indexed or written. Library `recordVaultLink` records an exact-entry hash and relative Markdown path in a generation-only local append op; it survives index rebuild and shard migration, stays machine-local, and does not change shared-v1 schemas or sync semantics. There is no MCP link writer.
|
|
419
421
|
|
package/package.json
CHANGED
package/src/cli.ts
CHANGED
|
@@ -11,7 +11,7 @@ import { serveMcp, writeConfirmed } from "./mcp.ts";
|
|
|
11
11
|
import { createStandalonePolicy } from "./memory/policy.ts";
|
|
12
12
|
import { configuredPolicy, configuredPolicyOptions, loadAmemConfig, repoConfigRoot } from "./memory/config.ts";
|
|
13
13
|
import { vaultCandidates } from "./memory/vault.ts";
|
|
14
|
-
import { importClaudeMemWorkState } from "./memory/import.ts";
|
|
14
|
+
import { importClaudeMemCapture, importClaudeMemWorkState } from "./memory/import.ts";
|
|
15
15
|
import { closeSync, constants, existsSync, fstatSync, lstatSync, openSync, readFileSync, readSync, rmSync, writeFileSync } from "node:fs";
|
|
16
16
|
import { join } from "node:path";
|
|
17
17
|
import { MemoryStore, looksSecret, machineId, openShardStore, type StoreOptions } from "./memory/store.ts";
|
|
@@ -549,7 +549,7 @@ export async function runCli(argv: readonly string[], context: CliContext = {}):
|
|
|
549
549
|
return options["--json"] ? JSON.stringify(result, null, 2) : `${result.dryRun ? "preview (nothing evicted; add --confirm):\n" : ""}${gcText(result)}`;
|
|
550
550
|
}
|
|
551
551
|
if (sub === "import" && ["claude-auto", "codex"].includes(parsed.path[1] ?? "")) {
|
|
552
|
-
for (const flag of ["--db", "--limit", "--restart"]) if (options[flag] !== undefined) throw new Error(`${flag} only applies to import claude-mem`);
|
|
552
|
+
for (const flag of ["--db", "--limit", "--restart", "--capture-days"]) if (options[flag] !== undefined) throw new Error(`${flag} only applies to import claude-mem`);
|
|
553
553
|
const source = parsed.path[1] as ExternalMemorySource, policy = policyOf();
|
|
554
554
|
const result = await importExternalMemory({ base: memoryHome(), project: boundProject(), source, cwd: policyCwd, env: context.env ?? process.env,
|
|
555
555
|
...(typeof options["--from"] === "string" ? { from: options["--from"] } : {}), policy: value => policy.screenStandalone(value) });
|
|
@@ -724,7 +724,14 @@ export async function runCli(argv: readonly string[], context: CliContext = {}):
|
|
|
724
724
|
}
|
|
725
725
|
// Only a real attempt is stamped: a local refusal (lock held, no remote, policy, usage) is not, and a future stamp is no history (src/memory/sync-cadence.ts).
|
|
726
726
|
const stampSync = (error: string | undefined, cause?: StopCause) => { if (syncOutcome(error, cause) !== "local") recordSyncAttempt(syncHome, error === undefined, Date.now()); };
|
|
727
|
-
const
|
|
727
|
+
const captureImport = sub === "import" && parsed.path[1] === "claude-mem", captureDays = options["--capture-days"] === undefined ? undefined : Number(options["--capture-days"]);
|
|
728
|
+
if (captureImport && captureDays !== undefined && (!Number.isSafeInteger(captureDays) || captureDays <= 0)) throw new Error("--capture-days takes a positive integer");
|
|
729
|
+
const captureArgs = new Set<number>();
|
|
730
|
+
if (captureImport) for (let index = 0; index < args.length; index++) {
|
|
731
|
+
if (args[index] === "--capture-days") { captureArgs.add(index); captureArgs.add(index + 1); }
|
|
732
|
+
else if (args[index]?.startsWith("--capture-days=")) captureArgs.add(index);
|
|
733
|
+
}
|
|
734
|
+
const syncArgs = args.filter((arg, index) => arg !== "--if-due" && arg !== "--force" && !captureArgs.has(index));
|
|
728
735
|
let output: string;
|
|
729
736
|
const stop = ifDue ? stopSignal(context.syncDeadlineMs ?? SYNC_IF_DUE_DEADLINE_MS, process) : undefined;
|
|
730
737
|
try {
|
|
@@ -747,10 +754,11 @@ export async function runCli(argv: readonly string[], context: CliContext = {}):
|
|
|
747
754
|
if (options["--json"]) { try { const result = JSON.parse(output) as { ok?: unknown; error?: unknown }; if (result.ok === false) failure = typeof result.error === "string" ? result.error : "sync failed"; } catch { /* not a sync result */ } }
|
|
748
755
|
stampSync(failure, stop?.cause());
|
|
749
756
|
}
|
|
750
|
-
if (
|
|
751
|
-
const policy = configured;
|
|
752
|
-
const work = await importClaudeMemWorkState({ base: memoryHome(), project: boundProject(), policy: row => policy.screen(row), ...
|
|
753
|
-
|
|
757
|
+
if (captureImport) {
|
|
758
|
+
const policy = configured, db = options["--db"] ? { db: String(options["--db"]) } : {}, limit = options["--limit"] ? { limit: Number(options["--limit"]) } : {}, restart = options["--restart"] === true;
|
|
759
|
+
const work = await importClaudeMemWorkState({ base: memoryHome(), project: boundProject(), policy: row => policy.screen(row), ...db, ...limit, restart });
|
|
760
|
+
const capture = captureDays === undefined ? undefined : await importClaudeMemCapture({ base: memoryHome(), project: boundProject(), cwd, days: captureDays, ...db, ...limit, restart, policy: { host: policy } });
|
|
761
|
+
return options["--json"] ? JSON.stringify({ ...JSON.parse(output), work, ...(capture ? { capture } : {}) }, null, 2) : `${output}\nwork lists: ${JSON.stringify(work)}${capture ? `\ncapture: ${JSON.stringify(capture)}` : ""}`;
|
|
754
762
|
}
|
|
755
763
|
return !options["--json"] && receipt.bridgeStatus
|
|
756
764
|
? `${output}\n(legacy malformed lines skipped: ${receipt.bridgeStatus.skippedRows})` : output;
|
package/src/commands-table.ts
CHANGED
|
@@ -76,9 +76,9 @@ export const COMMANDS: Record<string, CommandDefinition> = {
|
|
|
76
76
|
sync: { description: "Sync a private remote (--if-due: only when the schedule and the failure backoff allow it; --force ignores both)", options: { ...json, ...policy, "--remote": text("Private git URL"),
|
|
77
77
|
"--if-due": flag("Sync only when due (hourly after a success, 15 minutes doubling to a day after failures); stops after 30 seconds"), "--force": flag("With --if-due: ignore the schedule and the backoff") }, mutates: true, policyFree: true },
|
|
78
78
|
machine: { description: "Show or label this machine", options: { ...json, "--label": text("Machine label") }, mutates: "label" },
|
|
79
|
-
import: { description: "Import external memory", options: { ...json, ...policy, "--db": { description: "SQLite database file", value: "path" }, "--limit": text("Maximum observations"), "--restart": flag("Restart import cursor") }, mutates: true, policyFree: true,
|
|
79
|
+
import: { description: "Import external memory", options: { ...json, ...policy, "--db": { description: "SQLite database file", value: "path" }, "--limit": text("Maximum observations"), "--restart": flag("Restart import cursor"), "--capture-days": text("Import recent claude-mem observations and summaries into capture") }, mutates: true, policyFree: true,
|
|
80
80
|
commands: {
|
|
81
|
-
"claude-mem": { description: "Import claude-mem observations", options: { "--db": { description: "SQLite database file", value: "path" }, "--limit": text("Maximum observations"), "--restart": flag("Restart import cursor") } },
|
|
81
|
+
"claude-mem": { description: "Import claude-mem observations", options: { "--db": { description: "SQLite database file", value: "path" }, "--limit": text("Maximum observations"), "--restart": flag("Restart import cursor"), "--capture-days": text("Import recent observations and summaries into capture") } },
|
|
82
82
|
"claude-auto": { description: "Import Claude Code auto memory as review candidates; CLAUDE.md and AGENTS.md stay excluded", options: { "--from": { description: "Claude projects directory (default ~/.claude/projects)", value: "path" } } },
|
|
83
83
|
codex: { description: "Import Codex memories as review candidates; originals stay unchanged", options: { "--from": { description: "Codex memories directory (default ~/.codex/memories)", value: "path" } } },
|
|
84
84
|
} },
|
package/src/memory/capture.ts
CHANGED
|
@@ -359,7 +359,7 @@ export class CaptureStore {
|
|
|
359
359
|
this.db.run("CREATE TRIGGER IF NOT EXISTS rows_ad AFTER DELETE ON rows BEGIN INSERT INTO fts (fts, rowid, body) VALUES ('delete', old.id, old.excerpt); END");
|
|
360
360
|
this.db.run("CREATE TRIGGER IF NOT EXISTS rows_au AFTER UPDATE ON rows BEGIN INSERT INTO fts (fts, rowid, body) VALUES ('delete', old.id, old.excerpt); INSERT INTO fts (rowid, body) VALUES (new.id, new.excerpt); END");
|
|
361
361
|
// Rows written before FTS existed (or before a platform gained it) are indexed once; triggers keep it level.
|
|
362
|
-
if (!had && (this.db.query("SELECT COUNT(*) AS n FROM rows").get() as { n: number }).n > 0) this.db.run("INSERT INTO fts (
|
|
362
|
+
if (!had && (this.db.query("SELECT COUNT(*) AS n FROM rows").get() as { n: number }).n > 0) this.db.run("INSERT INTO fts (rowid, body) SELECT id, excerpt FROM rows");
|
|
363
363
|
}
|
|
364
364
|
|
|
365
365
|
private mcpReady = false;
|
package/src/memory/commands.ts
CHANGED
|
@@ -47,7 +47,7 @@ export interface GoalCommandOptions {
|
|
|
47
47
|
* supersede and rebuild are a person's (the identity gate refuses agent shells).
|
|
48
48
|
*/
|
|
49
49
|
/** The usage line of the memory commands under `name` (`amem` by default; agent-hub passes `ahub memory`). */
|
|
50
|
-
export const memoryUsage = (name: string = AMEM_NAMES.memory): string => `usage: ${name} [list] [--tier short|long] [--kind <kind>] [--peer <p>] [--machine <id>] [--since <iso>] [--limit N] | search <words> [--history] [--as-of <UTC>] | show <id> [--history] [--as-of <UTC>] | graph <id|node> [--hops 1|2] [--history] [--as-of <UTC>] | promote|demote|forget <id> | supersede <old> <new> | rebuild | sync [--remote <url>] | machine [--label <name>] | import claude-mem [--db <path>] [--limit N] [--restart] | dedupe [--apply] | suggestions | confirm|reject <suggestion id> [--json]`;
|
|
50
|
+
export const memoryUsage = (name: string = AMEM_NAMES.memory): string => `usage: ${name} [list] [--tier short|long] [--kind <kind>] [--peer <p>] [--machine <id>] [--since <iso>] [--limit N] | search <words> [--history] [--as-of <UTC>] | show <id> [--history] [--as-of <UTC>] | graph <id|node> [--hops 1|2] [--history] [--as-of <UTC>] | promote|demote|forget <id> | supersede <old> <new> | rebuild | sync [--remote <url>] | machine [--label <name>] | import claude-mem [--db <path>] [--limit N] [--restart] [--capture-days N] | dedupe [--apply] | suggestions | confirm|reject <suggestion id> [--json]`;
|
|
51
51
|
export const MEMORY_USAGE = memoryUsage();
|
|
52
52
|
/** The usage line of the goal commands under `name` (`amem goal` by default; agent-hub passes `ahub goal`). */
|
|
53
53
|
export const goalUsage = (name: string = AMEM_NAMES.goal): string => `usage: ${name} [list] [--all] | show <id> | new <title> [--milestone <text>]... [--depends <id>]... [--shared] | link <id> <task id> | state <id> open|active|blocked|done|dropped | done <id> | take <id> [--force] [--json]`;
|
|
@@ -154,7 +154,7 @@ export async function rawMemoryCommand(args: string[], cwd: string, opts: Memory
|
|
|
154
154
|
const db = flag("--db"), limit = flag("--limit");
|
|
155
155
|
const restart = rest.includes("--restart");
|
|
156
156
|
if (restart) rest.splice(rest.indexOf("--restart"), 1);
|
|
157
|
-
if (rest[0] !== "claude-mem" || rest.length !== 1) fail(`usage: ${names.memory} import claude-mem [--db <path>] [--limit N] [--restart]`);
|
|
157
|
+
if (rest[0] !== "claude-mem" || rest.length !== 1) fail(`usage: ${names.memory} import claude-mem [--db <path>] [--limit N] [--restart] [--capture-days N]`);
|
|
158
158
|
if (limit && !(Number(limit) > 0)) fail("--limit takes a positive number");
|
|
159
159
|
const isPii = opts.importPolicy();
|
|
160
160
|
const key = opts.project ?? memoryProjectKey(cwd);
|
package/src/memory/import.ts
CHANGED
|
@@ -7,6 +7,11 @@ import { join } from "node:path";
|
|
|
7
7
|
import { KINDS, looksSecret, type Kind, type MemoryStore } from "./store.ts";
|
|
8
8
|
import { aliasedKeys, resolveAlias } from "./shard.ts";
|
|
9
9
|
import { WorkListStore, type WorkListFields, type WorkListPolicy } from "./work-lists.ts";
|
|
10
|
+
import { CaptureStore, type CaptureInput } from "./capture.ts";
|
|
11
|
+
import { CAPTURE_MAX_EXCERPT } from "./capture-input.ts";
|
|
12
|
+
import { configuredPolicyOptions } from "./config.ts";
|
|
13
|
+
import { createStandalonePolicy, type StandalonePolicyOptions } from "./policy.ts";
|
|
14
|
+
import { stripPrivateSpans } from "./private-spans.ts";
|
|
10
15
|
|
|
11
16
|
/** The claude-mem store is another program's: opened read-only, never written, its locks waited on briefly. */
|
|
12
17
|
export const CLAUDE_MEM_BUSY_TIMEOUT_MS = 3000;
|
|
@@ -15,6 +20,92 @@ const MODEL_ANSWER = "ahub ask (model answer)";
|
|
|
15
20
|
|
|
16
21
|
export interface ImportResult { read: number; added: number; skipped: { model: number; pii: number; secret: number }; more: boolean; cursor: number }
|
|
17
22
|
export interface WorkStateImportResult { read: number; added: number; skipped: { invalid: number; policy: number }; more: boolean; cursor: number }
|
|
23
|
+
export interface CaptureImportResult { read: number; added: number; spooled: number; skipped: number; more: boolean }
|
|
24
|
+
|
|
25
|
+
type ClaudeMemCaptureRow = { id: number; created_at_epoch: number; origin_device_id: string | null; origin_local_id: string | null } & Record<string, any>;
|
|
26
|
+
const CLAUDE_MEM_CAPTURE_KINDS = [
|
|
27
|
+
{ table: "observations", event: "observation", columns: "id, title, text, narrative, created_at_epoch" },
|
|
28
|
+
{ table: "session_summaries", event: "summary", columns: "id, request, investigated, learned, completed, next_steps, notes, created_at_epoch" },
|
|
29
|
+
] as const;
|
|
30
|
+
type ClaudeMemCaptureKind = typeof CLAUDE_MEM_CAPTURE_KINDS[number];
|
|
31
|
+
interface ClaudeMemCaptureContext {
|
|
32
|
+
db: Database; store: CaptureStore; policy: ReturnType<typeof createStandalonePolicy>;
|
|
33
|
+
aliases: string[]; project: string; sourceMachine: string; cutoff: number; limit: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function claudeMemCaptureText(row: ClaudeMemCaptureRow, kind: ClaudeMemCaptureKind): string {
|
|
37
|
+
if (kind.table === "observations") return stripPrivateSpans([row.title, row.text ?? row.narrative].filter(value => typeof value === "string" && value.trim()).join("\n")).trim();
|
|
38
|
+
const fields = [["Request", row.request], ["Investigated", row.investigated], ["Learned", row.learned], ["Completed", row.completed], ["Next steps", row.next_steps], ["Notes", row.notes]];
|
|
39
|
+
return stripPrivateSpans(fields.filter(([, value]) => typeof value === "string" && value.trim()).map(([label, value]) => `${label}: ${value}`).join("\n")).trim();
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async function appendClaudeMemCapture(row: ClaudeMemCaptureRow, kind: ClaudeMemCaptureKind, context: ClaudeMemCaptureContext): Promise<"added" | "spooled" | "skipped"> {
|
|
43
|
+
const text = claudeMemCaptureText(row, kind);
|
|
44
|
+
if (!text || !Number.isFinite(row.created_at_epoch)) return "skipped";
|
|
45
|
+
let at: string;
|
|
46
|
+
try { at = new Date(row.created_at_epoch).toISOString(); } catch { return "skipped"; }
|
|
47
|
+
const input: CaptureInput = { project: context.project, agent: "claude-mem", event: kind.event, excerpt: text, at };
|
|
48
|
+
if (!await context.policy.screen(input)) return "skipped";
|
|
49
|
+
input.excerpt = text.slice(0, CAPTURE_MAX_EXCERPT);
|
|
50
|
+
const synced = typeof row.origin_device_id === "string" && !!row.origin_device_id && typeof row.origin_local_id === "string" && !!row.origin_local_id;
|
|
51
|
+
const receipt = createHash("sha256").update(JSON.stringify(["claude-mem", kind.table, synced ? row.origin_device_id : context.sourceMachine, synced ? row.origin_local_id : String(row.id)])).digest("hex");
|
|
52
|
+
const appended = context.store.append(input, { receipt });
|
|
53
|
+
return "spooled" in appended ? "spooled" : appended.already ? "skipped" : "added";
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function importClaudeMemCaptureTable(kind: ClaudeMemCaptureKind, context: ClaudeMemCaptureContext, initial: number, onCursor: (cursor: number) => void): Promise<CaptureImportResult> {
|
|
57
|
+
const result: CaptureImportResult = { read: 0, added: 0, spooled: 0, skipped: 0, more: false };
|
|
58
|
+
const columns = new Set((context.db.query(`PRAGMA table_info(${kind.table})`).all() as { name: string }[]).map(row => row.name));
|
|
59
|
+
const origin = columns.has("origin_device_id") && columns.has("origin_local_id"), where = `project IN (${context.aliases.map(() => "?").join(",")})`;
|
|
60
|
+
const max = context.db.query(`SELECT MAX(id) AS id FROM ${kind.table} WHERE ${where}`).get(...context.aliases) as { id: number | null };
|
|
61
|
+
let cursor = (max.id ?? 0) < initial ? 0 : initial; onCursor(cursor);
|
|
62
|
+
const rows = context.db.query(`SELECT ${kind.columns}${origin ? ", origin_device_id, origin_local_id" : ", NULL AS origin_device_id, NULL AS origin_local_id"} FROM ${kind.table} WHERE ${where} AND created_at_epoch >= ? AND id > ? ORDER BY id LIMIT ?`)
|
|
63
|
+
.all(...context.aliases, context.cutoff, cursor, context.limit) as ClaudeMemCaptureRow[];
|
|
64
|
+
result.read = rows.length; result.more = rows.length === context.limit;
|
|
65
|
+
for (const row of rows) {
|
|
66
|
+
result[await appendClaudeMemCapture(row, kind, context)]++;
|
|
67
|
+
cursor = row.id; onCursor(cursor);
|
|
68
|
+
}
|
|
69
|
+
return result;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Import only reviewed aliases into machine-local capture. ponytail: --limit is per source table; a merged cursor could add one combined ceiling. */
|
|
73
|
+
export async function importClaudeMemCapture(opts: { base: string; project: string; cwd: string; days: number; db?: string; limit?: number; restart?: boolean; now?: number; policy?: StandalonePolicyOptions; log?: (line: string) => void }): Promise<CaptureImportResult> {
|
|
74
|
+
const file = opts.db ?? join(homedir(), ".claude-mem", "claude-mem.db"), limit = opts.limit ?? 10_000;
|
|
75
|
+
if (!existsSync(file)) throw new Error(`no claude-mem store at ${file}`);
|
|
76
|
+
if (!Number.isSafeInteger(opts.days) || opts.days <= 0) throw new Error("--capture-days takes a positive integer");
|
|
77
|
+
if (!Number.isSafeInteger(limit) || limit <= 0) throw new Error("--limit takes a positive integer");
|
|
78
|
+
const now = opts.now ?? Date.now(); if (!Number.isFinite(now)) throw new Error("capture import clock unavailable");
|
|
79
|
+
const project = resolveAlias(opts.base, opts.project), aliases = aliasedKeys(opts.base, project);
|
|
80
|
+
const result: CaptureImportResult = { read: 0, added: 0, spooled: 0, skipped: 0, more: false };
|
|
81
|
+
if (!aliases.length) return result;
|
|
82
|
+
const db = new Database(file, { readonly: true });
|
|
83
|
+
let store: CaptureStore | undefined, state: string | undefined;
|
|
84
|
+
const progress = new Map<string, number>();
|
|
85
|
+
try {
|
|
86
|
+
db.run(`PRAGMA busy_timeout = ${CLAUDE_MEM_BUSY_TIMEOUT_MS}`);
|
|
87
|
+
const available = CLAUDE_MEM_CAPTURE_KINDS.filter(kind => db.query("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(kind.table));
|
|
88
|
+
if (!available.length) return result;
|
|
89
|
+
store = new CaptureStore({ memoryDir: opts.base, project, cwd: opts.cwd, now: () => now });
|
|
90
|
+
state = join(store.dir, "import-state.json");
|
|
91
|
+
const context: ClaudeMemCaptureContext = { db, store, aliases, project, limit,
|
|
92
|
+
sourceMachine: createHash("sha256").update(realPath(file)).digest("hex"), cutoff: Math.max(0, now - opts.days * 86_400_000),
|
|
93
|
+
policy: createStandalonePolicy(configuredPolicyOptions(opts.base, opts.cwd, opts.policy)) };
|
|
94
|
+
const cursors = readCursors(state);
|
|
95
|
+
for (const kind of available) {
|
|
96
|
+
const key = createHash("sha256").update(JSON.stringify([context.sourceMachine, aliases, project, kind.table])).digest("hex");
|
|
97
|
+
progress.set(key, opts.restart ? 0 : cursors[key] ?? 0);
|
|
98
|
+
const imported = await importClaudeMemCaptureTable(kind, context, progress.get(key)!, cursor => progress.set(key, cursor));
|
|
99
|
+
for (const field of ["read", "added", "spooled", "skipped"] as const) result[field] += imported[field];
|
|
100
|
+
result.more ||= imported.more;
|
|
101
|
+
}
|
|
102
|
+
} finally {
|
|
103
|
+
db.close();
|
|
104
|
+
if (state) for (const [key, cursor] of progress) writeCursor(state, key, cursor, opts.log ?? console.error);
|
|
105
|
+
store?.close();
|
|
106
|
+
}
|
|
107
|
+
return result;
|
|
108
|
+
}
|
|
18
109
|
|
|
19
110
|
/**
|
|
20
111
|
* Import only the source project names explicitly mapped by the person's D2 alias table. A basename or even
|
package/src/memory/platform.ts
CHANGED
|
@@ -49,13 +49,17 @@ export function realPath(path: string): string {
|
|
|
49
49
|
export const DENY_SEGMENTS = [".maru/secrets", ".agenthub/state", ".agenthub/archive"];
|
|
50
50
|
/** Regex sources, matched against a file's basename. */
|
|
51
51
|
export const DENY_NAMES = ["^\\.env", "\\.pem$", "\\.key$", "^id_rsa", "^id_ed25519", "^credentials(\\.|$)", "^auth\\.json$", "^\\.netrc$", "^\\.npmrc$", "^\\.pypirc$"];
|
|
52
|
+
const denyNamePatterns = new Map(DENY_NAMES.map(source => [source, new RegExp(source)]));
|
|
52
53
|
|
|
53
54
|
const hasSegment = (rel: string, seg: string) => `/${rel}/`.includes(`/${seg}/`);
|
|
54
55
|
|
|
55
56
|
/** True when a path names something the worker must never read, write or report. `extra`: substrings from `local.deny`. */
|
|
56
57
|
export function isDenied(path: string, extra: string[] = []): boolean {
|
|
57
58
|
const norm = path.split(sep).join("/");
|
|
58
|
-
|
|
59
|
+
if (DENY_SEGMENTS.some((s) => hasSegment(norm, s))) return true;
|
|
60
|
+
const name = basename(norm);
|
|
61
|
+
// Reuse the fixed patterns without caching arbitrary additions to the exported list.
|
|
62
|
+
return DENY_NAMES.some((re) => (denyNamePatterns.get(re) ?? new RegExp(re)).test(name)) || extra.some((e) => e && norm.includes(e));
|
|
59
63
|
}
|
|
60
64
|
|
|
61
65
|
/**
|
package/src/memory/policy.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { posix } from "node:path";
|
|
2
2
|
import { types } from "node:util";
|
|
3
3
|
import { looksSecret } from "./screen.ts";
|
|
4
|
-
import { isDenied } from "./platform.ts";
|
|
4
|
+
import { DENY_NAMES, DENY_SEGMENTS, isDenied } from "./platform.ts";
|
|
5
5
|
import { nodePath, pathCandidates } from "./project.ts";
|
|
6
6
|
|
|
7
7
|
/** True means the complete value is public. Hosts may narrow, never replace, standalone policy. */
|
|
@@ -115,18 +115,38 @@ export function createStandalonePolicy(options: StandalonePolicyOptions = {}): S
|
|
|
115
115
|
const pii = literals(descriptors.piiPatterns?.value, "PII patterns");
|
|
116
116
|
const host = descriptors.host?.value as MemoryPolicy | undefined;
|
|
117
117
|
if (host !== undefined && (!host || typeof host.screen !== "function")) throw new Error("invalid host policy");
|
|
118
|
+
// ponytail: retain at most 512 exact strings / 64 KiB per immutable policy configuration.
|
|
119
|
+
// Evict the batch when full; a measured need for better hit rates can upgrade this to an LRU.
|
|
120
|
+
const textChecks = new Map<string, boolean>(); let textBytes = 0;
|
|
121
|
+
let defaultDenyRevision = "";
|
|
122
|
+
const publicText = (text: string): boolean => {
|
|
123
|
+
const cached = textChecks.get(text);
|
|
124
|
+
if (cached !== undefined) return cached;
|
|
125
|
+
const check = () => {
|
|
126
|
+
if (looksSecret(text) || looksSecret(text.replace(/["']/g, ""))) return false;
|
|
127
|
+
const normalized = fold(text);
|
|
128
|
+
if (defaultPii(normalized) || pii.some(pattern => normalized.includes(pattern))) return false;
|
|
129
|
+
const candidates = normalized.includes("/") || /\.[a-z]/.test(normalized) ? pathCandidates(normalized) : [];
|
|
130
|
+
const paths = [normalized, nodePath(normalized), ...candidates, ...normalized.split(/[\s"'`(){}\[\],;:=<>]+/)];
|
|
131
|
+
return !paths.some(path => path && isDenied(posix.normalize(path), deny));
|
|
132
|
+
};
|
|
133
|
+
const allowed = check(), bytes = Buffer.byteLength(text);
|
|
134
|
+
if (bytes <= 1024) {
|
|
135
|
+
if (textChecks.size >= 512 || textBytes + bytes > 64 * 1024) { textChecks.clear(); textBytes = 0; }
|
|
136
|
+
textChecks.set(text, allowed); textBytes += bytes;
|
|
137
|
+
}
|
|
138
|
+
return allowed;
|
|
139
|
+
};
|
|
118
140
|
const standalone = (value: unknown): { ok: boolean; value?: unknown } => {
|
|
119
141
|
try {
|
|
142
|
+
// The exported default lists can be tightened by a host between calls.
|
|
143
|
+
const revision = JSON.stringify([DENY_NAMES, DENY_SEGMENTS]);
|
|
144
|
+
if (revision !== defaultDenyRevision) { textChecks.clear(); textBytes = 0; defaultDenyRevision = revision; }
|
|
120
145
|
const complete = snapshot(value);
|
|
121
146
|
if (!complete) return { ok: false };
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
if (defaultPii(normalized) || pii.some(pattern => normalized.includes(pattern))) return { ok: false };
|
|
126
|
-
const candidates = normalized.includes("/") || /\.[a-z]/.test(normalized) ? pathCandidates(normalized) : [];
|
|
127
|
-
const paths = [normalized, nodePath(normalized), ...candidates, ...normalized.split(/[\s"'`(){}\[\],;:=<>]+/)];
|
|
128
|
-
for (const path of paths) if (path && isDenied(posix.normalize(path), deny)) return { ok: false };
|
|
129
|
-
}
|
|
147
|
+
// Only pure text checks are reused. Shape/bounds/private flags and the host's detached
|
|
148
|
+
// snapshot are fresh on every call; no row or host verdict is cached.
|
|
149
|
+
for (const text of complete.texts) if (!publicText(text)) return { ok: false };
|
|
130
150
|
return { ok: true, value: complete.value };
|
|
131
151
|
} catch { return { ok: false }; }
|
|
132
152
|
};
|
|
@@ -102,10 +102,9 @@ export function decodeShardPack(path: string, bytes: Uint8Array, realm: ShardSyn
|
|
|
102
102
|
catch { throw new ShardSyncFormatError(label, "malformed pack row"); }
|
|
103
103
|
const row = publicRow(value, label);
|
|
104
104
|
if (hash(row.machine) !== match[1]) throw new ShardSyncFormatError(label, "pack author mismatch");
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
} else if (row.op === "note" || row.op === "goal") {
|
|
105
|
+
// Generation is preserved in the original prune payload. Acceptance here is structural,
|
|
106
|
+
// never permission to apply it: the engine proves each original generation and holds foreign rows.
|
|
107
|
+
if (row.op === "note" || row.op === "goal") {
|
|
109
108
|
if (realm.scope === "project" ? row.scope !== "project" || row.project !== realm.key : row.scope !== "shared") {
|
|
110
109
|
throw new ShardSyncFormatError(label, "row scope mismatch");
|
|
111
110
|
}
|
package/src/memory/shard-sync.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { randomBytes, createHash } from "node:crypto";
|
|
2
|
-
import { constants, closeSync, existsSync, fstatSync, fsyncSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, readSync, rmSync, writeSync } from "node:fs";
|
|
2
|
+
import { constants, closeSync, existsSync, fstatSync, fsyncSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, readSync, renameSync, rmSync, writeSync } from "node:fs";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { canonicalJSON, fsyncDirectory, hashJSON, readProfileJSON } from "./generation.ts";
|
|
5
5
|
import { generationSyncLock } from "./generation-sync.ts";
|
|
@@ -14,8 +14,14 @@ import type { Entry, MemoryStore } from "./store.ts";
|
|
|
14
14
|
const OID = /^[a-f0-9]{40}$/, PACK = /^rows\/[a-f0-9]{64}\/[a-f0-9]{64}\.jsonl$/;
|
|
15
15
|
const digest = (bytes: string | Uint8Array): string => createHash("sha256").update(bytes).digest("hex");
|
|
16
16
|
const zero = "0".repeat(40);
|
|
17
|
+
// The descriptor is at most 64 KiB and an original prune at most 16 KiB; keep receipt parsing bounded.
|
|
18
|
+
const HOLD_MAX_BYTES = 128 * 1024;
|
|
17
19
|
type PendingPrune = { plan: PrunePlan; row: PruneRow; line: string; offset: number; prefixHash: string };
|
|
18
|
-
type
|
|
20
|
+
type HoldSource = { ref: string; commit: string; pack: string; blob: string };
|
|
21
|
+
type HoldReceipt = { schema: "agent-mem.prune-hold/v1"; remote: string; realm: ShardSyncRealm; receivingGeneration: string; row: PruneRow; payloadHash: string; source: HoldSource };
|
|
22
|
+
type Hold = { file: string; bytes: Buffer };
|
|
23
|
+
type SyncPhase = "published" | "hold-written" | "hold-installed" | "imported";
|
|
24
|
+
type Target = { store: MemoryStore; realm: ShardSyncRealm; rows: Map<string, ShardSyncRow>; fetched: Set<string>; sources: PruneSourceRow[]; pending: PendingPrune[]; origins: Map<string, HoldSource>; holds: Hold[]; generation: string; local?: string; remote?: string; commit?: string };
|
|
19
25
|
class SyncRefusal extends Error {}
|
|
20
26
|
function refuse(reason: string): never { throw new SyncRefusal(reason); }
|
|
21
27
|
function directory(path: string): void {
|
|
@@ -146,8 +152,10 @@ function provePrunes(targets: readonly Target[]): void {
|
|
|
146
152
|
}
|
|
147
153
|
const snapshot: PruneSnapshot = { rows: [...sources.values()], expiries: [] };
|
|
148
154
|
const prunes = [...target.rows.values()].filter((row): row is PruneRow => row.op === "prune");
|
|
149
|
-
const
|
|
150
|
-
|
|
155
|
+
for (const generation of new Set(prunes.map(row => row.generation))) {
|
|
156
|
+
const proof = reducePruneRows(prunes.filter(row => row.generation === generation).map(row => ({ row, placement: "synced" as const })), snapshot, generation);
|
|
157
|
+
if (proof.invalid || proof.pending.length) refuse(`${shardSyncLabel(target.realm)}: prune lacks complete public witnesses`);
|
|
158
|
+
}
|
|
151
159
|
}
|
|
152
160
|
}
|
|
153
161
|
function installPack(store: MemoryStore, bytes: Buffer): void {
|
|
@@ -176,7 +184,7 @@ function importRows(target: Target): void {
|
|
|
176
184
|
for (const pending of target.pending) {
|
|
177
185
|
port.persistPrunePlan(pending.plan); port.appendPrune(pending.row, pending.line, pending.plan.seal); resumed.add(pending.row.id);
|
|
178
186
|
}
|
|
179
|
-
for (const row of target.rows.values()) if (row.op === "prune") {
|
|
187
|
+
for (const row of target.rows.values()) if (row.op === "prune" && row.generation === target.generation) {
|
|
180
188
|
if (resumed.has(row.id)) continue;
|
|
181
189
|
// A transport imports an already-authored person row without changing its author or generation.
|
|
182
190
|
// Native durable-plan append confirms fsync even for an already parsed id after a crash.
|
|
@@ -186,8 +194,104 @@ function importRows(target: Target): void {
|
|
|
186
194
|
port.refresh();
|
|
187
195
|
}
|
|
188
196
|
|
|
189
|
-
/**
|
|
190
|
-
|
|
197
|
+
/** Receipts are private, immutable transport metadata, outside every native generation reducer.
|
|
198
|
+
* ponytail: foreign targets remain visible; an explicit cross-generation apply protocol must
|
|
199
|
+
* re-prove witnesses and establish a compatible reader floor before lifting this hold ceiling. */
|
|
200
|
+
const hasHoldPath = (file: string): boolean => lstatSync(file, { throwIfNoEntry: false }) !== undefined;
|
|
201
|
+
function privateHoldPath(file: string, folder: boolean): void {
|
|
202
|
+
if (folder) directory(file); else regular(file);
|
|
203
|
+
if (lstatSync(file).mode & 0o077) refuse("shard sync prune holds require private permissions");
|
|
204
|
+
}
|
|
205
|
+
function checkPendingHold(file: string, bytes: Buffer): void {
|
|
206
|
+
privateHoldPath(file, false);
|
|
207
|
+
if (lstatSync(file).size > bytes.length || !readFileSync(file).equals(bytes.subarray(0, lstatSync(file).size))) refuse("shard sync partial prune hold conflicts with verified receipt");
|
|
208
|
+
}
|
|
209
|
+
function installHold(root: string, hold: Hold, phase?: (phase: SyncPhase) => void): void {
|
|
210
|
+
const parts = hold.file.slice(root.length + 1).split("/"), name = parts.pop()!;
|
|
211
|
+
let folder = root;
|
|
212
|
+
for (const part of parts) {
|
|
213
|
+
const parent = folder; folder = join(folder, part);
|
|
214
|
+
if (!hasHoldPath(folder)) mkdirSync(folder, { mode: 0o700 });
|
|
215
|
+
privateHoldPath(folder, true); fsyncDirectory(parent);
|
|
216
|
+
}
|
|
217
|
+
const file = join(folder, name), pending = file + ".pending";
|
|
218
|
+
if (hasHoldPath(file)) {
|
|
219
|
+
privateHoldPath(file, false);
|
|
220
|
+
if (!readFileSync(file).equals(hold.bytes)) refuse("shard sync immutable prune hold conflict");
|
|
221
|
+
const fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW); try { fsyncSync(fd); } finally { closeSync(fd); }
|
|
222
|
+
fsyncDirectory(folder); return;
|
|
223
|
+
}
|
|
224
|
+
if (hasHoldPath(pending)) checkPendingHold(pending, hold.bytes);
|
|
225
|
+
const fd = openSync(pending, constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW, 0o600);
|
|
226
|
+
try {
|
|
227
|
+
// Interrupted private staging bytes may be completed; committed receipts are never rewritten.
|
|
228
|
+
let offset = 0; while (offset < hold.bytes.length) { const n = writeSync(fd, hold.bytes, offset, hold.bytes.length - offset, offset); if (n <= 0) refuse("shard sync prune hold write unavailable"); offset += n; }
|
|
229
|
+
fsyncSync(fd);
|
|
230
|
+
} finally { closeSync(fd); }
|
|
231
|
+
phase?.("hold-written");
|
|
232
|
+
// The transport mutation lock serializes this create-once rename. Unlike hard-link installation,
|
|
233
|
+
// a crash cannot leave a committed receipt with a second link that later fails validation.
|
|
234
|
+
if (hasHoldPath(file)) refuse("shard sync prune hold changed concurrently");
|
|
235
|
+
renameSync(pending, file); fsyncDirectory(folder); phase?.("hold-installed");
|
|
236
|
+
}
|
|
237
|
+
function holdWarning(targets: readonly Target[], root: string): string | undefined {
|
|
238
|
+
const warnings: string[] = [];
|
|
239
|
+
for (const target of targets) if (target.holds.length) {
|
|
240
|
+
const rows = [...target.rows.values()].filter((row): row is PruneRow => row.op === "prune" && row.generation !== target.generation);
|
|
241
|
+
const ids = [...new Set(rows.flatMap(row => row.targets.map(item => item.id)))].sort(), shown = ids.slice(0, 10);
|
|
242
|
+
const location = target.holds[0]!.file.slice(root.length + 1).split("/").slice(0, -2).join("/");
|
|
243
|
+
warnings.push(`${shardSyncLabel(target.realm)}: ${rows.length} foreign-generation prune(s) held, not applied; targets ${shown.join(",")}${ids.length > shown.length ? ` (+${ids.length - shown.length} more)` : ""} may remain visible; receipts shard-sync/${location}; bring this machine to the origin generation, or prune the entry here as the person`);
|
|
244
|
+
}
|
|
245
|
+
return warnings.length ? warnings.join("; ") : undefined;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
type ProveHoldSource = (target: Target, receipt: HoldReceipt) => Promise<void>;
|
|
249
|
+
async function prepareHold(root: string, url: string, target: Target, row: PruneRow, proveSource: ProveHoldSource): Promise<Hold> {
|
|
250
|
+
const source = target.origins.get(row.id);
|
|
251
|
+
if (!source || !target.remote || !target.fetched.has(row.id)) refuse(`${shardSyncLabel(target.realm)}: foreign prune has no fetched provenance`);
|
|
252
|
+
const receipt: HoldReceipt = { schema: "agent-mem.prune-hold/v1", remote: digest(url), realm: target.realm,
|
|
253
|
+
receivingGeneration: target.generation, row, payloadHash: hashJSON(row), source };
|
|
254
|
+
const folders = ["prune-holds", receipt.remote, digest(shardSyncRef(target.realm)), target.generation, row.generation];
|
|
255
|
+
let folder = root;
|
|
256
|
+
for (const name of folders) { folder = join(folder, name); if (hasHoldPath(folder)) privateHoldPath(folder, true); }
|
|
257
|
+
const file = join(folder, `${row.id}-${receipt.payloadHash}.json`);
|
|
258
|
+
let selected = receipt;
|
|
259
|
+
if (hasHoldPath(file)) {
|
|
260
|
+
privateHoldPath(file, false);
|
|
261
|
+
const value = readProfileJSON(file, HOLD_MAX_BYTES) as HoldReceipt | undefined;
|
|
262
|
+
if (!value || !value.source || Object.keys(value.source).length !== 4 || !OID.test(value.source.commit)
|
|
263
|
+
|| !OID.test(value.source.blob) || !PACK.test(value.source.pack)
|
|
264
|
+
|| canonicalJSON({ ...value, source }) !== canonicalJSON(receipt)) refuse(`${shardSyncLabel(target.realm)}: invalid prune hold receipt`);
|
|
265
|
+
// A receipt is evidence of a hold, never application authority. Recheck its original
|
|
266
|
+
// immutable source against this invocation's fully audited selected remote history.
|
|
267
|
+
await proveSource(target, value);
|
|
268
|
+
selected = value;
|
|
269
|
+
}
|
|
270
|
+
const bytes = Buffer.from(canonicalJSON(selected) + "\n");
|
|
271
|
+
if (bytes.length > HOLD_MAX_BYTES) refuse("shard sync prune hold exceeds metadata bound");
|
|
272
|
+
if (hasHoldPath(file) && !readFileSync(file).equals(bytes)) refuse("shard sync prune hold is not canonical");
|
|
273
|
+
if (hasHoldPath(file + ".pending")) checkPendingHold(file + ".pending", bytes);
|
|
274
|
+
return { file, bytes };
|
|
275
|
+
}
|
|
276
|
+
async function prepareHolds(targets: readonly Target[], root: string, url: string, proveSource: ProveHoldSource): Promise<void> {
|
|
277
|
+
for (const target of targets) {
|
|
278
|
+
target.holds = [];
|
|
279
|
+
for (const row of target.rows.values()) if (row.op === "prune" && row.generation !== target.generation) {
|
|
280
|
+
target.holds.push(await prepareHold(root, url, target, row, proveSource));
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
function mergePackOrigins(into: Map<string, HoldSource>, byRealm: Map<string, Map<string, HoldSource>>, rows: readonly ShardSyncRow[], source: HoldSource): void {
|
|
285
|
+
const first = byRealm.get(source.ref) ?? new Map<string, HoldSource>(); byRealm.set(source.ref, first);
|
|
286
|
+
// Oldest-first history keeps a receipt's source stable across tip advancement and partial-write retry.
|
|
287
|
+
for (const row of rows) {
|
|
288
|
+
if (!first.has(row.id)) first.set(row.id, source);
|
|
289
|
+
if (!into.has(row.id)) into.set(row.id, first.get(row.id)!);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** Sync only borrowed canonical project/user targets. Internal error/phase observers are test-only, never CLI/SyncOptions ports. */
|
|
294
|
+
export async function syncShardedMemory(store: MemoryStore, opts: SyncOptions = {}, testOnlyUnexpectedError?: (error: unknown) => void, testOnlyPhase?: (phase: SyncPhase) => void): Promise<SyncResult> {
|
|
191
295
|
let pushed = false, releaseTransport: (() => void) | undefined, releaseSync: (() => void) | undefined;
|
|
192
296
|
const releases: (() => void)[] = [], transient: string[] = [], temporary: string[] = [];
|
|
193
297
|
const fail = (error: string): SyncResult => ({ ok: false, pushed, entries: 0, error });
|
|
@@ -216,7 +320,7 @@ export async function syncShardedMemory(store: MemoryStore, opts: SyncOptions =
|
|
|
216
320
|
targets = borrowed.map(opened => {
|
|
217
321
|
const profile = opened.profile;
|
|
218
322
|
if (!profile.sharded || profile.kind !== "generation" || !profile.generation || !profile.shardUser && !profile.shardKey) refuse("shard sync target identity unavailable");
|
|
219
|
-
return { store: opened, realm: profile.shardUser ? { scope: "user" as const } : { scope: "project" as const, key: profile.shardKey! }, rows: new Map<string, ShardSyncRow>(), fetched: new Set<string>(), sources: [], pending: [], generation: profile.generation };
|
|
323
|
+
return { store: opened, realm: profile.shardUser ? { scope: "user" as const } : { scope: "project" as const, key: profile.shardKey! }, rows: new Map<string, ShardSyncRow>(), fetched: new Set<string>(), sources: [], pending: [], origins: new Map<string, HoldSource>(), holds: [], generation: profile.generation };
|
|
220
324
|
});
|
|
221
325
|
if (!targets.length || targets.length > 2 || targets.some(target => target.store.profile.home !== store.profile.home)
|
|
222
326
|
|| new Set(targets.map(target => shardSyncRef(target.realm))).size !== targets.length
|
|
@@ -280,12 +384,13 @@ export async function syncShardedMemory(store: MemoryStore, opts: SyncOptions =
|
|
|
280
384
|
if (identity !== object) refuse("shard sync public blob identity mismatch");
|
|
281
385
|
blobCache.set(object, bytes); return bytes;
|
|
282
386
|
};
|
|
283
|
-
type History = { rows: Map<string, ShardSyncRow>; files: Map<string, Buffer> };
|
|
387
|
+
type History = { rows: Map<string, ShardSyncRow>; files: Map<string, Buffer>; origins: Map<string, HoldSource> };
|
|
284
388
|
// Keep only requested tips. Per-commit full row maps would make append-only history quadratic in memory.
|
|
285
389
|
const histories = new Map<string, History>(), audited = new Set<string>();
|
|
390
|
+
const firstOrigins = new Map<string, Map<string, HoldSource>>();
|
|
286
391
|
const treeState = async (commit: string, target: Target): Promise<History> => {
|
|
287
392
|
const entries = (await required(["ls-tree", "-r", "-z", commit], "shard sync tree unavailable")).split("\0").filter(Boolean);
|
|
288
|
-
const rows = new Map<string, ShardSyncRow>(), files = new Map<string, Buffer>();
|
|
393
|
+
const rows = new Map<string, ShardSyncRow>(), files = new Map<string, Buffer>(), origins = new Map<string, HoldSource>();
|
|
289
394
|
for (const entry of entries) {
|
|
290
395
|
const match = /^100644 blob ([a-f0-9]{40})\t(.+)$/.exec(entry);
|
|
291
396
|
if (!match || match[2] !== "shard.json" && !PACK.test(match[2]!)) refuse(`${shardSyncLabel(target.realm)}: forbidden historical tree entry`);
|
|
@@ -298,10 +403,11 @@ export async function syncShardedMemory(store: MemoryStore, opts: SyncOptions =
|
|
|
298
403
|
if (!pack) { pack = decodeShardPack(path, bytes, target.realm, target.generation); decoded.set(key, pack); }
|
|
299
404
|
else if (!path.endsWith(`/${digest(bytes)}.jsonl`) || pack.some(row => path !== `rows/${digest(row.machine)}/${digest(bytes)}.jsonl`)) refuse(`${shardSyncLabel(target.realm)}: immutable pack path changed`);
|
|
300
405
|
mergeRows(rows, pack, target.realm);
|
|
406
|
+
mergePackOrigins(origins, firstOrigins, pack, { ref: shardSyncRef(target.realm), commit, pack: path, blob: object });
|
|
301
407
|
}
|
|
302
408
|
}
|
|
303
409
|
if (!files.has("shard.json")) refuse(`${shardSyncLabel(target.realm)}: historical descriptor missing`);
|
|
304
|
-
return { rows, files };
|
|
410
|
+
return { rows, files, origins };
|
|
305
411
|
};
|
|
306
412
|
const history = async (tip: string, target: Target): Promise<History> => {
|
|
307
413
|
const key = `${shardSyncRef(target.realm)}:${tip}`, cached = histories.get(key); if (cached) return cached;
|
|
@@ -345,7 +451,7 @@ export async function syncShardedMemory(store: MemoryStore, opts: SyncOptions =
|
|
|
345
451
|
if (target.local) await required(["merge-base", "--is-ancestor", target.local, tip], `${shardSyncLabel(target.realm)}: remote history is not fast-forward`);
|
|
346
452
|
const remote = await history(tip, target); target.remote = tip;
|
|
347
453
|
mergeRows(target.rows, Array.from(remote.rows.values()), target.realm);
|
|
348
|
-
for (const row of remote.rows.values()) target.fetched.add(row.id);
|
|
454
|
+
for (const row of remote.rows.values()) { target.fetched.add(row.id); target.origins.set(row.id, remote.origins.get(row.id)!); }
|
|
349
455
|
}
|
|
350
456
|
};
|
|
351
457
|
const localTargets = () => {
|
|
@@ -385,10 +491,17 @@ export async function syncShardedMemory(store: MemoryStore, opts: SyncOptions =
|
|
|
385
491
|
await history(target.commit, target);
|
|
386
492
|
}
|
|
387
493
|
};
|
|
388
|
-
|
|
494
|
+
const proveHoldSource: ProveHoldSource = async (target, value) => {
|
|
495
|
+
await required(["merge-base", "--is-ancestor", value.source.commit, target.remote!], "shard sync prune hold source is not remote history");
|
|
496
|
+
const original = await history(value.source.commit, target), bytes = original.files.get(value.source.pack);
|
|
497
|
+
if (!bytes || createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex") !== value.source.blob
|
|
498
|
+
|| !decodeShardPack(value.source.pack, bytes, target.realm, target.generation).some(item => canonicalJSON(item) === canonicalJSON(value.row))
|
|
499
|
+
|| value.source.ref !== shardSyncRef(target.realm)) refuse(`${shardSyncLabel(target.realm)}: prune hold source mismatch`);
|
|
500
|
+
};
|
|
501
|
+
localTargets(); await fetchTargets(); localTargets(); provePrunes(targets); await prepareHolds(targets, root, url, proveHoldSource); check();
|
|
389
502
|
// Every selected realm, including its complete remote history/prunes, is preflighted before the first append or publication.
|
|
390
503
|
for (let attempt = 0; attempt < 2; attempt++) {
|
|
391
|
-
if (attempt) { await fetchTargets(); localTargets(); provePrunes(targets); check(); }
|
|
504
|
+
if (attempt) { await fetchTargets(); localTargets(); provePrunes(targets); await prepareHolds(targets, root, url, proveHoldSource); check(); }
|
|
392
505
|
await buildTargets(); check();
|
|
393
506
|
const sent = await git(["push", "--atomic", "--no-follow-tags", "-q", "origin", ...targets.map(target => `${target.commit!}:${shardSyncRef(target.realm)}`)]);
|
|
394
507
|
if (sent.code === 0) pushed = true;
|
|
@@ -396,13 +509,18 @@ export async function syncShardedMemory(store: MemoryStore, opts: SyncOptions =
|
|
|
396
509
|
if (sent.code === 0) {
|
|
397
510
|
pushed = true;
|
|
398
511
|
// No local append before a successful atomic publication: a raced foreign prune can still refuse the entire retry safely.
|
|
399
|
-
localTargets(); provePrunes(targets); check();
|
|
512
|
+
localTargets(); provePrunes(targets); check(); testOnlyPhase?.("published");
|
|
513
|
+
// No selected ordinary/native imports can precede durable receipts for every foreign prune.
|
|
514
|
+
for (const target of targets) for (const hold of target.holds) installHold(root, hold, testOnlyPhase);
|
|
515
|
+
check();
|
|
400
516
|
for (const target of targets) {
|
|
401
517
|
assertCuratedShardPaths(target.store.profile.home, target.realm.scope === "user" ? { user: true } : { project: target.realm.key });
|
|
402
|
-
importRows(target);
|
|
518
|
+
importRows(target); testOnlyPhase?.("imported");
|
|
403
519
|
}
|
|
404
520
|
for (const target of targets) await required(["update-ref", "--no-deref", shardSyncRef(target.realm), target.commit!, target.local ?? zero], "shard sync cached ref changed concurrently");
|
|
405
|
-
|
|
521
|
+
const warning = holdWarning(targets, root);
|
|
522
|
+
return { ok: true, pushed, entries: targets.reduce((n, target) => n + [...target.rows.values()].filter(row => row.op === "note").length, 0),
|
|
523
|
+
...(warning ? { warning } : {}) };
|
|
406
524
|
}
|
|
407
525
|
if (opts.signal?.aborted) return fail("stopped");
|
|
408
526
|
}
|