@staix/agent-mem 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/.claude-plugin/marketplace.json +13 -0
  2. package/.claude-plugin/plugin.json +12 -2
  3. package/README.md +143 -18
  4. package/package.json +1 -1
  5. package/src/cli.ts +537 -39
  6. package/src/commands-table.ts +60 -16
  7. package/src/hook-spool.ts +24 -2
  8. package/src/hooks.ts +9 -5
  9. package/src/index.ts +20 -0
  10. package/src/ingest.ts +3 -2
  11. package/src/mcp-memory.ts +168 -33
  12. package/src/mcp.ts +43 -7
  13. package/src/memory/aliases.ts +4 -0
  14. package/src/memory/bridge.ts +8 -2
  15. package/src/memory/candidate-pending.ts +119 -0
  16. package/src/memory/candidates.ts +228 -47
  17. package/src/memory/capacity-split.ts +76 -0
  18. package/src/memory/capacity.ts +139 -11
  19. package/src/memory/capture-input.ts +8 -3
  20. package/src/memory/capture.ts +30 -5
  21. package/src/memory/catalog.ts +43 -10
  22. package/src/memory/commands.ts +98 -44
  23. package/src/memory/config.ts +167 -0
  24. package/src/memory/consolidate.ts +78 -0
  25. package/src/memory/consolidation-clusters.ts +56 -0
  26. package/src/memory/consolidation-model.ts +97 -0
  27. package/src/memory/cross-project.ts +75 -0
  28. package/src/memory/curated-read.ts +29 -0
  29. package/src/memory/decay-review.ts +136 -0
  30. package/src/memory/doctor.ts +415 -0
  31. package/src/memory/editor.ts +58 -0
  32. package/src/memory/eval-fixture.ts +145 -0
  33. package/src/memory/eval-set-ko.json +90 -0
  34. package/src/memory/eval-set.json +443 -0
  35. package/src/memory/eval.ts +152 -0
  36. package/src/memory/evidence.ts +76 -0
  37. package/src/memory/external-import.ts +267 -0
  38. package/src/memory/gc.ts +97 -0
  39. package/src/memory/generation-index.ts +22 -17
  40. package/src/memory/generation-sync.ts +3 -0
  41. package/src/memory/goal-promotion.ts +71 -0
  42. package/src/memory/home.ts +15 -4
  43. package/src/memory/intent-brief.ts +89 -0
  44. package/src/memory/lifecycle-log.ts +54 -0
  45. package/src/memory/lifecycle.ts +61 -0
  46. package/src/memory/metrics.ts +2 -2
  47. package/src/memory/note-add.ts +100 -0
  48. package/src/memory/path-glob.ts +94 -0
  49. package/src/memory/plan-guard.ts +47 -0
  50. package/src/memory/platform.ts +12 -1
  51. package/src/memory/policy.ts +44 -13
  52. package/src/memory/private-spans.ts +32 -0
  53. package/src/memory/project-export.ts +157 -0
  54. package/src/memory/project-import.ts +147 -0
  55. package/src/memory/recall-staleness.ts +56 -0
  56. package/src/memory/recall.ts +74 -0
  57. package/src/memory/repo-scope.ts +404 -0
  58. package/src/memory/shard-sync-format.ts +116 -0
  59. package/src/memory/shard-sync.ts +421 -0
  60. package/src/memory/store.ts +344 -35
  61. package/src/memory/sync-cadence.ts +149 -0
  62. package/src/memory/sync-screen.ts +86 -0
  63. package/src/memory/sync.ts +91 -11
  64. package/src/memory/usage.ts +35 -0
  65. package/src/memory/use-telemetry.ts +51 -0
  66. package/src/memory/work-lists.ts +10 -4
  67. package/src/session-brief-worker.ts +3 -2
  68. package/src/session-brief.ts +74 -15
  69. package/src/setup-owned.ts +120 -0
  70. package/src/setup-toml.ts +296 -0
  71. package/src/setup.ts +327 -0
@@ -0,0 +1,94 @@
1
+ /**
2
+ * `pii.paths` matching, ported from agent-hub (`src/hub/path-policy.ts`: `policyPaths`, `patternScope`, `matchesPolicyLexically`; `src/hub/memory-policy.ts`:
3
+ * `mentionsPrivatePath`) so the repo scope screens with the semantics the project's own config was written for: exact paths and subtrees, `*`, `?` and
4
+ * whole-segment `**` globs, case and Unicode folded like a case-insensitive disk. amem imports no agent-hub code, so this is a port, and
5
+ * `test/path-policy-parity.test.ts` pins it to values produced by the hub's real matcher (`scripts/gen-path-policy-parity.ts`).
6
+ *
7
+ * Lexical only. The hub's `matchesPolicy` also resolves real paths and symlink aliases against the filesystem; a note merely names a path, so nothing here
8
+ * reads the disk. The port keeps the hub's `includeParents` stance (a directory that could hold a protected descendant counts), the over-protective side:
9
+ * for a file that exists the hub is narrower, never broader.
10
+ * ponytail: no symlink alias discovery. Upgrade: ask the host (agent-hub) for the verdict through the library API once amem has a host policy port for paths.
11
+ */
12
+ import { posix } from "node:path";
13
+ import { nodePath, pathCandidates } from "./project.ts";
14
+
15
+ /** One spelling per name, as a case-insensitive disk sees it (agent-hub `foldSegment`). */
16
+ const foldSegment = (segment: string): string => segment.normalize("NFKC").replace(/\p{Default_Ignorable_Code_Point}/gu, "").toLowerCase().toUpperCase().toLowerCase();
17
+
18
+ /** Bounded project-relative exact paths/subtrees or `*`, `?` and whole-segment `**` globs (agent-hub `policyPaths`). Throws on anything the hub would refuse. */
19
+ export function policyEntries(raw: unknown, field: string): string[] {
20
+ const bad = raw === undefined ? false : !Array.isArray(raw) || raw.length > 64 || raw.some(p => typeof p !== "string" || !p || p.length > 200 || posix.isAbsolute(p) || /^[A-Za-z]:/.test(p) || /[\[\]{}!]/.test(p) || /[\x00-\x1f\x7f]/.test(p) || p.split(/[\\/]/).includes("..")
21
+ || (/[?*]/.test(p) && (p.includes("\\") || (p.match(/[?*]/g)?.length ?? 0) > 16 || p.split("/").some(s => s.includes("**") && s !== "**"))));
22
+ if (bad) throw new Error(`${field} takes up to 64 bounded project-relative paths or *, ? and whole-segment ** patterns of at most 200 characters (no absolute paths, .., brackets, braces or !)`);
23
+ if (raw === undefined) return [];
24
+ return [...new Set((raw as string[]).map(p => p.replace(/\/+/g, "/").replace(/\/$/, "") || "."))];
25
+ }
26
+
27
+ const within = (path: string, root: string) => path === root || path.startsWith(root.endsWith("/") ? root : `${root}/`);
28
+ const beneath = (path: string, root: string) => within(path.toLowerCase(), root.toLowerCase()) || within(foldSegment(path), foldSegment(root));
29
+
30
+ /** Either spelling protects: the lowercase comparison keeps `?` matching one code point whose fold expands, the folded one catches NFD/NFC and compatibility aliases. */
31
+ const segmentMatches = (pattern: string, name: string): boolean => wildcard(pattern.toLowerCase(), name.toLowerCase()) || wildcard(foldSegment(pattern), foldSegment(name));
32
+ /** Linear-space wildcard DP, no regex backtracking on untrusted paths. */
33
+ function wildcard(pattern: string, name: string): boolean {
34
+ const chars = [...name];
35
+ let previous = new Array<boolean>(chars.length + 1).fill(false); previous[0] = true;
36
+ for (const char of pattern) {
37
+ const next = new Array<boolean>(chars.length + 1).fill(false);
38
+ next[0] = char === "*" && previous[0]!;
39
+ for (let i = 1; i <= chars.length; i++) next[i] = char === "*" ? previous[i]! || next[i - 1]! : previous[i - 1]! && (char === "?" || char === chars[i - 1]);
40
+ previous = next;
41
+ }
42
+ return previous[chars.length]!;
43
+ }
44
+ function closure(pattern: string[], states: Set<number>): Set<number> {
45
+ const out = new Set(states);
46
+ for (const i of out) if (pattern[i] === "**") out.add(i + 1);
47
+ return out;
48
+ }
49
+ function advance(pattern: string[], states: Set<number>, name: string): Set<number> {
50
+ const next = new Set<number>();
51
+ for (const i of closure(pattern, states)) {
52
+ if (pattern[i] === "**") next.add(i);
53
+ else if (pattern[i] !== undefined && segmentMatches(pattern[i]!, name)) next.add(i + 1);
54
+ }
55
+ return closure(pattern, next);
56
+ }
57
+ function patternScope(pattern: string[], path: string, includeParents: boolean): boolean {
58
+ if (posix.isAbsolute(path) || path === ".." || path.startsWith("../")) return false;
59
+ let states = closure(pattern, new Set([0]));
60
+ if (states.has(pattern.length)) return true;
61
+ for (const name of path.split("/").filter(p => p && p !== ".")) {
62
+ states = advance(pattern, states, name);
63
+ if (states.has(pattern.length)) return true; // a matched directory protects its entire subtree
64
+ if (!states.size) return false;
65
+ }
66
+ return includeParents && states.size > 0; // this scope can contain a future protected descendant
67
+ }
68
+
69
+ const ROOT = "/project";
70
+ const segments = (entry: string) => entry.split("/").filter(p => p && p !== ".");
71
+
72
+ /** Whether `path` (project-relative as written in a note) falls under any entry: the hub's `matchesPolicy` without the filesystem. */
73
+ export function matchesEntries(path: string, entries: readonly string[], includeParents = true): boolean {
74
+ if (!entries.length) return false;
75
+ try {
76
+ if (path.length > 4096) return true;
77
+ const lexical = posix.resolve(ROOT, path), relative = posix.relative(ROOT, lexical);
78
+ return entries.some(entry => {
79
+ if (/[?*]/.test(entry)) return patternScope(segments(entry), relative, includeParents);
80
+ const target = posix.resolve(ROOT, entry);
81
+ return beneath(lexical, target) || includeParents && beneath(target, lexical) || patternScope([...segments(entry), "**"], relative, includeParents);
82
+ });
83
+ } catch { return true; }
84
+ }
85
+
86
+ /** Whether a note's text or its path links name a private path (agent-hub `mentionsPrivatePath`): written paths, and an entry's literal part written out. */
87
+ export function mentionsPrivatePath(text: string, about: readonly string[], entries: readonly string[]): boolean {
88
+ if (!entries.length) return false;
89
+ const nfc = (value: string) => value.normalize("NFC"), body = nfc(text), policy = entries.map(nfc);
90
+ const paths = [...about.map(nodePath), ...pathCandidates(body)].filter((p): p is string => p !== undefined).map(nfc);
91
+ if (paths.some(path => matchesEntries(path, policy, true))) return true;
92
+ // The entry's literal part (up to its first glob character) written out: catches a path a word split breaks (a space).
93
+ return policy.some(entry => { const literal = entry.replace(/^\.\//, "").split(/[*?[\]{}]/)[0]!.replace(/\/+$/, ""); return /[/\s]/.test(literal) && body.includes(literal); });
94
+ }
@@ -0,0 +1,47 @@
1
+ import { extractNodes, type View } from "./store.ts";
2
+ import { nodePath } from "./project.ts";
3
+
4
+ export interface PlanGuardContext { cwd: string; project: string; repo: string; deny?: string[]; now?: () => number }
5
+ /** Structural input accepts host task adapters without importing a host or a task board. */
6
+ export interface PlanGuardInput {
7
+ paths?: readonly string[]; symbols?: readonly string[];
8
+ refs?: { paths?: readonly string[] };
9
+ plan?: { paths?: readonly string[]; symbols?: readonly string[] };
10
+ }
11
+ export function normalizeMemorySymbolInput(value: string): string {
12
+ const at = value.lastIndexOf("@"), name = at < 0 ? value : value.slice(0, at);
13
+ const normalized = name.startsWith("section ") || name.startsWith("source ") ? name : name.replace(/\s*\(.*$/, "").trim();
14
+ return at < 0 ? normalized : `${normalized}@${value.slice(at + 1)}`;
15
+ }
16
+ /** Explicit inputs only. Per-input extraction retains more than v1's 64-node extraction cap. */
17
+ export function planGuardNodes(input: PlanGuardInput, context: PlanGuardContext): string[] {
18
+ const nodes = new Set<string>();
19
+ for (const path of [...input.paths ?? [], ...input.refs?.paths ?? [], ...input.plan?.paths ?? []]) {
20
+ for (const node of extractNodes("", { ...context, paths: [path], checkExists: false })) nodes.add(node);
21
+ }
22
+ const only = nodes.size === 1 ? nodePath([...nodes][0]!) : undefined;
23
+ for (const symbol of [...input.symbols ?? [], ...input.plan?.symbols ?? []]) {
24
+ if (typeof symbol !== "string") continue;
25
+ const at = symbol.lastIndexOf("@");
26
+ const explicitPath = at < 0 ? undefined : extractNodes("", { ...context, paths: [symbol.slice(at + 1)], checkExists: false })[0];
27
+ if (at >= 0 && !explicitPath) continue;
28
+ const qualified = explicitPath ? `${symbol.slice(0, at)}@${nodePath(explicitPath)}` : symbol;
29
+ for (const form of new Set([qualified, normalizeMemorySymbolInput(qualified)])) {
30
+ for (const node of extractNodes("", { ...context, paths: only ? [only] : [], symbols: [form], checkExists: false })) nodes.add(node);
31
+ }
32
+ }
33
+ return [...nodes].sort();
34
+ }
35
+ const samePath = (a: string, b: string) => a === b || a.startsWith(`${b}/`) || b.startsWith(`${a}/`);
36
+ /** Symbols require name and file identity; path scopes intersect only on whole components. */
37
+ export function planNodesIntersect(left: string, right: string): boolean {
38
+ if (left === right) return true;
39
+ if (left.startsWith("symbol:") && right.startsWith("symbol:")) return normalizeMemorySymbolInput(left.slice(7)) === normalizeMemorySymbolInput(right.slice(7));
40
+ const a = nodePath(left), b = nodePath(right);
41
+ return !!a && !!b && samePath(a, b);
42
+ }
43
+ /** Callers screen complete live notes first; this pure matcher performs no I/O or writes. */
44
+ export function matchingPlanFails<T extends Pick<View, "kind" | "about" | "supersededBy"> & Partial<Pick<View, "expiresAt">>>(input: PlanGuardInput, notes: readonly T[], context: PlanGuardContext): T[] {
45
+ const nodes = planGuardNodes(input, context), now = (context.now ?? Date.now)();
46
+ return nodes.length && Number.isFinite(now) ? notes.filter(note => note.kind === "fail" && !note.supersededBy && (note.expiresAt == null || Number.isFinite(Date.parse(note.expiresAt)) && Date.parse(note.expiresAt) > now) && note.about.some(about => nodes.some(node => planNodesIntersect(node, about)))) : [];
47
+ }
@@ -99,7 +99,18 @@ export function memoryGitEnv(env: NodeJS.ProcessEnv = process.env, options: { re
99
99
  /** spawnSync form of the same call: the env option is merged with memoryGitEnv, everything else passes through. */
100
100
  export function memoryGitSync(args: string[], options: SpawnSyncOptionsWithStringEncoding): SpawnSyncReturns<string> {
101
101
  const argv = memoryGitArgv(args);
102
- return childProcess().spawnSync(argv[0]!, argv.slice(1), { ...options, env: memoryGitEnv(options.env ?? process.env) });
102
+ return childProcess().spawnSync(argv[0]!, argv.slice(1), { ...(ambientGitTimeout !== undefined ? { timeout: ambientGitTimeout, killSignal: "SIGKILL" as const } : {}), ...options, env: memoryGitEnv(options.env ?? process.env) });
103
+ }
104
+ let ambientGitTimeout: number | undefined;
105
+ /**
106
+ * Runs `work` with a timeout on every `memoryGitSync` call that sets none of its own: a synchronous git call blocks the event loop, so a signal sent to this process alone waits for it.
107
+ * A call that times out reads as a failed git (`error` set, no status), which every caller already treats as "unknown". The setting is process-wide while `work` runs, so use it only in a
108
+ * command that does nothing else at the same time (`amem doctor`).
109
+ * ponytail: one global instead of a parameter on every caller between doctor and its git calls (project key, repository root, visibility). Upgrade: thread an options argument through them.
110
+ */
111
+ export async function withGitTimeout<T>(ms: number, work: () => Promise<T>): Promise<T> {
112
+ const prior = ambientGitTimeout; ambientGitTimeout = ms;
113
+ try { return await work(); } finally { ambientGitTimeout = prior; }
103
114
  }
104
115
 
105
116
  /**
@@ -6,6 +6,18 @@ import { nodePath, pathCandidates } from "./project.ts";
6
6
 
7
7
  /** True means the complete value is public. Hosts may narrow, never replace, standalone policy. */
8
8
  export interface MemoryPolicy { screen(value: unknown): Promise<boolean> }
9
+ /**
10
+ * The standalone policy. `screenStandalone` is its own part only (secrets, default PII, configured PII patterns and deny paths) as a synchronous
11
+ * answer, for callers that cannot await (the store's promotion screen); it never asks the host, so a caller with a host uses `screen`.
12
+ */
13
+ export interface StandalonePolicy extends MemoryPolicy {
14
+ screenStandalone(value: unknown): boolean;
15
+ /**
16
+ * Only the personal-data and configured-pattern and deny-path part of `screenStandalone`, without the secret screen, for a caller that counts secrets
17
+ * separately (the claude-mem import reports `pii` and `secret` skips as two numbers). True means the value names personal data or something configured.
18
+ */
19
+ declinesPersonal(text: string): boolean;
20
+ }
9
21
  export interface StandalonePolicyOptions {
10
22
  denyPaths?: readonly string[];
11
23
  /** Case-insensitive literal substrings, not executable regular expressions. */
@@ -14,7 +26,7 @@ export interface StandalonePolicyOptions {
14
26
  }
15
27
 
16
28
  const MAX_DEPTH = 16, MAX_NODES = 4096, MAX_BYTES = 256 * 1024, MAX_STRING_BYTES = 16 * 1024;
17
- const fold = (text: string) => text.normalize("NFKC").replace(/\p{Default_Ignorable_Code_Point}/gu, "").toLowerCase().replace(/\\/g, "/");
29
+ export const fold = (text: string) => text.normalize("NFKC").replace(/\p{Default_Ignorable_Code_Point}/gu, "").toLowerCase().replace(/\\/g, "/");
18
30
  const flags = new Set(["private", "pii", "privateturn", "isprivate", "ispii"]);
19
31
 
20
32
  /** Inspect descriptors, not getters/toJSON, and build the only value a host is allowed to see. */
@@ -67,6 +79,9 @@ function snapshot(value: unknown): { value: unknown; texts: string[] } | undefin
67
79
  catch { return undefined; }
68
80
  }
69
81
 
82
+ /** Safe structural data copy for optional readers, before any getters/toJSON can run. It is not a public verdict. */
83
+ export function memoryPolicySnapshot<T>(value: T): T | undefined { return snapshot(value)?.value as T | undefined; }
84
+
70
85
  function literals(value: unknown, label: string): string[] {
71
86
  if (value === undefined) return [];
72
87
  if (types.isProxy(value) || !Array.isArray(value) || Object.getPrototypeOf(value) !== Array.prototype || value.length > 64 || Object.getOwnPropertySymbols(value).length) throw new Error(`invalid ${label}`);
@@ -92,7 +107,7 @@ function defaultPii(text: string): boolean {
92
107
  }
93
108
 
94
109
  /** No filesystem, model, network or storage work occurs while constructing or screening this policy. */
95
- export function createStandalonePolicy(options: StandalonePolicyOptions = {}): MemoryPolicy {
110
+ export function createStandalonePolicy(options: StandalonePolicyOptions = {}): StandalonePolicy {
96
111
  if (!options || typeof options !== "object" || types.isProxy(options) || Object.getPrototypeOf(options) !== Object.prototype || Object.getOwnPropertySymbols(options).length) throw new Error("invalid standalone policy options");
97
112
  const descriptors = Object.getOwnPropertyDescriptors(options);
98
113
  for (const [key, descriptor] of Object.entries(descriptors)) if (!["denyPaths", "piiPatterns", "host"].includes(key) || !("value" in descriptor)) throw new Error("invalid standalone policy options");
@@ -100,20 +115,36 @@ export function createStandalonePolicy(options: StandalonePolicyOptions = {}): M
100
115
  const pii = literals(descriptors.piiPatterns?.value, "PII patterns");
101
116
  const host = descriptors.host?.value as MemoryPolicy | undefined;
102
117
  if (host !== undefined && (!host || typeof host.screen !== "function")) throw new Error("invalid host policy");
118
+ const standalone = (value: unknown): { ok: boolean; value?: unknown } => {
119
+ try {
120
+ const complete = snapshot(value);
121
+ if (!complete) return { ok: false };
122
+ for (const text of complete.texts) {
123
+ if (looksSecret(text) || looksSecret(text.replace(/["']/g, ""))) return { ok: false };
124
+ const normalized = fold(text);
125
+ if (defaultPii(normalized) || pii.some(pattern => normalized.includes(pattern))) return { ok: false };
126
+ const candidates = normalized.includes("/") || /\.[a-z]/.test(normalized) ? pathCandidates(normalized) : [];
127
+ const paths = [normalized, nodePath(normalized), ...candidates, ...normalized.split(/[\s"'`(){}\[\],;:=<>]+/)];
128
+ for (const path of paths) if (path && isDenied(posix.normalize(path), deny)) return { ok: false };
129
+ }
130
+ return { ok: true, value: complete.value };
131
+ } catch { return { ok: false }; }
132
+ };
133
+ const personal = (text: string): boolean => {
134
+ const normalized = fold(text);
135
+ if (defaultPii(normalized) || pii.some(pattern => normalized.includes(pattern))) return true;
136
+ const candidates = normalized.includes("/") || /\.[a-z]/.test(normalized) ? pathCandidates(normalized) : [];
137
+ const paths = [normalized, nodePath(normalized), ...candidates, ...normalized.split(/[\s"'`(){}\[\],;:=<>]+/)];
138
+ return paths.some(path => path && isDenied(posix.normalize(path), deny));
139
+ };
103
140
  return {
141
+ screenStandalone: value => standalone(value).ok,
142
+ declinesPersonal: text => personal(text),
104
143
  async screen(value: unknown): Promise<boolean> {
105
144
  try {
106
- const complete = snapshot(value);
107
- if (!complete) return false;
108
- for (const text of complete.texts) {
109
- if (looksSecret(text) || looksSecret(text.replace(/["']/g, ""))) return false;
110
- const normalized = fold(text);
111
- if (defaultPii(normalized) || pii.some(pattern => normalized.includes(pattern))) return false;
112
- const candidates = normalized.includes("/") || /\.[a-z]/.test(normalized) ? pathCandidates(normalized) : [];
113
- const paths = [normalized, nodePath(normalized), ...candidates, ...normalized.split(/[\s"'`(){}\[\],;:=<>]+/)];
114
- for (const path of paths) if (path && isDenied(posix.normalize(path), deny)) return false;
115
- }
116
- return host === undefined || await host.screen(complete.value) === true;
145
+ const own = standalone(value);
146
+ if (!own.ok) return false;
147
+ return host === undefined || await host.screen(own.value) === true;
117
148
  } catch { return false; }
118
149
  },
119
150
  };
@@ -0,0 +1,32 @@
1
+ /**
2
+ * `<private>...</private>` spans (the claude-mem convention) are dropped before anything else reads the text, so they are never screened, counted or
3
+ * stored. A leaf module with no imports: capture shaping, the candidate and work-list boundaries, notes, goals and imports all use it.
4
+ *
5
+ * Tags match the way an HTML reader would, so a variant cannot slip a span through: the name is case-insensitive, an opening tag may carry attributes
6
+ * (`<private class="x">`), whitespace before the name (`< private>`) and a self-closing slash (`<private/>`, which opens a span like any other opening
7
+ * tag, since a person who typed it meant what follows to stay private). A closing tag is `</private>` with the same latitude. Nested spans close from the
8
+ * inside out. An unclosed span drops everything to the end of the text. A stray closing tag is kept as plain text. `<privateer>` is not a tag.
9
+ */
10
+ const TAG = /<\s*(\/?)\s*private(?=[\s/>])[^<>]*>/gi;
11
+
12
+ export function stripPrivateSpans(text: string): string {
13
+ if (!/<\s*\/?\s*private(?=[\s/>])/i.test(text)) return text;
14
+ let out = "", depth = 0, last = 0, match: RegExpExecArray | null;
15
+ TAG.lastIndex = 0;
16
+ while ((match = TAG.exec(text))) {
17
+ const closing = match[1] === "/", stray = closing && depth === 0;
18
+ if (depth === 0) out += text.slice(last, stray ? match.index + match[0].length : match.index); // a stray closer is plain text: keep it
19
+ if (closing) { if (depth > 0) depth--; } else depth++;
20
+ last = match.index + match[0].length;
21
+ }
22
+ if (depth === 0) out += text.slice(last);
23
+ return out;
24
+ }
25
+
26
+ /** The same strip over every string inside a plain value (one level of arrays and objects, as the callers' inputs are), leaving other values as they are. */
27
+ export function stripPrivateDeep<T>(value: T, depth = 0): T {
28
+ if (typeof value === "string") return stripPrivateSpans(value) as T;
29
+ if (depth > 4 || value === null || typeof value !== "object") return value;
30
+ if (Array.isArray(value)) return value.map(item => stripPrivateDeep(item, depth + 1)) as T;
31
+ return Object.fromEntries(Object.entries(value as Record<string, unknown>).map(([key, item]) => [key, stripPrivateDeep(item, depth + 1)])) as T;
32
+ }
@@ -0,0 +1,157 @@
1
+ import { Database } from "bun:sqlite";
2
+ import { closeSync, constants, existsSync, lstatSync, openSync, rmSync, statSync, writeSync } from "node:fs";
3
+ import { basename, dirname, join, resolve, sep } from "node:path";
4
+ import { walkShards } from "./catalog.ts";
5
+ import { openCuratedForRead, requireShard } from "./curated-read.ts";
6
+ import { projectState } from "./lifecycle.ts";
7
+ import { realPath } from "./platform.ts";
8
+ import { createStandalonePolicy, type MemoryPolicy } from "./policy.ts";
9
+ import { looksSecret } from "./screen.ts";
10
+ import { aliasedKeys, shardDir } from "./shard.ts";
11
+ import { WorkListStore } from "./work-lists.ts";
12
+
13
+ /**
14
+ * #15 T5 / D3 and F17: `amem project export <key> --to <file>`. One project's capture rows, curated notes, goals and work lists as a portable JSON bundle,
15
+ * JSONL stream or Markdown document, written once to a new file with mode 0600. It changes no content file of the home: it opens the capture databases read-only and the curated store only to
16
+ * read, and only derived files (the curated `index.db`, `search.db`, SQLite sidecars) may refresh, as they do for any read. A destination inside the home is refused.
17
+ *
18
+ * The bundle leaves the home, so it is screened like sync and promotion are: a secret-looking value is withheld, and so is anything the host policy (when
19
+ * one is passed) declines. `withheld` counts the capture rows, notes and goals left out; a work list the policy declines is left out silently by its own store.
20
+ * A forgotten project cannot be exported (the person's decision stands); an archived one can. A failure removes the partial file it created.
21
+ * ponytail: JSONL capture, goal and work-list rows stream, but curated authored rows are materialized to preserve ids. Add a bounded authored-row iterator before raising shard limits.
22
+ */
23
+
24
+ export const EXPORT_SCHEMA = "agent-mem.project-export/v1";
25
+ /**
26
+ * ponytail: a seam, not a lock. The destination is checked (for the message) and then claimed with O_EXCL (for the guarantee); a test sets `afterCheck` to create
27
+ * the file in the gap between them, which is the only way to show that O_EXCL, not the check, keeps an export from replacing someone else's file.
28
+ */
29
+ export const exportHooks: { afterCheck?: (file: string) => void } = {};
30
+ export type ProjectExportFormat = "json" | "jsonl" | "markdown";
31
+ export interface ExportCounts { capture: number; curated: number; goals: number; workLists: number }
32
+ export interface ExportResult { file: string; project: string; format: ProjectExportFormat; counts: ExportCounts; withheld: number }
33
+ type ExportSection = keyof ExportCounts;
34
+ const PAGE = 500;
35
+
36
+ interface CaptureRowRaw { id: number; at: string; project: string; agent: string; event: string; tool: string | null; paths: string; status: string | null; excerpt: string }
37
+
38
+ function destination(base: string, to: string): string {
39
+ if (typeof to !== "string" || !to.trim()) throw new Error("export needs --to <file>");
40
+ const file = resolve(to), parent = dirname(file), name = basename(file);
41
+ if (!name || file === parent) throw new Error("--to names a file, not a directory root");
42
+ let parentReal: string;
43
+ try { if (!statSync(parent).isDirectory()) throw new Error(); parentReal = realPath(parent); }
44
+ catch { throw new Error("the destination directory does not exist"); }
45
+ let homeReal = base;
46
+ try { homeReal = realPath(base); } catch { /* a home that is not there yet holds nothing to collide with */ }
47
+ if (parentReal === homeReal || parentReal.startsWith(homeReal + sep)) throw new Error("the destination is inside the memory home; choose a folder outside it");
48
+ if (lstatSync(file, { throwIfNoEntry: false })) throw new Error("the destination exists; export never overwrites");
49
+ return join(parentReal, name);
50
+ }
51
+
52
+ function markdownInline(text: string): string {
53
+ return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
54
+ }
55
+
56
+ function markdownRecord(section: ExportSection, value: unknown): string {
57
+ const row = value as Record<string, unknown>;
58
+ const body = section === "capture" ? row.excerpt : section === "curated" ? row.text : undefined;
59
+ const identity = typeof row.id === "string" ? ` ${row.id}` : "";
60
+ const metadata = { ...row };
61
+ if (section === "capture") delete metadata.excerpt;
62
+ if (section === "curated") delete metadata.text;
63
+ const lines = [`### ${section}${identity}`, "", ` ${JSON.stringify(metadata)}`, ""];
64
+ if (typeof body === "string") lines.push(...body.split(/\r\n|\r|\n/).map(line => ` ${line}`), "");
65
+ return `${lines.join("\n")}\n`;
66
+ }
67
+
68
+ export async function exportProject(base: string, key: string, opts: { to: string; policy?: MemoryPolicy; format?: ProjectExportFormat }): Promise<ExportResult> {
69
+ const project = requireShard(base, key), format = opts.format ?? "json";
70
+ if (!["json", "jsonl", "markdown"].includes(format)) throw new Error("export format must be json, jsonl or markdown");
71
+ if (projectState(base, project) === "forgotten") throw new Error(`project ${project} is forgotten: it cannot be exported`);
72
+ const file = destination(base, opts.to), policy = opts.policy ?? createStandalonePolicy();
73
+ exportHooks.afterCheck?.(file);
74
+ const allowed = async (value: unknown): Promise<boolean> => !looksSecret(JSON.stringify(value)) && await policy.screen(value);
75
+ const counts: ExportCounts = { capture: 0, curated: 0, goals: 0, workLists: 0 };
76
+ let withheld = 0;
77
+ let fd: number;
78
+ try { fd = openSync(file, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
79
+ catch { throw new Error("the destination exists; export never overwrites"); }
80
+ const out = (text: string) => { writeSync(fd, text); };
81
+ const exportedAt = new Date().toISOString();
82
+ let first = true, section: ExportSection = "capture";
83
+ const begin = (jsonPrefix: string, next: ExportSection, heading: string) => {
84
+ section = next; first = true;
85
+ if (format === "json") out(jsonPrefix);
86
+ else if (format === "markdown") out(`\n## ${heading}\n\n`);
87
+ };
88
+ const emit = (value: unknown) => {
89
+ if (format === "json") out(`${first ? "" : ","}${JSON.stringify(value)}`);
90
+ else if (format === "jsonl") out(`${JSON.stringify({ schema: EXPORT_SCHEMA, project, type: section, value })}\n`);
91
+ else out(markdownRecord(section, value));
92
+ first = false;
93
+ };
94
+ const stores: { close(): void }[] = [];
95
+ try {
96
+ if (format === "jsonl") out(`${JSON.stringify({ schema: EXPORT_SCHEMA, project, exportedAt, type: "header", aliases: aliasedKeys(base, project) })}\n`);
97
+ else if (format === "markdown") out(`# Project export ${markdownInline(JSON.stringify(project))}\n\nExported at ${exportedAt}\n`);
98
+ begin(`{"schema":${JSON.stringify(EXPORT_SCHEMA)},"project":${JSON.stringify(project)},"exportedAt":${JSON.stringify(exportedAt)},"capture":[`, "capture", "Capture");
99
+ const shard = walkShards(base).find(candidate => candidate.key === project);
100
+ for (const dir of shard?.machineDirs ?? []) {
101
+ const dbFile = join(dir, "capture.db");
102
+ if (!existsSync(dbFile)) continue;
103
+ const db = new Database(dbFile, { readonly: true });
104
+ try {
105
+ db.run("PRAGMA busy_timeout = 3000");
106
+ for (let after = 0; ;) {
107
+ const rows = db.query("SELECT id, at, project, agent, event, tool, paths, status, excerpt FROM rows WHERE id > ? ORDER BY id LIMIT ?").all(after, PAGE) as CaptureRowRaw[];
108
+ for (const r of rows) {
109
+ after = r.id;
110
+ const row = { at: r.at, agent: r.agent, event: r.event, tool: r.tool, paths: JSON.parse(r.paths) as string[], status: r.status, excerpt: r.excerpt };
111
+ if (!await allowed(row)) { withheld++; continue; }
112
+ emit(row); counts.capture++;
113
+ }
114
+ if (rows.length < PAGE) break;
115
+ }
116
+ } finally { db.close(); }
117
+ }
118
+ begin(`],"curated":[`, "curated", "Curated notes");
119
+ const store = openCuratedForRead(base, project);
120
+ if (store) stores.push(store);
121
+ // The project's own rows and those of a key aliased to it; another project's shared note is not this project's and stays out of its bundle.
122
+ const keys = new Set([project, ...aliasedKeys(base, project)]);
123
+ const authored = format === "jsonl" ? new Map((store?.portableNotes(project) ?? []).map(row => [row.id, row])) : undefined;
124
+ for (const view of store?.list({ project, limit: Number.MAX_SAFE_INTEGER, includeSuperseded: true }) ?? []) {
125
+ if (!keys.has(view.project)) continue;
126
+ const value = format === "jsonl" ? authored!.get(view.id) : view;
127
+ if (!value) throw new Error("exported note's authored row is unavailable");
128
+ if (!await allowed(value)) { withheld++; continue; }
129
+ emit(value); counts.curated++;
130
+ }
131
+ begin(`],"goals":[`, "goals", "Goals");
132
+ for (const goal of store?.goals(project) ?? []) {
133
+ if (!keys.has(goal.project)) continue;
134
+ if (!await allowed(goal)) { withheld++; continue; }
135
+ emit(goal); counts.goals++;
136
+ }
137
+ begin(`],"workLists":[`, "workLists", "Work lists");
138
+ // The bundle holds what the project's own reads show: the canonical work log and that of every key aliased to it (the same reconciliation the CLI, MCP and brief use).
139
+ const workKeys = [project, ...aliasedKeys(base, project)];
140
+ if (workKeys.some(key => existsSync(join(shardDir(base, key), "work")))) {
141
+ const work = new WorkListStore({ base, project, policy: allowed, aliases: workKeys.slice(1) });
142
+ stores.push({ close: () => work.close() });
143
+ for (const list of await work.read(undefined, true)) { emit(list); counts.workLists++; }
144
+ }
145
+ if (format === "json") out(`],"counts":${JSON.stringify(counts)},"withheld":${withheld}}\n`);
146
+ else if (format === "jsonl") out(`${JSON.stringify({ schema: EXPORT_SCHEMA, project, type: "counts", counts, withheld })}\n`);
147
+ else out(`\n## Counts\n\n${JSON.stringify({ counts, withheld })}\n`);
148
+ } catch (error) {
149
+ try { closeSync(fd); } catch { /* already closed */ }
150
+ rmSync(file, { force: true }); // the file is ours: O_EXCL created it, so a partial bundle is removed, never left behind
151
+ throw error;
152
+ } finally {
153
+ for (const store of stores) { try { store.close(); } catch { /* best effort */ } }
154
+ }
155
+ closeSync(fd);
156
+ return { file, project, format, counts, withheld };
157
+ }
@@ -0,0 +1,147 @@
1
+ import { createHash } from "node:crypto";
2
+ import { constants, createReadStream, fstatSync, openSync } from "node:fs";
3
+ import { createInterface } from "node:readline";
4
+ import { isDeepStrictEqual } from "node:util";
5
+ import { legacyEntries } from "./bridge.ts";
6
+ import { stripPrivateDeep, stripPrivateSpans } from "./private-spans.ts";
7
+ import { EXPORT_SCHEMA } from "./project-export.ts";
8
+ import { aliasedKeys } from "./shard.ts";
9
+ import { looksSecret, type MemoryStore, type NoteInput } from "./store.ts";
10
+
11
+ export const PROJECT_IMPORT_MAX_BYTES = 64 * 1024 * 1024;
12
+ export const PROJECT_IMPORT_MAX_LINE_BYTES = 64 * 1024;
13
+ export const PROJECT_IMPORT_FUTURE_SKEW_MS = 5 * 60 * 1000;
14
+ const NOTE_KEYS = new Set(["schema", "id", "at", "machine", "op", "project", "scope", "tier", "kind", "peer", "title", "text", "task", "about", "supersedes", "promotedFrom", "source", "expires"]);
15
+ const OPERATIONS_WARNING = "Supersede, demote, settle and forget operations are not exported; re-import restores authored note rows only.";
16
+
17
+ type ImportedNote = { id: string; at: string; machine: string; note: NoteInput };
18
+ type Provenance = { source: Record<string, number>; peer: Record<string, number>; scope: Record<string, number> };
19
+ export interface ProjectImportResult {
20
+ applied: boolean;
21
+ project: string;
22
+ read: number;
23
+ added: number;
24
+ wouldAdd: number;
25
+ skipped: { invalid: number; policy: number; other: number };
26
+ provenance: Provenance;
27
+ warnings: string[];
28
+ }
29
+
30
+ function importedNote(value: unknown, projects: readonly string[]): ImportedNote | undefined {
31
+ try {
32
+ if (!value || typeof value !== "object" || Array.isArray(value) || !Object.keys(value).every(key => NOTE_KEYS.has(key))) return undefined;
33
+ const bytes = Buffer.from(`${JSON.stringify(value)}\n`);
34
+ const [row] = legacyEntries([{ source: "legacy-local", path: "log/project-import.jsonl", bytes, offset: bytes.length,
35
+ hash: createHash("sha256").update(bytes).digest("hex"), local: false, tailBytes: 0 }], { strict: true });
36
+ if (row?.op !== "note" || !projects.includes(row.project)) return undefined;
37
+ const { id, at, machine, ...rest } = row;
38
+ const note = Object.fromEntries(Object.entries(rest).filter(([key]) => key !== "schema")) as NoteInput;
39
+ return { id, at, machine, note };
40
+ } catch { return undefined; }
41
+ }
42
+
43
+ function increment(counts: Map<string, number>, value: string | undefined): void {
44
+ const key = value ?? "unspecified";
45
+ counts.set(key, (counts.get(key) ?? 0) + 1);
46
+ }
47
+
48
+ function provenance(notes: Iterable<ImportedNote>): Provenance {
49
+ const source = new Map<string, number>(), peer = new Map<string, number>(), scope = new Map<string, number>();
50
+ for (const { note } of notes) {
51
+ increment(source, note.source);
52
+ increment(peer, note.peer);
53
+ increment(scope, note.scope);
54
+ }
55
+ return { source: Object.fromEntries(source), peer: Object.fromEntries(peer), scope: Object.fromEntries(scope) };
56
+ }
57
+
58
+ function unchangedAfterNormalization(note: NoteInput): boolean {
59
+ return stripPrivateSpans(note.text).slice(0, 4000) === note.text
60
+ && (typeof note.title !== "string" || stripPrivateSpans(note.title).slice(0, 200) === note.title);
61
+ }
62
+
63
+ /** Preview by default; confirm applies only after every row, relationship, identity and policy decision has passed without a curated write. */
64
+ export async function importProject(store: MemoryStore, opts: {
65
+ from: string;
66
+ project: string;
67
+ confirm?: boolean;
68
+ allowShared?: boolean;
69
+ policy: (note: NoteInput) => boolean | Promise<boolean>;
70
+ }): Promise<ProjectImportResult> {
71
+ const fd = openSync(opts.from, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
72
+ const stream = createReadStream(opts.from, { fd, encoding: "utf8" });
73
+ const result: ProjectImportResult = { applied: opts.confirm === true, project: opts.project, read: 0, added: 0, wouldAdd: 0,
74
+ skipped: { invalid: 0, policy: 0, other: 0 }, provenance: { source: {}, peer: {}, scope: {} }, warnings: [OPERATIONS_WARNING] };
75
+ let header = false, bytes = 0;
76
+ let aliases: string[] = [];
77
+ const pending = new Map<string, ImportedNote>(), accepted: ImportedNote[] = [];
78
+ try {
79
+ const stat = fstatSync(fd);
80
+ if (!stat.isFile() || stat.size > PROJECT_IMPORT_MAX_BYTES) throw new Error("project import needs a regular JSONL file of at most 64 MiB");
81
+ const lines = createInterface({ input: stream, crlfDelay: Infinity });
82
+ try {
83
+ for await (const line of lines) {
84
+ bytes += Buffer.byteLength(line, "utf8") + 1;
85
+ if (bytes > PROJECT_IMPORT_MAX_BYTES) throw new Error("project import exceeded 64 MiB while reading");
86
+ if (Buffer.byteLength(line, "utf8") > PROJECT_IMPORT_MAX_LINE_BYTES) { result.skipped.invalid++; continue; }
87
+ if (!line.trim()) continue;
88
+ let row: Record<string, unknown>;
89
+ try {
90
+ const parsed: unknown = JSON.parse(line);
91
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error();
92
+ row = parsed as Record<string, unknown>;
93
+ } catch { result.skipped.invalid++; continue; }
94
+ if (row.schema !== EXPORT_SCHEMA || row.project !== opts.project) throw new Error("project import schema or project mismatch");
95
+ if (row.type === "header") {
96
+ if (header) throw new Error("project import has more than one header");
97
+ if (!Array.isArray(row.aliases) || !row.aliases.every(alias => typeof alias === "string")) throw new Error("project import aliases are invalid");
98
+ aliases = [...row.aliases] as string[];
99
+ const available = aliasedKeys(store.profile.home, opts.project);
100
+ if (aliases.some(alias => !available.includes(alias))) throw new Error("project import requires the exported aliases in the target home");
101
+ header = true; continue;
102
+ }
103
+ if (!header) throw new Error("project import needs its header first");
104
+ if (row.type === "counts") continue;
105
+ if (row.type !== "curated") { result.skipped.other++; continue; }
106
+ result.read++;
107
+ const imported = importedNote(row.value, [opts.project, ...aliases]);
108
+ if (!imported || Date.parse(imported.at) > store.readNow() + PROJECT_IMPORT_FUTURE_SKEW_MS
109
+ || imported.note.scope === "shared" && opts.allowShared !== true || !unchangedAfterNormalization(imported.note)) {
110
+ result.skipped.invalid++;
111
+ continue;
112
+ }
113
+ const prior = pending.get(imported.id);
114
+ if (prior) {
115
+ if (!isDeepStrictEqual(prior, imported)) throw new Error("project import has conflicting rows for one entry id");
116
+ continue;
117
+ }
118
+ pending.set(imported.id, imported);
119
+ }
120
+ } finally { lines.close(); }
121
+ if (!header) throw new Error("project import has no header");
122
+
123
+ for (const imported of pending.values()) {
124
+ if (["supersedes", "promotedFrom"].some(field => {
125
+ const target = imported.note[field as "supersedes" | "promotedFrom"] as string | null | undefined;
126
+ return target !== undefined && target !== null && !pending.has(target);
127
+ })) {
128
+ result.skipped.invalid++;
129
+ continue;
130
+ }
131
+ const screened = stripPrivateDeep(imported.note);
132
+ if (looksSecret(JSON.stringify(screened)) || !await opts.policy(screened)) { result.skipped.policy++; continue; }
133
+ accepted.push(imported);
134
+ }
135
+ result.provenance = provenance(accepted);
136
+ const existing = new Set(store.portableNotes(opts.project).map(note => note.id));
137
+ result.wouldAdd = accepted.filter(note => !existing.has(note.id)).length;
138
+ if (opts.confirm === true) {
139
+ for (const imported of accepted) {
140
+ const written = store.importNote(imported.note, imported.at, { machine: imported.machine, expectedId: imported.id });
141
+ if (written.added) result.added++;
142
+ }
143
+ store.refresh();
144
+ }
145
+ return result;
146
+ } finally { stream.destroy(); }
147
+ }