@staix/agent-mem 0.6.0 → 0.8.0

package/src/memory/sync-cadence.ts

@@ -0,0 +1,149 @@
+import { closeSync, constants, fstatSync, lstatSync, mkdirSync, openSync, readSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { generationDir, readSelector } from "./generation.ts";
+
+/**
+ * F16, `amem sync --if-due`: the person's own shell (an EXIT trap) asks amem to sync when it is due; amem itself runs nothing between sessions and never
+ * registers a hook for it. Like every mutating command it needs a terminal, so an unattended scheduler (cron, launchd) is not a caller (a hook may not do network work: AGENTS.md, test/hooks-network.test.ts). The only state is one small stamp beside the logs.
+ * The stamp holds times and a failure count, never an error text (a git error can carry a remote URL).
+ */
+export interface SyncState { lastAttemptAt: string | null; lastOkAt: string | null; failures: number }
+export const SYNC_STATE_FILE = "sync-state.json";
+/** After a success, the next attempt is due this long after the last one. */
+export const SYNC_INTERVAL_MS = 60 * 60_000;
+/** After a failure the wait starts here and doubles with each further failure. */
+export const SYNC_BACKOFF_BASE_MS = 15 * 60_000;
+export const SYNC_BACKOFF_CAP_MS = 24 * 60 * 60_000;
+const MAX_FAILURES = 30, MAX_BYTES = 4096;
+const EMPTY: SyncState = { lastAttemptAt: null, lastOkAt: null, failures: 0 };
+const iso = (value: unknown): string | null => typeof value === "string" && Number.isFinite(Date.parse(value)) && new Date(value).toISOString() === value ? value : null;
+
+/** Never throws: a missing, oversized or malformed stamp is "no history", which makes a sync due. */
+export function readSyncState(home: string): SyncState {
+  let fd: number | undefined;
+  try {
+    fd = openSync(join(home, SYNC_STATE_FILE), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+    const stat = fstatSync(fd);
+    if (!stat.isFile() || stat.size > MAX_BYTES) return { ...EMPTY };
+    const buffer = Buffer.alloc(stat.size), size = readSync(fd, buffer, 0, buffer.length, 0);
+    const value = JSON.parse(buffer.subarray(0, size).toString("utf8")) as Record<string, unknown>;
+    const failures = Number.isSafeInteger(value.failures) && (value.failures as number) >= 0 ? Math.min(value.failures as number, MAX_FAILURES) : 0;
+    return { lastAttemptAt: iso(value.lastAttemptAt), lastOkAt: iso(value.lastOkAt), failures };
+  } catch { return { ...EMPTY }; }
+  finally { if (fd !== undefined) try { closeSync(fd); } catch { /* nothing to close */ } }
+}
+
+/** The wait after the last attempt: the plain interval after a success, 15 minutes doubling to a day after failures. */
+export function syncWaitMs(failures: number): number {
+  return failures <= 0 ? SYNC_INTERVAL_MS : Math.min(SYNC_BACKOFF_BASE_MS * 2 ** (failures - 1), SYNC_BACKOFF_CAP_MS);
+}
+
+/** The earliest next attempt, or null when none was recorded (due now). */
+export function nextSyncAt(state: SyncState): number | null {
+  return state.lastAttemptAt === null ? null : Date.parse(state.lastAttemptAt) + syncWaitMs(state.failures);
+}
+
+/** `force` (`--force`) ignores the schedule and the backoff. */
+export function syncDue(state: SyncState, now: number, force = false): boolean {
+  const next = nextSyncAt(state);
+  return force || next === null || now >= next;
+}
+
+/** `--if-due` stops a running git step after this long, so a shell that runs it on exit is held for a bounded time. */
+export const SYNC_IF_DUE_DEADLINE_MS = 30_000;
+
+/**
+ * The stop signal of one `--if-due` run: the 30 second deadline, or any way the person ends it: Ctrl-C (SIGINT), SIGTERM, SIGHUP (the terminal closed, which is also how an
+ * EXIT trap's shell can go away) and SIGQUIT. git runs in its own process group while a stop signal is given (so the stop reaches its transport children), which also means a
+ * terminal's signals no longer reach git by themselves: amem turns each of them into the abort. `on`, not `once`: a second Ctrl-C must not fall back to the default action
+ * (which would end amem and strand git) while the first is still stopping git; the handlers stay until `dispose`. A process that dies some other way is covered by the exit
+ * handler in sync.ts, which signals every live git group. The emitter is a seam for tests; the CLI passes `process`.
+ */
+export const STOP_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP", "SIGQUIT"] as const;
+export type StopCause = "person" | "deadline";
+export function stopSignal(ms: number, emitter: { on(name: string, fn: () => void): unknown; off(name: string, fn: () => void): unknown }, names: readonly string[] = STOP_SIGNALS): { signal: AbortSignal; dispose: () => void; cause: () => StopCause | undefined; signalName: () => string | undefined } {
+  const controller = new AbortController(), deadline = AbortSignal.timeout(ms);
+  let cause: StopCause | undefined, named: string | undefined;
+  // The first thing that ends the run is its cause: a person who closes the terminal after the deadline already fired did not stop it.
+  const handlers = names.map(name => [name, () => { if (cause === undefined) { cause = "person"; named = name; } controller.abort(); }] as const);
+  deadline.addEventListener("abort", () => { cause ??= "deadline"; }, { once: true });
+  for (const [name, handler] of handlers) emitter.on(name, handler);
+  return { signal: AbortSignal.any([controller.signal, deadline]), dispose: () => { for (const [name, handler] of handlers) emitter.off(name, handler); }, cause: () => cause, signalName: () => named };
+}
+
+/**
+ * End this process the way the signal would have, once amem's own handlers are removed (a handler suppresses the default action, so a run a person ended would otherwise exit 1
+ * instead of 130, 143, 129). SIGQUIT's default action is a core dump, and Bun honours it: a re-raised SIGQUIT on Linux writes a core file the size of the process image (measured
+ * 5.9 GB, taking seconds, in a container with `ulimit -c unlimited`) while the shell waits for the exit, so it exits with the status that death would have had (128 + 3) at once.
+ */
+export function endLikeSignal(name: string): void {
+  if (name === "SIGQUIT") process.exit(128 + 3);
+  process.kill(process.pid, name as NodeJS.Signals);
+}
+
+/**
+ * The repository the active profile syncs through: a curated split keeps a private cache in `shard-sync/objects.git`, a generation home keeps its own `.git` inside the
+ * generation directory (the memory home's own `.git` is then only the legacy input), and shared-v1 keeps `.git` in the memory home. Undefined when the profile cannot be
+ * resolved (an unreadable selector): no remote is then reported rather than a guess.
+ */
+function syncRepository(home: string): string | undefined {
+  try {
+    if (lstatSync(join(home, "shards.json"), { throwIfNoEntry: false })) return join(home, "shard-sync", "objects.git", "config");
+    const { selector } = readSelector(home);
+    if (selector?.kind === "generation") return join(generationDir(home, selector.generation!), ".git", "config");
+    return join(home, ".git", "config");
+  } catch { return undefined; }
+}
+
+/**
+ * Whether the active profile has a sync remote configured, from its repository's own config file: no git process, and never the URL (it can carry a credential).
+ */
+export function remoteConfigured(home: string): boolean {
+  const file = syncRepository(home);
+  if (file === undefined) return false;
+  let fd: number | undefined;
+  try {
+    fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+    const stat = fstatSync(fd);
+    if (!stat.isFile() || stat.size > 65536) return false;
+    const buffer = Buffer.alloc(stat.size), size = readSync(fd, buffer, 0, buffer.length, 0);
+    return /^\s*\[remote "origin"\]/m.test(buffer.subarray(0, size).toString("utf8"));
+  } catch { return false; }   // no such repository
+  finally { if (fd !== undefined) try { closeSync(fd); } catch { /* nothing to close */ } }
+}
+
+/** A stamp from the future (a clock set back, a hand edit) would block the sync until then plus the wait: it counts as no history, and so does a success it claims. */
+export function usableState(state: SyncState, now: number): SyncState {
+  const future = (value: string | null) => value !== null && Date.parse(value) > now;
+  if (future(state.lastAttemptAt)) return { lastAttemptAt: null, lastOkAt: future(state.lastOkAt) ? null : state.lastOkAt, failures: 0 };
+  return future(state.lastOkAt) ? { ...state, lastOkAt: null } : state;
+}
+
+/**
+ * What a finished sync means for the schedule. `ok` clears the backoff. `local` is a refusal that the schedule cannot fix and that no remote ever saw, or a run its person ended (another sync or
+ * mutation holds the lock, no remote is configured, the configured policy declines a note, the remote is public, a usage error): it is not stamped, so it neither
+ * backs the schedule off nor makes a contended run look like a failed one. Everything else is a `failed` transport or git step, and an error this module does not
+ * recognise counts as failed (the safe side: it waits instead of retrying every minute).
+ */
+export type SyncOutcome = "ok" | "failed" | "local";
+const LOCAL = /^(usage:|another (sync|memory mutation|generation|shard sync)|no (generation )?sync remote|the configured privacy policy|the sync remote is public|this (machine's|store's) long-term|a line of this machine's long-term|more than \d+ long-term)/;
+export function syncOutcome(error: string | undefined, cause?: StopCause): SyncOutcome {
+  if (error === undefined) return "ok";
+  // A run its person ended (Ctrl-C, a closing terminal) is not a failure of the remote: the same Ctrl-C reads "fetch failed" or "stopped" depending on which step it hit, so the cause,
+  // not the text, decides. Only the deadline (a remote that did not answer in time) stays a failure to back off from.
+  if (cause === "person") return "local";
+  return LOCAL.test(error.replace(/^amem sync: /, "")) ? "local" : "failed";
+}
+
+/** Records one attempt. A stamp that cannot be written never fails the sync it describes. */
+export function recordSyncAttempt(home: string, ok: boolean, at: number): SyncState {
+  const prior = readSyncState(home), when = new Date(at).toISOString();
+  const next: SyncState = { lastAttemptAt: when, lastOkAt: ok ? when : prior.lastOkAt, failures: ok ? 0 : Math.min(prior.failures + 1, MAX_FAILURES) };
+  const file = join(home, SYNC_STATE_FILE), temp = `${file}.${process.pid}.tmp`;
+  try {
+    mkdirSync(home, { recursive: true, mode: 0o700 });
+    writeFileSync(temp, JSON.stringify(next) + "\n", { mode: 0o600 });
+    renameSync(temp, file);
+  } catch { try { rmSync(temp, { force: true }); } catch { /* best effort */ } }
+  return next;
+}

package/src/memory/sync-screen.ts

@@ -0,0 +1,86 @@
+import { closeSync, constants, fstatSync, openSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { loadAmemConfig } from "./config.ts";
+import { createStandalonePolicy } from "./policy.ts";
+import type { MemoryStore } from "./store.ts";
+
+/**
+ * F4 (agent-mem#40): what leaves this machine is screened by the person's configured policy. Sync pushes this machine's own long-term segment
+ * (`log/<machine>.jsonl`) to a private remote. Before it does, every note in that segment is read through the configured standalone policy (secrets,
+ * personal data, the configured PII patterns and deny paths). One declined entry refuses the whole sync and names the ids (never the text), because a
+ * partial push would silently drop it while it stays in the log.
+ *
+ * What is read is what goes out. A single-log home pushes this machine's own segment, so that file is read. A generation or sharded home publishes rows from the
+ * verified snapshot (this machine's own rows and its bridged legacy rows, wherever they sit on disk, e.g. under `legacy/log/` after a shard migration), so those
+ * rows are read, for the canonical shard and its user overlay alike. The other machines' rows came from the remote and were screened where they were written.
+ * The policy is the machine's configured one AND whatever the caller adds (the CLI adds the repository-narrowed policy), so a repository's `.amem/config` binds sync too.
+ * Reads only (the log is append-only). Bounded: over `MAX_ROWS` rows or `MAX_BYTES` it refuses, since an unscreened row must not pass because the
+ * scan gave up.
+ * ponytail: a person clears a declined entry by removing the offending pattern or editing the policy; sync does not skip a row, and `amem forget` is an
+ * append-only op that leaves the original row in the log. Upgrade: a per-row placement that keeps a declined note local without refusing the sync.
+ */
+const MAX_BYTES = 8 * 1024 * 1024, MAX_ROWS = 20_000;
+
+export type SyncScreen = { ok: true } | { ok: false; reason: string };
+
+/**
+ * What a row says: everything except the fields the library itself generates (ids, hashes, times, the op and enum names, machine and actor ids). Left IN on purpose
+ * are `project`, `peer`, `task`, `about`, `title`, `text`, `milestones` and any field this file has never heard of: an unknown text field must be screened, never skipped.
+ */
+const GENERATED = new Set(["id", "schema", "op", "at", "machine", "actor", "target", "promotedFrom", "supersedes", "expires", "by", "tier", "scope", "kind", "source"]);
+function textOf(row: Record<string, unknown>): Record<string, unknown> {
+  const out: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(row)) if (!GENERATED.has(key)) out[key] = value;
+  return out;
+}
+
+/** A row passes when the machine's configured policy and the caller's extra screen (if any) both accept its text. */
+export type RowScreen = (row: Record<string, unknown>) => boolean;
+export function outboundScreen(home: string, extra?: RowScreen): RowScreen {
+  const { config } = loadAmemConfig(home);
+  const policy = createStandalonePolicy({ denyPaths: config.denyPaths, piiPatterns: config.piiPatterns });
+  return row => { const text = textOf(row); return policy.screenStandalone(text) && (extra?.(text) ?? true); };
+}
+const refusal = (declined: string[]): SyncScreen => declined.length ? { ok: false, reason: `the configured privacy policy declines ${declined.length} long-term entr${declined.length === 1 ? "y" : "ies"} (${declined.slice(0, 8).join(", ")}${declined.length > 8 ? ", ..." : ""}); fix the entry or the policy, then sync again` } : { ok: true };
+const idOf = (row: Record<string, unknown>) => typeof row.id === "string" ? row.id.slice(0, 8) : "(no id)";
+
+/** Screen the rows a generation or sharded store would publish: its snapshot's synced rows that are this machine's own or bridged from its legacy input. */
+export function screenOutboundRows(store: MemoryStore, pass: RowScreen): SyncScreen {
+  const declined: string[] = [];
+  let rows = 0;
+  let snapshot: ReturnType<ReturnType<MemoryStore["prunePort"]>["snapshot"]>;
+  // A bootstrap-only handle has no snapshot: that is a refusal like any unreadable source, never an exception out of sync (which promises a result).
+  try { snapshot = store.prunePort().snapshot(); } catch { return { ok: false, reason: "this store's long-term rows cannot be read, so they were not screened" }; }
+  for (const source of snapshot.rows) {
+    if (source.placement !== "synced" || !(source.own || source.legacy)) continue;
+    if (++rows > MAX_ROWS) return { ok: false, reason: `more than ${MAX_ROWS} long-term rows, so they were not screened` };
+    if (!pass(source.row as unknown as Record<string, unknown>)) declined.push(idOf(source.row as unknown as Record<string, unknown>));
+  }
+  return refusal(declined);
+}
+
+/** Screen one machine's own long-term segment under `dir`. */
+export function screenOwnSegment(dir: string, machineId: string, pass: RowScreen): SyncScreen {
+  const file = join(dir, "log", `${machineId}.jsonl`);
+  let fd: number;
+  try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
+  catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return { ok: true };   // nothing authored here: nothing to push
+    return { ok: false, reason: "this machine's long-term segment cannot be read, so it was not screened" };
+  }
+  try {
+    const stat = fstatSync(fd);
+    if (!stat.isFile() || stat.size > MAX_BYTES) return { ok: false, reason: `this machine's long-term segment is not a plain file within ${MAX_BYTES} bytes, so it was not screened` };
+    const declined: string[] = [];
+    let rows = 0;
+    for (const line of readFileSync(fd, "utf8").split("\n")) {
+      if (!line.trim()) continue;
+      if (++rows > MAX_ROWS) return { ok: false, reason: `more than ${MAX_ROWS} long-term rows, so they were not screened` };
+      let row: Record<string, unknown>;
+      try { row = JSON.parse(line) as Record<string, unknown>; } catch { return { ok: false, reason: "a line of this machine's long-term segment does not parse, so it was not screened" }; }
+      // Every row kind is read, not only notes: a goal's title and milestones sync too, and so does any text an op adds later (see `textOf`).
+      if (!pass(row)) declined.push(idOf(row));
+    }
+    return refusal(declined);
+  } finally { closeSync(fd); }
+}

package/src/memory/sync.ts

@@ -4,6 +4,7 @@ import { memoryGitArgv, memoryGitEnv, processLiveness, processSignature } from "
 import { acquireMutation } from "./mutation.ts";
 import type { MemoryStore } from "./store.ts";
 import { syncGenerationMemory } from "./generation-sync.ts";
+import { outboundScreen, screenOutboundRows, screenOwnSegment, type RowScreen } from "./sync-screen.ts";
 
 /**
  * #258 phase 2: hub memory between a person's machines through a git remote they configure. The memory directory is a
@@ -16,18 +17,47 @@ export interface SyncResult { ok: boolean; pushed: boolean; entries: number; war
 const GITIGNORE = "/*\n!/.gitignore\n!/log/\n/log/*\n!/log/*.jsonl\n";
 const STEP_MS = 60_000;
 
+/**
+ * Git runs in its own process group when the caller passes a stop signal (see `git` below), so that group no longer receives what the terminal sends to amem's own. Every
+ * such group is registered here while it lives, and a process that exits for any reason (a second Ctrl-C, SIGHUP when the terminal closes, SIGQUIT, a throw) signals each one
+ * before it goes: a git left at ppid 1 with no deadline would hold `sync.lock`'s repository while the next sync removes its `index.lock`. The exit handler is synchronous and
+ * installed once, only when the first group is registered, so a process that never runs a deadline git does not get one. SIGKILL of amem itself cannot be handled by anyone.
+ */
+const liveGroups = new Set<number>();
+let exitHandler = false;
+function signalGroups(name: "SIGTERM" | "SIGKILL"): void { for (const pid of liveGroups) { try { process.kill(-pid, name); } catch { /* the group is already gone */ } } }
+function register(pid: number): () => void {
+  if (!exitHandler) { exitHandler = true; process.on("exit", () => signalGroups("SIGKILL")); }
+  liveGroups.add(pid);
+  return () => { liveGroups.delete(pid); };
+}
+/** Test seam: how many deadline-git groups are alive right now, and a way to stop them as the exit handler would. */
+export const syncGroupsForTest = { live: () => liveGroups.size, stopAll: () => signalGroups("SIGKILL") };
+
 async function git(dir: string, args: string[], signal?: AbortSignal, ms = STEP_MS): Promise<{ code: number; out: string; err: string }> {
   if (signal?.aborted) return { code: -1, out: "", err: "stopped" };
-  const p = Bun.spawn(memoryGitArgv(["-C", dir, ...args], { credentials: true }), { stdout: "pipe", stderr: "pipe", stdin: "ignore", env: memoryGitEnv() });
+  // A caller that passes a stop signal (`amem sync --if-due`, a stopping hub) gets git in its own process group, so a stop reaches the transport children git started (ssh, a
+  // credential helper): killing only git's pid left them running. The price is that a terminal Ctrl-C no longer reaches git by itself, so such a caller must turn its own
+  // interrupt into the signal (`stopSignal` in sync-cadence.ts does). A call with no signal keeps the unchanged spawn.
+  const group = signal !== undefined;
+  const p = Bun.spawn(memoryGitArgv(["-C", dir, ...args], { credentials: true }), { stdout: "pipe", stderr: "pipe", stdin: "ignore", env: memoryGitEnv(), ...(group ? { detached: true } : {}) });
+  const send = (name: "SIGTERM" | "SIGKILL") => { if (group) { try { process.kill(-p.pid, name); return; } catch { /* no group left: fall through to the pid */ } } p.kill(name); };
+  const unregister = group ? register(p.pid) : () => {};
   // SIGTERM first, so git removes its lock files; SIGKILL only if it is still there after a grace period.
   let hard: ReturnType<typeof setTimeout> | undefined;
-  const kill = () => { p.kill("SIGTERM"); hard ??= setTimeout(() => p.kill("SIGKILL"), 2_000); };
+  let signalled = false;
+  const kill = () => { signalled = true; send("SIGTERM"); hard ??= setTimeout(() => send("SIGKILL"), 2_000); };
   const timer = setTimeout(kill, ms);
   signal?.addEventListener("abort", kill, { once: true });
   try {
     const [code, out, err] = await Promise.all([p.exited, new Response(p.stdout).text(), new Response(p.stderr).text()]);
     return { code, out: out.trim(), err: err.trim() };
-  } finally { clearTimeout(timer); if (hard) clearTimeout(hard); signal?.removeEventListener("abort", kill); }
+  } finally {
+    // git is gone, but a transport child in its group that ignored SIGTERM (a credential helper, a proxy) may not be, and the timer that would have killed it is cleared below. Once the
+    // group was signalled, SIGKILL whatever is left of it (ESRCH when nothing is) before it is forgotten: it would otherwise live on at ppid 1 holding the repository.
+    if (group && signalled) try { process.kill(-p.pid, "SIGKILL"); } catch { /* the group is already gone */ }
+    unregister(); clearTimeout(timer); if (hard) clearTimeout(hard); signal?.removeEventListener("abort", kill);
+  }
 }
 
 export { git as syncGit };
@@ -37,29 +67,70 @@ export async function generationSyncGit(dir: string, args: string[], signal?: Ab
   if (signal?.aborted) return { code: -1, out: "", err: "stopped" };
   // A fixed child-only umask keeps Git-created metadata private without changing the daemon's process umask.
   // Args are positional shell parameters, never interpolated or evaluated as shell code.
-  const p = Bun.spawn(["/bin/sh", "-c", 'umask 077; exec "$@"', "amem-git", ...memoryGitArgv(["-C", dir, ...args], { credentials: true })], { stdout: "pipe", stderr: "pipe", stdin: "ignore", env: memoryGitEnv(process.env, { repositoryIsolation: true }) });
+  const group = signal !== undefined;   // see `git` above: own process group when the caller has a deadline
+  const p = Bun.spawn(["/bin/sh", "-c", 'umask 077; exec "$@"', "amem-git", ...memoryGitArgv(["-C", dir, ...args], { credentials: true })], { stdout: "pipe", stderr: "pipe", stdin: "ignore", env: memoryGitEnv(process.env, { repositoryIsolation: true }), ...(group ? { detached: true } : {}) });
+  const send = (name: "SIGTERM" | "SIGKILL") => { if (group) { try { process.kill(-p.pid, name); return; } catch { /* no group left: fall through to the pid */ } } p.kill(name); };
+  const unregister = group ? register(p.pid) : () => {};
   // SIGTERM first, so git removes its lock files; SIGKILL only if it is still there after a grace period.
   let hard: ReturnType<typeof setTimeout> | undefined;
-  const kill = () => { p.kill("SIGTERM"); hard ??= setTimeout(() => p.kill("SIGKILL"), 2_000); };
+  let signalled = false;
+  const kill = () => { signalled = true; send("SIGTERM"); hard ??= setTimeout(() => send("SIGKILL"), 2_000); };
   const timer = setTimeout(kill, ms);
   signal?.addEventListener("abort", kill, { once: true });
   try {
     const [code, out, err] = await Promise.all([p.exited, new Response(p.stdout).text(), new Response(p.stderr).text()]);
     return { code, out: out.trim(), err: err.trim() };
-  } finally { clearTimeout(timer); if (hard) clearTimeout(hard); signal?.removeEventListener("abort", kill); }
+  } finally {
+    // git is gone, but a transport child in its group that ignored SIGTERM (a credential helper, a proxy) may not be, and the timer that would have killed it is cleared below. Once the
+    // group was signalled, SIGKILL whatever is left of it (ESRCH when nothing is) before it is forgotten: it would otherwise live on at ppid 1 holding the repository.
+    if (group && signalled) try { process.kill(-p.pid, "SIGKILL"); } catch { /* the group is already gone */ }
+    unregister(); clearTimeout(timer); if (hard) clearTimeout(hard); signal?.removeEventListener("abort", kill);
+  }
 }
 
 
-/** GitHub remotes are asked through `gh`; any other host, or a `gh` that cannot answer, is unknown. */
+/**
+ * The `owner/repo` of a remote URL whose host is EXACTLY `github.com` (case-insensitive) over `https://`, `ssh://git@github.com/` or the scp form
+ * `[git@]github.com:`; anything else is undefined, so it is `unknown` and never asked. An unanchored match let `https://notgithub.com/acme/notes`,
+ * `git@evil-github.com:acme/notes` and `https://git.example.org/mirror/github.com/acme/notes` resolve to `acme/notes` (task #121 review round 1; published 0.6.0
+ * has the unanchored form). `http://` and `git://` are not verifiable as private transports and an explicit port is not github.com's, so both are unknown.
+ */
+export function githubRepoOf(url: string): string | undefined {
+  const text = url.trim();
+  if (!text || /[\s\u0000-\u001f\u007f\\]/.test(text) || text.length > 512) return undefined;
+  let host: string, path: string;
+  const scp = /^(?:([A-Za-z0-9._-]+)@)?([^:/@]+):([^/].*)$/.exec(text); // `[user@]host:path`, never a URL with `://`
+  if (!text.includes("://") && scp) {
+    if (scp[1] !== undefined && scp[1] !== "git") return undefined;
+    host = scp[2]!; path = scp[3]!;
+  } else {
+    let parsed: URL;
+    try { parsed = new URL(text); } catch { return undefined; }
+    if (parsed.protocol !== "https:" && parsed.protocol !== "ssh:") return undefined;
+    if (parsed.port) return undefined;
+    if (parsed.search || parsed.hash) return undefined;
+    if (parsed.protocol === "ssh:" && parsed.username !== "git") return undefined;
+    host = parsed.hostname; path = parsed.pathname.replace(/^\//, "");
+  }
+  if (host.toLowerCase() !== "github.com") return undefined;
+  const match = /^([A-Za-z0-9](?:[A-Za-z0-9-]{0,38}))\/([A-Za-z0-9._-]{1,100}?)(?:\.git)?\/?$/.exec(path);
+  return match && match[2] !== "." && match[2] !== ".." ? `${match[1]}/${match[2]}` : undefined;
+}
+
+/** GitHub remotes are asked through `gh`, pinned to github.com; any other host, or a `gh` that cannot answer, is unknown. */
 export async function remoteVisibility(url: string): Promise<Visibility> {
-  const repo = /github\.com[:/]([^/\s]+\/[^/\s]+?)(?:\.git)?\/?$/.exec(url)?.[1];
+  const repo = githubRepoOf(url);
   if (!repo) return "unknown";
   try {
-    const p = Bun.spawn(["gh", "repo", "view", repo, "--json", "visibility", "--jq", ".visibility"], { stdout: "pipe", stderr: "ignore", stdin: "ignore" });
+    // GH_HOST (and the enterprise token variables) would send `gh repo view` to a different server: pin it, so the answer is about github.com's repository.
+    const env: Record<string, string | undefined> = { ...process.env, GH_HOST: "github.com" };
+    delete env.GH_ENTERPRISE_TOKEN; delete env.GITHUB_ENTERPRISE_TOKEN;
+    const p = Bun.spawn(["gh", "repo", "view", repo, "--json", "visibility", "--jq", ".visibility"], { stdout: "pipe", stderr: "ignore", stdin: "ignore", env: env as Record<string, string> });
     const timer = setTimeout(() => p.kill("SIGKILL"), 15_000);
     const [code, out] = await Promise.all([p.exited, new Response(p.stdout).text()]);
     clearTimeout(timer);
     const v = out.trim().toLowerCase();
+    // ponytail: `internal` (GitHub Enterprise Cloud: visible to the enterprise's members, not the public) counts as private, as `amem sync` always has. A stricter policy would map it to unknown.
     return code === 0 && (v === "private" || v === "internal") ? "private" : code === 0 && v === "public" ? "public" : "unknown";
   } catch { return "unknown"; }
 }
@@ -88,16 +159,25 @@ export interface SyncOptions {
   /** Set (or replace) the remote first: a person's choice. */
   remote?: string;
   visibility?: (url: string) => Promise<Visibility>;
+  /** An extra screen over each outbound row's text, on top of the machine's configured policy (the CLI passes the repository-narrowed policy). */
+  screen?: RowScreen;
   /** A stopping hub aborts: the running git step is killed and the sync ends without touching the store. */
   signal?: AbortSignal;
 }
 
 export async function syncMemory(store: MemoryStore, opts: SyncOptions = {}): Promise<SyncResult> {
+  // F4: before anything is staged or pushed, this machine's own long-term notes pass the person's configured policy (every profile: single log, generation, shards).
+  const pass = outboundScreen(store.profile.home, opts.screen);
+  for (const target of store.openedSyncTargets()) {
+    const screened = target.profile.kind === "generation" ? screenOutboundRows(target, pass) : screenOwnSegment(target.dir, target.machine.id, pass);
+    if (!screened.ok) return { ok: false, pushed: false, entries: 0, error: screened.reason };
+  }
+  if (store.profile.sharded) return (await import("./shard-sync.ts")).syncShardedMemory(store, opts);
   if (store.profile.kind === "generation") return syncGenerationMemory(store, opts);
   const dir = store.dir;
   const fail = (error: string): SyncResult => ({ ok: false, pushed: false, entries: 0, error });
   if (opts.signal?.aborted) return fail("stopped");
-  const noRemote = "no sync remote: a person sets one with ahub memory sync --remote <private git url>";
+  const noRemote = "no sync remote: a person sets one with `amem sync --remote <private git url>`";
   if (!opts.remote && !existsSync(join(dir, ".git"))) return fail(noRemote); // never set up: nothing to do, nothing created
   const release = lock(dir);
   if (!release) return fail("another sync is running on this machine");
@@ -124,7 +204,7 @@ export async function syncMemory(store: MemoryStore, opts: SyncOptions = {}): Pr
     if (!url) return fail(noRemote);
     // Long-term entries hold note text: a public remote would publish them.
     const visibility = await (opts.visibility ?? remoteVisibility)(url);
-    if (visibility === "public") return fail("the sync remote is public; hub memory syncs only to a private remote");
+    if (visibility === "public") return fail("the sync remote is public; memory syncs only to a private remote");
     const warning = visibility === "unknown" ? "the sync remote's visibility is unknown; keep it private" : undefined;
 
     const own = `log/${store.machine.id}.jsonl`;

package/src/memory/usage.ts

@@ -0,0 +1,35 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { readCatalogFresh, type CatalogProject } from "./catalog.ts";
+
+/**
+ * #15 T5 / D3: `amem usage`. Totals and the largest shards, derived from the shards through the catalog scan (never a number of its own). A forgotten
+ * project is out of the default view and the totals; `includeForgotten` shows it. A curated count the catalog could not read is `null`, never zero.
+ */
+export interface UsageTotals { shards: number; captureRows: number; captureBytes: number; curatedEntries: number | null; diskBytes: number }
+/** `curatedSplit`: curated notes are counted per project only once the home is split into shards (`amem migrate --shards`); before that they live in one shared store and no per-project count exists, so it is unavailable, never 0. */
+export interface UsageReport { projects: CatalogProject[]; totals: UsageTotals; largest: CatalogProject[]; curatedSplit: boolean }
+
+export function usageReport(base: string, opts: { limit?: number; includeForgotten?: boolean } = {}): UsageReport {
+  const limit = opts.limit ?? 5;
+  if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) throw new Error("--limit takes a number from 1 to 100");
+  const projects = readCatalogFresh(base).filter(project => opts.includeForgotten === true || project.state !== "forgotten");
+  const curatedSplit = existsSync(join(base, "shards.json"));
+  const curated = curatedSplit && projects.every(project => project.curatedEntries !== null) ? projects.reduce((sum, project) => sum + (project.curatedEntries ?? 0), 0) : null;
+  const totals: UsageTotals = { shards: projects.length, captureRows: projects.reduce((s, p) => s + p.captureRows, 0), captureBytes: projects.reduce((s, p) => s + p.captureBytes, 0),
+    curatedEntries: curated, diskBytes: projects.reduce((s, p) => s + p.diskBytes, 0) };
+  const largest = [...projects].sort((a, b) => b.diskBytes - a.diskBytes || b.captureBytes - a.captureBytes || a.key.localeCompare(b.key)).slice(0, limit);
+  return { projects, totals, largest, curatedSplit };
+}
+
+const MB = 1048576;
+const mb = (n: number): string => `${(n / MB).toFixed(1)} MB`;
+
+/** The non-JSON rendering of `amem usage`. */
+export function usageText(report: UsageReport): string {
+  const t = report.totals;
+  return [
+    `${t.shards} project shard(s): ${t.captureRows} capture rows (${mb(t.captureBytes)}), ${t.curatedEntries === null ? `curated count unavailable${report.curatedSplit ? "" : " (home not split into shards)"}` : `${t.curatedEntries} curated entries`}, ${mb(t.diskBytes)} on disk`,
+    ...(report.largest.length ? ["largest:", ...report.largest.map(p => `  ${mb(p.diskBytes).padStart(10)}  ${p.key}${p.state === "active" ? "" : ` (${p.state})`}  ${p.captureRows} rows`)] : []),
+  ].join("\n");
+}

package/src/memory/use-telemetry.ts

@@ -0,0 +1,51 @@
+import type { CandidateStore, Counter, CounterKey, CounterValues } from "./candidates.ts";
+
+export type MemoryUsePort = Pick<CandidateStore, "recordCounter" | "counters">;
+export interface MemoryUseContext { allowed?: () => boolean; assertCurrent?: () => void; aliases?: readonly string[] }
+export interface MemoryUseSummary { state: "known" | "unknown"; records: number | null; rendered: number | null; delivered: number | null; cited: number | null }
+const unknown = (): MemoryUseSummary => ({ state: "unknown", records: null, rendered: null, delivered: null, cited: null });
+const entry = (id: string): string => { const normalized = id.replace(/^mem:/, ""); if (!/^[a-f0-9]{32}$/.test(normalized)) throw Error("invalid memory use entry"); return normalized; };
+const projectKey = (project: string): string => { if (typeof project !== "string" || !project.trim() || project.length > 400 || project.includes("\0")) throw Error("invalid memory use project"); return project; };
+function guard(context: MemoryUseContext): void { if (context.allowed?.() === false) throw Error("memory use unavailable"); context.assertCurrent?.(); }
+const count = (value: unknown): value is number => Number.isSafeInteger(value) && Number(value) >= 0;
+/** Counts are explicit per-delivery totals, never increments or inferred delivery/citation events. */
+export function recordMemoryUse(port: MemoryUsePort, key: CounterKey, values: CounterValues, context: MemoryUseContext = {}): boolean {
+  try {
+    guard(context);
+    const normalized = { ...key, project: projectKey(key.project), entryId: entry(key.entryId) };
+    if (typeof key.deliveryId !== "string" || !key.deliveryId.trim() || key.deliveryId.length > 200 || key.deliveryId.includes("\0")) return false;
+    for (const metric of [values.rendered, values.delivered, values.cited]) if (metric !== undefined && metric !== null && !count(metric)) return false;
+    guard(context); port.recordCounter(normalized, { ...values }, "public"); guard(context); return true;
+  } catch { return false; }
+}
+/** Unknown individual metrics stay unknown across the complete matching delivery set. */
+export function readMemoryUse(port: MemoryUsePort, project: string, id: string, context: MemoryUseContext = {}): MemoryUseSummary {
+  try {
+    guard(context); const normalized = entry(id), projects = [...new Set([projectKey(project), ...(context.aliases ?? []).map(projectKey)])];
+    const rows = new Map<string, Counter>();
+    for (const scope of projects) {
+      guard(context); const found = port.counters(scope, normalized); guard(context);
+      if (!Array.isArray(found)) return unknown();
+      for (const row of found) {
+        if (!row?.key || !projects.includes(row.key.project) || row.key.entryId !== normalized) continue;
+        if (typeof row.id !== "string" || !row.id) return unknown();
+        const prior = rows.get(row.id);
+        if (prior && JSON.stringify(prior) !== JSON.stringify(row)) return unknown();
+        rows.set(row.id, row);
+      }
+    }
+    const metric = (name: keyof CounterValues): number | null => {
+      if (!rows.size) return null;
+      let total = 0;
+      for (const row of rows.values()) { const value = row[name]; if (!count(value) || !Number.isSafeInteger(total + value)) return null; total += value; }
+      return total;
+    };
+    const result: MemoryUseSummary = { state: "known", records: rows.size, rendered: metric("rendered"), delivered: metric("delivered"), cited: metric("cited") };
+    guard(context); return result;
+  } catch { return unknown(); }
+}
+/** Creating callbacks does nothing. Each explicit callback merges only its named metric using the same delivery key. */
+export function memoryUseCallbacks(port: MemoryUsePort, key: CounterKey, context: MemoryUseContext = {}): { rendered: (count?: number) => boolean; delivered: (count?: number) => boolean; cited: (count?: number) => boolean } {
+  const bound = { ...key };
+  return { rendered: (value = 1) => recordMemoryUse(port, bound, { rendered: value }, context), delivered: (value = 1) => recordMemoryUse(port, bound, { delivered: value }, context), cited: (value = 1) => recordMemoryUse(port, bound, { cited: value }, context) };
+}

package/src/memory/work-lists.ts

@@ -7,6 +7,7 @@ import { canonicalJSON, hashJSON } from "./generation.ts";
 import { mutationDatabase } from "./mutation.ts";
 import { canonicalPath } from "./platform.ts";
 import { looksSecret } from "./screen.ts";
+import { stripPrivateDeep, stripPrivateSpans } from "./private-spans.ts";
 import { ensureShardDescriptor, resolveAlias, shardDir } from "./shard.ts";
 import { machineId } from "./machine.ts";
 
@@ -337,9 +338,12 @@ export class WorkListStore {
     };
   }
 
-  async write(list: string, input: WorkListFields, source?: WorkListSource): Promise<{ id: string; already: boolean }> {
+  async write(rawList: string, input: WorkListFields, source?: WorkListSource): Promise<{ id: string; already: boolean }> {
+    if (!bounded(rawList, 300)) return invalid();
+    // The list NAME is text too: a `<private>` span in it is never stored (a name that is nothing but a span is invalid), and the MCP write and the claude-mem import both land here.
+    const list = stripPrivateSpans(rawList).trim();   // trimmed so "queue <private>x</private>" and "queue" are one list
     if (!bounded(list, 300)) return invalid();
-    const fields = fieldsOf(input), origin = sourceOf(source);
+    const fields = fieldsOf(stripPrivateDeep(input)), origin = sourceOf(source);   // `<private>` spans are never stored
     const at = origin?.at ?? this.locked(() => {
       this.refresh();
       const previous = this.db.query("SELECT max(at) AS at FROM work_ops WHERE machine = ?").get(this.machine) as { at: number | null };
@@ -369,9 +373,11 @@ export class WorkListStore {
     });
   }
 
-  async read(list?: string, includeClosed = false): Promise<WorkListView[]> {
+  async read(rawList?: string, includeClosed = false): Promise<WorkListView[]> {
     this.checkBudget();
-    if (list !== undefined && !bounded(list, 300) || typeof includeClosed !== "boolean") return invalid();
+    if (rawList !== undefined && !bounded(rawList, 300) || typeof includeClosed !== "boolean") return invalid();
+    const list = rawList === undefined ? undefined : stripPrivateSpans(rawList).trim();   // the same name a write stored
+    if (list !== undefined && !bounded(list, 300)) return invalid();
     const rows = this.merged();
     const views = new Map<string, { fields: WorkListFields; items: Map<string, WorkListFields> }>();
     // ponytail: memory scales with the current distinct lists/items/fields, not historical operations. A caller

package/src/session-brief-worker.ts

@@ -6,6 +6,7 @@ import { resolveAlias } from "./memory/aliases.ts";
 import { HOOK_EVENTS, HOOK_INPUT_BYTES } from "./hook-contract.ts";
 import { drainHookCaptures, hookCaptureScope, quarantineHookScope } from "./hook-spool.ts";
 import { createStandalonePolicy } from "./memory/policy.ts";
+import { configuredPolicyOptions } from "./memory/config.ts";
 
 async function readOptions(): Promise<string> {
   const reader = Bun.stdin.stream().getReader(), chunks: Uint8Array[] = []; let bytes = 0;
@@ -45,7 +46,7 @@ if (import.meta.main) {
     const remaining = () => options.deadlineAt - (performance.timeOrigin + performance.now());
     if (remaining() <= 0) stop();
     setTimeout(stop, remaining()).unref();
-    const policy = createStandalonePolicy(options.policy), event = options.event;
+    const policy = createStandalonePolicy(configuredPolicyOptions(options.base!, options.cwd, options.policy)), event = options.event;
     if (!(HOOK_EVENTS as readonly string[]).includes(event)) stop();
     const expectedScope = hookCaptureScope(options.cwd, options.base!);
     let project: string;
@@ -62,7 +63,7 @@ if (import.meta.main) {
       let store: CaptureStore | undefined;
       try {
         const drained = await drainHookCaptures({ ...options, base: options.base!, project, expectedScope, timeRemaining: remaining }, (input, receipt) => {
-          store ??= new CaptureStore({ memoryDir: options.base!, project, busyTimeoutMs: 0 });
+          store ??= new CaptureStore({ memoryDir: options.base!, project, busyTimeoutMs: 0, cwd: options.cwd });
           return store.append(input, { receipt });
         });
         captured = drained.captured.includes(options.current);