agents-can-communicate 0.9.1 → 0.10.1

package/node_modules/@agents-can-communicate/storage-filesystem/src/index-cache.mjs

@@ -0,0 +1,144 @@
+import { AccError, EXIT, TRANSACTION_INDEXES, assertIndexTuple, indexKeysFor }
+  from "@agents-can-communicate/protocol";
+import { assertPublicationDeadline } from "./deadline.mjs";
+import { createIndexIO } from "./index-io.mjs";
+import { createIndexTree } from "./index-tree.mjs";
+import { encodeIndexPage, IndexCacheUnavailable } from "./index-pages.mjs";
+
+const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+const matches = (kind, record, index, tuple) => indexKeysFor(kind, record)
+  .some(key => key.index === index && same(key.tuple, tuple));
+const references = page => page.type === "branch" ? page.children.map(([, hash]) => hash)
+  : page.type === "leaf" ? [...page.entries.map(entry => entry.ids), page.next].filter(Boolean)
+    : [page.next].filter(Boolean);
+
+export function createIndexCache({ paths, root, workspaceId, publishOptions, loadPrimary }) {
+  let diagnostic = [];
+  const note = (code, reason) => { diagnostic = [{ code, reason }]; };
+  const tools = deadlineAt => {
+    const io = createIndexIO({ paths, root, publishOptions: { ...publishOptions, deadlineAt } });
+    const bounded = Object.fromEntries(Object.entries(io).map(([name, operation]) => [name, (...args) => {
+      assertPublicationDeadline(deadlineAt); return operation(...args);
+    }]));
+    return { io: bounded, tree: createIndexTree(bounded) };
+  };
+  async function valid(io, active) {
+    const manifest = await io.readManifest();
+    if (manifest === null) return null;
+    if (active.state !== "idle" || manifest.workspaceId !== workspaceId
+      || manifest.journalGeneration !== active.generation) throw new IndexCacheUnavailable("stale_manifest");
+    // Descendants are validated on their selected route. A root fault cannot
+    // be carried across an unrelated writer by simply changing its epoch.
+    for (const hash of Object.values(manifest.roots)) if (hash !== null) await io.readPage(hash);
+    return manifest;
+  }
+  async function phase(name) {
+    try { await publishOptions.failAt?.(name); }
+    catch (error) { throw new IndexCacheUnavailable("cache_publication_failed", error); }
+  }
+  async function publishManifest(io, manifest) {
+    await phase("after-index-pages");
+    await phase("before-index-manifest");
+    await io.writeManifest(manifest);
+  }
+  async function rebuild(active, deadlineAt, { required = false } = {}) {
+    // Authoritative reads stay outside every cache-error catch.
+    const primaries = await loadPrimary({ deadlineAt });
+    const deltas = Object.fromEntries(Object.keys(TRANSACTION_INDEXES).map(index => [index, []]));
+    for (const entry of primaries.values()) {
+      assertPublicationDeadline(deadlineAt);
+      if (entry.record.workspaceId !== workspaceId) {
+        throw new AccError(EXIT.DATA, "indexed primary belongs to a different workspace");
+      }
+      for (const key of indexKeysFor(entry.kind, entry.record)) deltas[key.index].push({ ...key, id: entry.id, operation: "put" });
+    }
+    const pages = new Map();
+    const tree = createIndexTree({ readPage: async hash => pages.get(hash), writePage: async page => {
+      assertPublicationDeadline(deadlineAt);
+      const { hash } = encodeIndexPage(page); pages.set(hash, page); return hash;
+    } });
+    const roots = {};
+    for (const index of Object.keys(deltas)) roots[index] = await tree.apply(null, deltas[index]);
+    const manifest = { indexVersion: 1, workspaceId, journalGeneration: active.generation, roots };
+    const { io } = tools(deadlineAt);
+    try {
+      // Publish only the final reachable tree, children before parents. The
+      // path copies made during a rebuild need never reach the filesystem.
+      const published = new Set();
+      async function publish(hash) {
+        if (hash === null || published.has(hash)) return;
+        const page = pages.get(hash);
+        for (const child of references(page)) await publish(child);
+        await io.writePage(page); published.add(hash);
+      }
+      for (const hash of Object.values(roots)) await publish(hash);
+      await publishManifest(io, manifest);
+      note("index_rebuilt", "verified_primary");
+    } catch (error) {
+      if (!(error instanceof IndexCacheUnavailable)) throw error;
+      note("index_unavailable", error.reason);
+      if (required) throw error;
+    }
+    return { manifest, tree };
+  }
+  async function lookup(index, tuple, { active, staged, deadlineAt, verify }) {
+    assertIndexTuple(index, tuple);
+    let tree, manifest;
+    try {
+      const selected = tools(deadlineAt);
+      tree = selected.tree;
+      manifest = await valid(selected.io, active);
+      if (manifest === null) throw new IndexCacheUnavailable("missing_manifest");
+      return await select(await tree.lookup(manifest.roots[index], tuple));
+    } catch (error) {
+      if (!(error instanceof IndexCacheUnavailable)) throw error;
+    }
+    ({ manifest, tree } = await rebuild(active, deadlineAt));
+    return select(await tree.lookup(manifest.roots[index], tuple));
+
+    async function select(ids) {
+      const selected = new Set(ids), kind = TRANSACTION_INDEXES[index].kind;
+      for (const entry of staged.values()) {
+        if (entry.kind !== kind) continue;
+        selected.delete(entry.id);
+        if (entry.removed !== true && matches(kind, entry.record, index, tuple)) selected.add(entry.id);
+      }
+      const result = [];
+      // Match eager listJsonFiles ordering, including prefix IDs such as a/a-b.
+      for (const id of [...selected].sort((a, b) => `${a}.json`.localeCompare(`${b}.json`))) {
+        assertPublicationDeadline(deadlineAt);
+        if (await verify(id)) result.push(id);
+      }
+      return result;
+    }
+  }
+  function prepareDeltas(loaded, staged) {
+    const deltas = [];
+    for (const [id, next] of staged) {
+      const previous = loaded.get(id);
+      const oldKeys = previous == null ? [] : indexKeysFor(previous.kind, previous.record);
+      const newKeys = next.removed === true ? [] : indexKeysFor(next.kind, next.record);
+      for (const key of oldKeys) if (!newKeys.some(found => same(found, key))) deltas.push({ ...key, id: next.id, operation: "remove" });
+      for (const key of newKeys) if (!oldKeys.some(found => same(found, key))) deltas.push({ ...key, id: next.id, operation: "put" });
+    }
+    return deltas;
+  }
+  async function commit({ before, after, deltas, deadlineAt }) {
+    try {
+      await phase("after-primary-commit");
+      const { io, tree } = tools(deadlineAt), manifest = await valid(io, before);
+      if (manifest === null) return; // An unrelated writer never constructs a cache.
+      const roots = { ...manifest.roots };
+      for (const index of Object.keys(roots)) {
+        roots[index] = await tree.apply(roots[index], deltas.filter(delta => delta.index === index));
+      }
+      await publishManifest(io, { ...manifest, roots, journalGeneration: after.generation });
+    } catch (error) {
+      if (!(error instanceof IndexCacheUnavailable)
+        && !(error.code === EXIT.CONFLICT && Date.now() >= deadlineAt)) throw error;
+      note("index_unavailable", error instanceof IndexCacheUnavailable ? error.reason : "deadline_expired");
+    }
+  }
+  return Object.freeze({ lookup, prepareDeltas, commit, rebuild,
+    diagnostics: () => diagnostic.map(item => ({ ...item })) });
+}

package/node_modules/@agents-can-communicate/storage-filesystem/src/index-io.mjs

@@ -0,0 +1,65 @@
+import path from "node:path";
+import { assertPortableId } from "@agents-can-communicate/protocol";
+import { publishAtomic } from "./atomic-json.mjs";
+import { withRegularNoFollow } from "./safe-file.mjs";
+import { assertPageHash, decodeIndexPage, encodeIndexPage, INDEX_PAGE_BYTES, IndexCacheUnavailable }
+  from "./index-pages.mjs";
+
+function manifestShape(value) {
+  const keys = object => object && !Array.isArray(object) ? Object.keys(object).sort().join(",") : "";
+  if (keys(value) !== "indexVersion,journalGeneration,roots,workspaceId"
+    || value.indexVersion !== 1 || !/^[0-9]{16}$/.test(value.journalGeneration)
+    || keys(value.roots) !== "messageByClientKey,receiptsByMessage") {
+    throw new IndexCacheUnavailable("invalid_manifest");
+  }
+  assertPortableId(value.workspaceId, "index workspace ID");
+  for (const hash of Object.values(value.roots)) if (hash !== null) assertPageHash(hash);
+  return value;
+}
+export function createIndexIO({ paths, root, publishOptions }) {
+  const directory = path.join(root, "indexes", "v1");
+  const options = { ...publishOptions, root, tmpDir: paths.tmp, stageDir: paths.stage, durability: "none" };
+  const cache = async operation => {
+    try { return await operation(); }
+    catch (error) {
+      if (error instanceof IndexCacheUnavailable) throw error;
+      throw new IndexCacheUnavailable(error.code === "ENOENT" ? "missing_page" : "unsafe_cache_io", error);
+    }
+  };
+  const readBytes = async file => {
+    const bytes = await withRegularNoFollow(file, root, "r", (handle, stat) =>
+      stat.size > INDEX_PAGE_BYTES ? null : handle.readFile());
+    if (bytes === null) throw new IndexCacheUnavailable("page_byte_limit");
+    return bytes;
+  };
+  return Object.freeze({
+    readPage: hash => cache(async () => decodeIndexPage(assertPageHash(hash),
+      await readBytes(path.join(directory, "pages", hash + ".json")))),
+    writePage: page => cache(async () => {
+      const { hash, bytes } = encodeIndexPage(page);
+      const file = path.join(directory, "pages", hash + ".json");
+      let damaged = false;
+      try { damaged = !(await readBytes(file)).equals(bytes); }
+      catch (error) {
+        if (error instanceof IndexCacheUnavailable && error.reason === "page_byte_limit") damaged = true;
+        else if (error.code !== "ENOENT") throw error;
+      }
+      // Valid pages stay immutable. A safely opened but damaged cache file is
+      // replaced with the verified bytes for its existing content address.
+      await publishAtomic(file, bytes, { ...options, replace: damaged });
+      return hash;
+    }),
+    readManifest: () => cache(async () => {
+      let bytes;
+      try { bytes = await readBytes(path.join(directory, "cache.json")); }
+      catch (error) { if (error.code === "ENOENT") return null; throw error; }
+      try { return manifestShape(JSON.parse(bytes.toString("utf8"))); }
+      catch (error) { throw new IndexCacheUnavailable("invalid_manifest", error); }
+    }),
+    writeManifest: manifest => cache(async () => {
+      manifestShape(manifest);
+      await publishAtomic(path.join(directory, "cache.json"),
+        Buffer.from(JSON.stringify(manifest) + "\n"), { ...options, replace: true });
+    }),
+  });
+}

package/node_modules/@agents-can-communicate/storage-filesystem/src/index-pages.mjs

@@ -0,0 +1,85 @@
+import { createHash } from "node:crypto";
+import { assertPortableId } from "@agents-can-communicate/protocol";
+
+export const INDEX_PAGE_BYTES = 65_536;
+export const INDEX_PAGE_ENTRIES = 32;
+const HASH = /^[a-f0-9]{64}$/;
+export class IndexCacheUnavailable extends Error {
+  constructor(reason, cause) {
+    super(`transaction index unavailable: ${reason}`, { cause });
+    this.name = "IndexCacheUnavailable";
+    this.reason = reason;
+  }
+}
+const refuse = reason => { throw new IndexCacheUnavailable(reason); };
+export function assertPageHash(hash) {
+  if (typeof hash !== "string" || !HASH.test(hash)) refuse("invalid_reference");
+  return hash;
+}
+function closed(value, keys) {
+  if (value === null || typeof value !== "object" || Array.isArray(value)
+    || Object.keys(value).sort().join(",") !== [...keys].sort().join(",")) refuse("invalid_page_shape");
+}
+function unique(values) {
+  if (new Set(values).size !== values.length) refuse("duplicate_page_entry");
+}
+export function canonicalTuple(tuple) {
+  if (!Array.isArray(tuple) || ![2, 3].includes(tuple.length)) refuse("invalid_tuple");
+  try { tuple.forEach(id => assertPortableId(id, "index tuple member")); }
+  catch (error) { throw new IndexCacheUnavailable("invalid_tuple", error); }
+  return JSON.stringify(tuple);
+}
+function limited(values, limit = INDEX_PAGE_ENTRIES) {
+  if (!Array.isArray(values) || values.length === 0 || values.length > limit) refuse("page_entry_limit");
+}
+export function validateIndexPage(page) {
+  if (page?.type === "branch") {
+    closed(page, ["type", "children"]); limited(page.children, 16);
+    const routes = [];
+    for (const child of page.children) {
+      if (!Array.isArray(child) || child.length !== 2 || !/^[a-f0-9]$/.test(child[0])) refuse("invalid_child");
+      assertPageHash(child[1]); routes.push(child[0]);
+    }
+    unique(routes);
+    if (routes.join("") !== [...routes].sort().join("")) refuse("noncanonical_routes");
+  } else if (page?.type === "leaf") {
+    closed(page, ["type", "entries", "next"]); limited(page.entries);
+    if (page.next !== null) assertPageHash(page.next);
+    const keys = [];
+    for (const entry of page.entries) {
+      closed(entry, ["digest", "tuple", "ids"]);
+      assertPageHash(entry.digest); assertPageHash(entry.ids); keys.push(canonicalTuple(entry.tuple));
+    }
+    unique(keys);
+    if (keys.some((key, i) => i > 0 && keys[i - 1].localeCompare(key) >= 0)) refuse("noncanonical_keys");
+  } else if (page?.type === "group") {
+    closed(page, ["type", "ids", "next"]); limited(page.ids);
+    if (page.next !== null) assertPageHash(page.next);
+    try { page.ids.forEach(id => assertPortableId(id, "index record ID")); }
+    catch (error) { throw new IndexCacheUnavailable("invalid_record_id", error); }
+    unique(page.ids);
+    if (page.ids.some((id, i) => i > 0 && page.ids[i - 1].localeCompare(id) >= 0)) refuse("noncanonical_ids");
+  } else refuse("unknown_page_type");
+  return page;
+}
+export function encodeIndexPage(page) {
+  validateIndexPage(page);
+  // Fixed field order makes insertion order irrelevant to content addressing.
+  const canonical = page.type === "branch" ? { type: page.type, children: page.children }
+    : page.type === "group" ? { type: page.type, ids: page.ids, next: page.next }
+      : { type: page.type, entries: page.entries.map(({ digest, tuple, ids }) => ({ digest, tuple, ids })), next: page.next };
+  const bytes = Buffer.from(JSON.stringify(canonical) + "\n");
+  if (bytes.length > INDEX_PAGE_BYTES) refuse("page_byte_limit");
+  return { bytes, hash: createHash("sha256").update(bytes).digest("hex") };
+}
+export function decodeIndexPage(hash, bytes) {
+  assertPageHash(hash);
+  if (!Buffer.isBuffer(bytes) || bytes.length > INDEX_PAGE_BYTES) refuse("page_byte_limit");
+  let page;
+  try { page = JSON.parse(bytes.toString("utf8")); }
+  catch (error) { throw new IndexCacheUnavailable("invalid_page_json", error); }
+  const encoded = encodeIndexPage(page);
+  if (!encoded.bytes.equals(bytes)) refuse("noncanonical_bytes");
+  if (encoded.hash !== hash) refuse("page_hash_mismatch");
+  return page;
+}

package/node_modules/@agents-can-communicate/storage-filesystem/src/index-reclaim-progress.mjs

@@ -0,0 +1,35 @@
+import { createHash } from "node:crypto";
+import path from "node:path";
+import { publishAtomic } from "./atomic-json.mjs";
+import { withRegularNoFollow } from "./safe-file.mjs";
+
+const hash = value => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
+const digest = bytes => createHash("sha256").update(bytes).digest("hex");
+const file = root => path.join(root, "indexes/v1/reclaim.json");
+const fresh = () => ({ version: 1, pending: [], visited: [], cursor: null, done: false });
+const valid = state => state && Object.keys(state).sort().join(",") === "cursor,done,pending,version,visited"
+  && state.version === 1 && typeof state.done === "boolean"
+  && [state.pending, state.visited].every(items => Array.isArray(items) && items.every(hash))
+  && (state.cursor === null || /^[a-f0-9]{64}\.json$/.test(state.cursor))
+  && (!state.done || state.pending.length === 0);
+
+// This is optional, checksummed bookkeeping, never primary authority. Atomic
+// replacement means process death leaves the previous complete checkpoint;
+// lost/torn bytes restart marking before any further deletion. No fsync is due.
+export async function readReclaimProgress(root) {
+  try {
+    const bytes = await withRegularNoFollow(file(root), root, "r", (handle, stat) =>
+      stat.size > 64 * 1024 * 1024 ? null : handle.readFile());
+    if (bytes === null) return fresh();
+    const value = JSON.parse(bytes.toString("utf8"));
+    if (!valid(value.state) || value.sha256 !== digest(JSON.stringify(value.state))) return fresh();
+    return value.state.done ? fresh() : value.state;
+  } catch { return fresh(); }
+}
+
+export async function writeReclaimProgress(paths, root, state, deadlineAt) {
+  const body = JSON.stringify(state);
+  await publishAtomic(file(root), Buffer.from(JSON.stringify({ state, sha256: digest(body) }) + "\n"), {
+    root, tmpDir: paths.tmp, stageDir: paths.stage, replace: true, durability: "none", deadlineAt,
+  });
+}

package/node_modules/@agents-can-communicate/storage-filesystem/src/index-reclaim.mjs

@@ -0,0 +1,142 @@
+import path from "node:path";
+import { idleJournal, readActiveJournal } from "./active-journal.mjs";
+import { listDirectoryEntries, nameCommittedLater } from "./atomic-json.mjs";
+import { condemn, detachDoomed, discard, expired } from "./doomed-directory.mjs";
+import { createIndexIO } from "./index-io.mjs";
+import { IndexCacheUnavailable } from "./index-pages.mjs";
+import { readReclaimProgress, writeReclaimProgress } from "./index-reclaim-progress.mjs";
+import { readStoreIdentity } from "./identity.mjs";
+import { journalEntry, writeJournalEntry } from "./journal.mjs";
+import { syncPruneDirectories } from "./prune-durability.mjs";
+import { assertManagedDirectory } from "./safe-directory.mjs";
+
+const children = page => page.type === "branch" ? page.children.map(([, hash]) => hash)
+  : page.type === "leaf" ? [...page.entries.map(entry => entry.ids), page.next].filter(Boolean)
+    : [page.next].filter(Boolean);
+
+// The caller holds the writer mutex. Persist both mark and sweep progress: a
+// live graph or a directory larger than one pass must eventually finish. Roots
+// added by intervening writers are marked before sweeping resumes. Previously
+// marked roots stay in the conservative live superset until the next cycle.
+export async function reclaimIndexPages(paths, { root, roots, limit = 512, deadlineAt } = {}) {
+  let spent = 0, reclaimed = 0;
+  const result = (remaining, deferred = false) => ({ reclaimed, remaining, spent, deferred });
+  if (expired(deadlineAt)) return result(true, true);
+  const directory = path.join(root, "indexes", "v1", "pages");
+  try { await assertManagedDirectory(root, directory); }
+  catch (error) { return result(error.code !== "ENOENT", error.code !== "ENOENT"); }
+  if (limit < 4) return result(true);
+  const io = createIndexIO({ paths, root, publishOptions: {} });
+  if (roots === undefined) {
+    let manifest;
+    try { manifest = await io.readManifest(); }
+    catch (error) { if (error instanceof IndexCacheUnavailable) return result(true, true); throw error; }
+    if (manifest === null) return result(true, true);
+    // Primary authority faults stay outside cache-error handling.
+    const active = await readActiveJournal(paths, root), identity = await readStoreIdentity(paths);
+    if (active.state !== "idle" || manifest.journalGeneration !== active.generation
+      || manifest.workspaceId !== identity?.workspaceId) return result(true, true);
+    roots = manifest.roots;
+  }
+  const progress = await readReclaimProgress(root);
+  spent += 1;
+  // Reserve the checkpoint publication within the same budget as page work.
+  const budget = limit - 1;
+  const reachable = new Set(progress.visited), pending = progress.pending;
+  for (const hash of Object.values(roots).filter(Boolean)) {
+    if (!reachable.has(hash) && !pending.includes(hash)) pending.push(hash);
+  }
+  const checkpoint = async (remaining) => {
+    progress.visited = [...reachable];
+    progress.done = !remaining;
+    try {
+      await writeReclaimProgress(paths, root, progress, deadlineAt);
+      spent += 1;
+      return result(remaining);
+    } catch { return result(true, true); }
+  };
+  try {
+    while (pending.length > 0) {
+      const hash = pending.pop();
+      if (reachable.has(hash)) continue;
+      if (spent >= budget || expired(deadlineAt)) {
+        pending.push(hash);
+        return checkpoint(true);
+      }
+      spent += 1;
+      pending.push(...children(await io.readPage(hash)));
+      reachable.add(hash);
+    }
+  } catch (error) {
+    if (error instanceof IndexCacheUnavailable) {
+      // A former root can disappear while the cache rebuilds. Do not let an
+      // unreadable old stack poison every later pass; restart from authority.
+      pending.length = 0;
+      reachable.clear();
+      progress.cursor = null;
+      await checkpoint(false);
+      return result(true, true);
+    }
+    throw error;
+  }
+  let entries;
+  try { entries = await listDirectoryEntries(directory, { root }); }
+  catch { return result(true, true); } // Cache paths alone are optional maintenance.
+  let doomed, reserved = 0, remaining = false;
+  entries.sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0);
+  for (const entry of entries) {
+    if (!/^[a-f0-9]{64}\.json$/.test(entry.name) || (progress.cursor !== null && entry.name <= progress.cursor)) continue;
+    if (spent + reserved >= budget || expired(deadlineAt)) { remaining = true; break; }
+    spent += 1;
+    if (!entry.isFile() || reachable.has(entry.name.slice(0, -5))) { progress.cursor = entry.name; continue; }
+    if (spent + reserved + 2 > budget) { remaining = true; break; }
+    doomed ??= await detachDoomed(root);
+    await assertManagedDirectory(root, directory);
+    await condemn(path.join(directory, entry.name), doomed);
+    spent += 1; reserved += 1;
+    progress.cursor = entry.name;
+  }
+  if (doomed !== undefined) {
+    const removed = await discard(doomed, root, budget - spent, deadlineAt);
+    reclaimed += removed.spent; spent += removed.spent;
+    remaining ||= !removed.drained;
+  }
+  return checkpoint(remaining);
+}
+
+async function discardCache(root, collect) {
+  const directory = path.join(root, "indexes", "v1");
+  // Only a validated cache directory may move. Damage/unsafe cache paths stay
+  // untouched, and the changed durable epoch already makes their roots stale.
+  try {
+    await assertManagedDirectory(root, directory);
+    const doomed = await detachDoomed(root);
+    await condemn(directory, doomed);
+    collect(directory, doomed);
+  } catch { /* Cache loss or refusal cannot fail a completed primary prune. */ }
+}
+
+// Existing empty-publication journals fence physical retirement without
+// introducing a new authoritative marker or a per-message flush.
+export async function withIndexedPrune(paths, options, operation) {
+  const { ids, firstSequence, root, deadlineAt, ...publish } = options;
+  const entry = journalEntry(ids.next("transaction"), firstSequence, [], options.clock.now());
+  await writeJournalEntry(paths, { ...publish, root, deadlineAt }, entry);
+  const directories = new Set([root]);
+  const collect = (file, doomed) => { directories.add(path.dirname(file)); directories.add(doomed); };
+  try {
+    await options.failAt?.("after-indexed-prune-activated");
+    await discardCache(root, collect);
+    return await operation(async (file, doomed) => {
+      collect(file, doomed);
+      await options.failAt?.("after-indexed-prune-move");
+    });
+  } finally {
+    // A decided/applied prefix must finish its durability even after its
+    // caller's deadline. POSIX flushes names; the final Windows journal flush
+    // commits every earlier NTFS metadata rename.
+    await syncPruneDirectories(root, directories, options);
+    await options.failAt?.("before-indexed-prune-idle");
+    await idleJournal(paths, { ...publish, root, durability: nameCommittedLater(options.platform) }, entry.transactionId);
+  }
+}

package/node_modules/@agents-can-communicate/storage-filesystem/src/index-tree.mjs

@@ -0,0 +1,142 @@
+import { createHash } from "node:crypto";
+import { assertPortableId } from "@agents-can-communicate/protocol";
+import { assertPageHash, canonicalTuple, IndexCacheUnavailable } from "./index-pages.mjs";
+
+const compare = (a, b) => a.localeCompare(b);
+const keyOf = entry => canonicalTuple(entry.tuple);
+const digest = tuple => createHash("sha256").update(canonicalTuple(tuple)).digest("hex");
+const fault = reason => { throw new IndexCacheUnavailable(reason); };
+export function createIndexTree({ readPage, writePage, digestTuple = digest }) {
+  async function read(hash, seen) {
+    assertPageHash(hash);
+    if (seen.has(hash)) fault("page_cycle");
+    seen.add(hash);
+    return readPage(hash);
+  }
+  async function groupLookup(root) {
+    const ids = [], seen = new Set();
+    while (root !== null) {
+      const page = await read(root, seen);
+      if (page.type !== "group") fault("invalid_group_reference");
+      for (const id of page.ids) {
+        if (ids.length && compare(ids.at(-1), id) >= 0) fault("duplicate_or_unordered_group");
+        ids.push(id);
+      }
+      root = page.next;
+    }
+    return ids;
+  }
+  async function groupUpdate(root, id, operation, seen = new Set()) {
+    if (root === null) return operation === "remove" ? null : writePage({ type: "group", ids: [id], next: null });
+    const page = await read(root, seen);
+    if (page.type !== "group") fault("invalid_group_reference");
+    const position = page.ids.findIndex(found => compare(found, id) >= 0);
+    if (position === -1 && page.next !== null) {
+      const next = await groupUpdate(page.next, id, operation, seen);
+      return next === page.next ? root : writePage({ ...page, next });
+    }
+    const ids = [...page.ids], at = position === -1 ? ids.length : position;
+    if (ids[at] === id) {
+      if (operation === "put") return root;
+      ids.splice(at, 1);
+    } else {
+      if (operation === "remove") return root;
+      ids.splice(at, 0, id);
+    }
+    if (ids.length === 0) return page.next;
+    if (ids.length <= 32) return writePage({ ...page, ids });
+    const next = await writePage({ type: "group", ids: ids.slice(16), next: page.next });
+    return writePage({ type: "group", ids: ids.slice(0, 16), next });
+  }
+  function checkNode(page, depth, prefix) {
+    if (page.type === "branch") {
+      if (depth >= 64) fault("radix_depth_exceeded");
+    } else if (page.type === "leaf") {
+      if (page.next !== null && depth !== 64) fault("unexpected_collision_page");
+      for (const entry of page.entries) {
+        if (!entry.digest.startsWith(prefix) || entry.digest !== digestTuple(entry.tuple)) fault("misrouted_tuple");
+      }
+    } else fault("invalid_tree_reference");
+  }
+  async function lookup(root, tuple) {
+    const key = canonicalTuple(tuple), hash = assertPageHash(digestTuple(tuple)), seen = new Set();
+    let depth = 0, prefix = "", idsRoot = null;
+    while (root !== null) {
+      const page = await read(root, seen); checkNode(page, depth, prefix);
+      if (page.type === "branch") {
+        const route = hash[depth++]; prefix += route;
+        root = page.children.find(([digit]) => digit === route)?.[1] ?? null;
+      } else {
+        const found = page.entries.find(entry => keyOf(entry) === key);
+        if (found) {
+          if (idsRoot !== null) fault("duplicate_collision_key");
+          idsRoot = found.ids;
+        }
+        root = page.next;
+      }
+    }
+    return idsRoot === null ? [] : groupLookup(idsRoot);
+  }
+  const leaf = (entries, next = null) => writePage({ type: "leaf",
+    entries: [...entries].sort((a, b) => compare(keyOf(a), keyOf(b))), next });
+  async function split(entries, depth) {
+    if (entries.length <= 32) return leaf(entries);
+    if (depth === 64) {
+      let next = null;
+      for (let end = entries.length; end > 0; end -= 32) {
+        next = await leaf(entries.slice(Math.max(0, end - 32), end), next);
+      }
+      return next;
+    }
+    const routes = new Map();
+    for (const entry of entries) {
+      const route = entry.digest[depth];
+      if (!routes.has(route)) routes.set(route, []);
+      routes.get(route).push(entry);
+    }
+    const children = [];
+    for (const route of [...routes.keys()].sort()) children.push([route, await split(routes.get(route), depth + 1)]);
+    return writePage({ type: "branch", children });
+  }
+  async function update(root, tuple, id, operation, depth = 0, prefix = "", seen = new Set()) {
+    const hash = assertPageHash(digestTuple(tuple)), key = canonicalTuple(tuple);
+    if (root === null) {
+      if (operation === "remove") return null;
+      return leaf([{ digest: hash, tuple, ids: await groupUpdate(null, id, operation) }]);
+    }
+    const page = await read(root, seen); checkNode(page, depth, prefix);
+    if (page.type === "branch") {
+      const route = hash[depth], old = page.children.find(([digit]) => digit === route)?.[1] ?? null;
+      const next = await update(old, tuple, id, operation, depth + 1, prefix + route, seen);
+      if (old === next) return root;
+      const children = page.children.filter(([digit]) => digit !== route);
+      if (next !== null) children.push([route, next]);
+      children.sort(([a], [b]) => compare(a, b));
+      return children.length === 0 ? null : writePage({ type: "branch", children });
+    }
+    const entries = [...page.entries], at = entries.findIndex(entry => keyOf(entry) === key);
+    if (at !== -1) {
+      const ids = await groupUpdate(entries[at].ids, id, operation);
+      if (ids === entries[at].ids) return root;
+      if (ids === null) entries.splice(at, 1);
+      else entries[at] = { ...entries[at], ids };
+      return entries.length === 0 ? page.next : leaf(entries, page.next);
+    }
+    if (page.next !== null) {
+      const next = await update(page.next, tuple, id, operation, depth, prefix, seen);
+      return next === page.next ? root : leaf(entries, next);
+    }
+    if (operation === "remove") return root;
+    entries.push({ digest: hash, tuple, ids: await groupUpdate(null, id, operation) });
+    return split(entries, depth);
+  }
+  async function apply(root, deltas) {
+    for (const { tuple, id, operation } of deltas) {
+      assertPortableId(id, "index record ID");
+      if (!["put", "remove"].includes(operation)) fault("invalid_delta");
+      root = await update(root, tuple, id, operation);
+    }
+    return root;
+  }
+  return Object.freeze({ lookup, apply });
+}

package/node_modules/@agents-can-communicate/storage-filesystem/src/index.mjs

@@ -5,3 +5,4 @@ export { readStoreIdentity, requireStoreIdentity, STORE_VERSION } from "./identi
 export { withWriterMutex } from "./writer-mutex.mjs";
 export { readSessionRecord } from "./session-read.mjs";
 export { openNoFollow } from "./portable-fs.mjs";
+export { migrateFilesystemStore } from "./store-migration.mjs";

package/node_modules/@agents-can-communicate/storage-filesystem/src/journal.mjs

@@ -5,6 +5,7 @@ import { AccError, EXIT, assertPortableId } from "@agents-can-communicate/protoc
 import { activateJournal, idleJournal, readActiveJournal } from "./active-journal.mjs";
 import { encode, nameCommittedLater, publishAtomic, readJsonIfPresent, retainFile }
   from "./atomic-json.mjs";
+import { recoverPruneDurability } from "./prune-durability.mjs";
 
 export const JOURNAL_VERSION = 2;
 
@@ -133,6 +134,7 @@ export async function readJournalCeiling(paths, root) {
 // twice changes nothing and a genuine conflict is never papered over.
 export async function rollForward(paths, options, entry) {
   const published = [];
+  if (entry.publications.length === 0) await recoverPruneDurability(paths, options);
   for (const publication of entry.publications) {
     const destination = publicationDestination(options.root, publication.path);
     if (publication.retainedPath !== null) {