agents-can-communicate 0.6.3 → 0.7.0

package/node_modules/@agents-can-communicate/storage-filesystem/src/doomed-directory.mjs

@@ -0,0 +1,98 @@
+import { randomUUID } from "node:crypto";
+import { open, rename, rm, rmdir } from "node:fs/promises";
+import path from "node:path";
+
+import { listDirectoryEntries } from "./atomic-json.mjs";
+import { assertManagedDirectory, ensureManagedDirectory } from "./safe-directory.mjs";
+
+// The store never unlinks a live name: Node exposes unlink only by pathname, so
+// a checked parent can be replaced between the check and the call. What it does
+// instead is move a doomed entry into a directory this process named with a
+// value no other process knows, and then remove that directory whole. Every
+// reclaimer in the store shares the primitive, so the rule is stated once.
+//
+// The prefix still reads "stage" because a store written by 0.6.3 leaves
+// directories under this exact name when a pass is interrupted, and the resume
+// below finds them by prefix. Renaming it would orphan them.
+const DOOMED = "stage.sweeping-";
+
+// Not assertPublicationDeadline: that one throws, and a maintenance pass on the
+// path that opens the store must stop rather than fail the open. What is left
+// is reported as remaining, and the next pass adopts it.
+export const expired = deadlineAt => deadlineAt !== undefined && Date.now() >= deadlineAt;
+
+export async function syncDirectory(directory) {
+  const handle = await open(directory, "r");
+  try {
+    await handle.sync();
+  } finally {
+    await handle.close();
+  }
+}
+
+export async function doomedDirectories(root) {
+  return (await listDirectoryEntries(root, { root }))
+    .filter(entry => entry.isDirectory() && entry.name.startsWith(DOOMED))
+    .map(entry => path.join(root, entry.name))
+    .sort((left, right) => left.localeCompare(right));
+}
+
+export function doomedName(root) {
+  return path.join(root, `${DOOMED}${randomUUID()}`);
+}
+
+export async function detachDoomed(root) {
+  const directory = doomedName(root);
+  await ensureManagedDirectory(root, directory);
+  return directory;
+}
+
+/**
+ * Move one doomed entry out of the store's live names.
+ *
+ * The destination name is unique because the sources are not: two records of
+ * one kind each keep a marker named for its own sequence, and a journal entry
+ * and a stage file can collide on nothing at all. The doomed directory is
+ * discarded whole, so the names inside it carry no meaning.
+ */
+export async function condemn(filePath, doomed) {
+  await rename(filePath, path.join(doomed, `${randomUUID()}.doomed`));
+}
+
+/**
+ * Remove a doomed directory, one entry at a time, within a budget.
+ *
+ * @returns {Promise<{ spent: number, drained: boolean }>}
+ */
+export async function discard(directory, root, budget, deadlineAt) {
+  let spent = 0;
+  await assertManagedDirectory(root, directory);
+  for (const entry of await listDirectoryEntries(directory, { root })) {
+    if (spent >= budget || expired(deadlineAt)) return { spent, drained: false };
+    await rm(path.join(directory, entry.name), { recursive: true, force: true });
+    spent += 1;
+  }
+  await rmdir(directory).catch(error => {
+    if (error.code !== "ENOENT" && error.code !== "ENOTEMPTY") throw error;
+  });
+  return { spent, drained: true };
+}
+
+/**
+ * Finish what an interrupted pass left behind, within a budget.
+ *
+ * Every pass runs this first, and for a second reason: a pass that stops here
+ * never detaches a further directory, so an exhausted budget cannot make them
+ * accumulate.
+ *
+ * @returns {Promise<{ spent: number, drained: boolean }>}
+ */
+export async function discardLeftovers(root, budget, deadlineAt) {
+  let spent = 0;
+  for (const directory of await doomedDirectories(root)) {
+    const removed = await discard(directory, root, budget - spent, deadlineAt);
+    spent += removed.spent;
+    if (!removed.drained) return { spent, drained: false };
+  }
+  return { spent, drained: true };
+}

package/node_modules/@agents-can-communicate/storage-filesystem/src/event-floor.mjs

@@ -0,0 +1,52 @@
+import path from "node:path";
+
+import { AccError, EXIT } from "@agents-can-communicate/protocol";
+
+import { encode, publishAtomic, readJsonIfPresent } from "./atomic-json.mjs";
+
+const SEQUENCE = /^[0-9]{16}$/;
+
+/**
+ * The highest event sequence this store has trimmed away.
+ *
+ * Trimming the oldest events is safe on its own, because `nextSequence` reads
+ * the newest file. An emptied directory is what needs a floor: without one the
+ * sequence would restart at 1 and collide with a cursor a peer still holds,
+ * and a reader whose cursor precedes the boundary would be served a short page
+ * with no way to tell it was short.
+ */
+const floorPath = paths => path.join(paths.locks, "retention.json");
+
+export function assertSequence(value, field) {
+  if (typeof value !== "string" || !SEQUENCE.test(value)) {
+    throw new AccError(EXIT.USAGE,
+      `${field} is the 16-digit sequence a previous sync returned`, { [field]: value });
+  }
+  return value;
+}
+
+// An unreadable floor is treated as none. The alternative - refusing to serve
+// events because the bookkeeping about their removal is damaged - would take a
+// workspace offline over a file that only ever adds information.
+export async function readEventFloor(paths, root) {
+  const found = await readJsonIfPresent(floorPath(paths), root).catch(() => null);
+  const value = found?.value?.trimmedThrough;
+  return typeof value === "string" && SEQUENCE.test(value) ? value : null;
+}
+
+/**
+ * Raise the floor, never lower it.
+ *
+ * Written before a single event file moves. A floor ahead of the trim only
+ * reports a boundary for events still present, which costs a reader nothing;
+ * a trim ahead of the floor is a silent gap, which is the failure this exists
+ * to prevent.
+ */
+export async function raiseEventFloor(paths, options, trimmedThrough) {
+  assertSequence(trimmedThrough, "trimmedThrough");
+  const current = await readEventFloor(paths, options.root);
+  if (current !== null && current >= trimmedThrough) return current;
+  await publishAtomic(floorPath(paths), encode({ trimmedThrough }),
+    { ...options, replace: true });
+  return trimmedThrough;
+}

package/node_modules/@agents-can-communicate/storage-filesystem/src/reclaim.mjs

@@ -0,0 +1,272 @@
+import path from "node:path";
+
+import { AccError, EXIT } from "@agents-can-communicate/protocol";
+
+import { readActiveJournal } from "./active-journal.mjs";
+import { listDirectoryEntries, readJsonIfPresent } from "./atomic-json.mjs";
+import { condemn, detachDoomed, discard, discardLeftovers, expired } from "./doomed-directory.mjs";
+import { assertSequence, raiseEventFloor, readEventFloor } from "./event-floor.mjs";
+import { statePath } from "./record-id.mjs";
+
+// Shared with the stage sweep that calls this, so one pass cannot spend two
+// budgets. Exported for the caller that wants to reclaim without a bound.
+export const RECLAIM_BUDGET = 512;
+
+const jsonNames = async (directory, root) =>
+  (await listDirectoryEntries(directory, { root }))
+    .filter(entry => entry.isFile() && entry.name.endsWith(".json"))
+    .map(entry => entry.name);
+
+/**
+ * Which transaction the store is in the middle of, or null when it cannot tell.
+ *
+ * `readActiveJournal` throws on a store with no authority record. Not knowing
+ * which entry is active is not a licence to guess, so that answer suppresses
+ * journal reclamation rather than failing the store open it runs inside.
+ *
+ * It suppresses it without holding the pass open. A store missing its authority
+ * is one `openFilesystemStore` would have initialised, so it is broken rather
+ * than busy, and doctor is what reports a broken store. Reporting work as
+ * remaining would re-run this pass on every open forever, to keep finding the
+ * same answer about files that are inert either way.
+ */
+async function activeTransaction(paths, root) {
+  try {
+    const active = await readActiveJournal(paths, root);
+    return { known: true, transactionId: active.state === "open" ? active.transactionId : null };
+  } catch {
+    return { known: false, transactionId: null };
+  }
+}
+
+/**
+ * Condemn every journal entry that has been retired, with its completion marker.
+ *
+ * `retireJournalEntry` calls `retainFile`, which by construction never unlinks,
+ * so a completed entry was kept forever; `completeJournal` then wrote a marker
+ * that nothing reads. Both halves go together: a marker outliving its entry
+ * records the completion of a file nothing can find.
+ *
+ * The marker directory is listed once rather than read per entry. A workspace
+ * in daily use reaches thousands of retired transactions, and one open each
+ * would be the cost this pass exists to remove.
+ */
+async function condemnRetiredJournals(paths, root, doomed, budget, deadlineAt) {
+  let spent = 0;
+  const active = await activeTransaction(paths, root);
+  if (!active.known) return { spent, drained: true };
+
+  const markerDirectory = path.join(paths.retained, "journal");
+  const completed = new Set(await jsonNames(markerDirectory, root));
+  for (const name of await jsonNames(paths.journal, root)) {
+    if (spent + 2 > budget || expired(deadlineAt)) return { spent, drained: false };
+    if (!completed.has(name)) continue;
+    // The marker is written before the pointer goes idle, so a crash between
+    // the two leaves a completed marker on the entry recovery still owns.
+    if (path.basename(name, ".json") === active.transactionId) continue;
+    await condemn(path.join(paths.journal, name), doomed);
+    await condemn(path.join(markerDirectory, name), doomed);
+    spent += 2;
+  }
+  return { spent, drained: true };
+}
+
+// Ordered as numbers, not as strings: a sequence can outgrow its padding, which
+// is the same reason latestEphemeralMarker sorts this way. A name that is not a
+// sequence is left alone rather than guessed at.
+function supersededNames(names) {
+  const sequenced = names.map(name => {
+    const digits = path.basename(name, ".json");
+    return /^\d+$/.test(digits) ? { name, order: BigInt(digits) } : null;
+  }).filter(Boolean).sort((left, right) => (left.order < right.order ? 1 : -1));
+  return sequenced.slice(1).map(entry => entry.name);
+}
+
+/**
+ * Condemn every ephemeral retention marker except the newest of each record.
+ *
+ * `latestEphemeralMarker` reads only the newest and lists the rest to find it,
+ * so an older marker is pure listing cost on a path that runs inside hooks.
+ */
+async function condemnSupersededMarkers(paths, root, doomed, budget, deadlineAt) {
+  let spent = 0;
+  const area = path.join(paths.retained, "ephemeral");
+  for (const kind of await listDirectoryEntries(area, { root })) {
+    if (!kind.isDirectory()) continue;
+    const kindDirectory = path.join(area, kind.name);
+    for (const record of await listDirectoryEntries(kindDirectory, { root })) {
+      if (!record.isDirectory()) continue;
+      const directory = path.join(kindDirectory, record.name);
+      for (const name of supersededNames(await jsonNames(directory, root))) {
+        if (spent >= budget || expired(deadlineAt)) return { spent, drained: false };
+        await condemn(path.join(directory, name), doomed);
+        spent += 1;
+      }
+    }
+  }
+  return { spent, drained: true };
+}
+
+/**
+ * Reclaim what the store keeps and no longer reads.
+ *
+ * Both classes are provably unread - a retired journal entry has no reader at
+ * all, and a superseded ephemeral marker is skipped by the only function that
+ * looks - which is what lets this run without an operator asking. Anything that
+ * carries meaning waits for `acc prune`.
+ *
+ * @returns {Promise<{ reclaimed: number, remaining: boolean }>}
+ */
+export async function reclaimRetired(paths,
+  { root, limit = RECLAIM_BUDGET, deadlineAt } = {}) {
+  if (expired(deadlineAt)) return { reclaimed: 0, remaining: true };
+  let spent = 0;
+
+  const leftovers = await discardLeftovers(root, limit, deadlineAt);
+  spent += leftovers.spent;
+  // What a previous pass already moved has left its live name; removing it is
+  // what reclaims it, so it counts here exactly as this pass's own work does.
+  let reclaimed = leftovers.spent;
+  if (!leftovers.drained) return { reclaimed, remaining: true };
+
+  const doomed = await detachDoomed(root);
+  const journals = await condemnRetiredJournals(paths, root, doomed, limit - spent, deadlineAt);
+  spent += journals.spent;
+  const markers = journals.drained
+    ? await condemnSupersededMarkers(paths, root, doomed, limit - spent, deadlineAt)
+    : { spent: 0, drained: false };
+  spent += markers.spent;
+
+  const removed = await discard(doomed, root, limit - spent, deadlineAt);
+  reclaimed += removed.spent;
+  return { reclaimed,
+    remaining: !journals.drained || !markers.drained || !removed.drained };
+}
+
+/**
+ * Reclaim named state records, each with every retention marker it owns.
+ *
+ * `tx.remove` is the wrong instrument here and the difference matters. It
+ * publishes a deletion marker, which hides the record and *adds* a file, and
+ * `listState` then reads the record before asking whether that marker exists.
+ * Removing records that way would make the store larger and its listings
+ * slower, which is the opposite of what an operator ran this for. A marker only
+ * exists because the store cannot unlink a live name; when the record itself
+ * can be moved out, the marker has nothing left to say.
+ *
+ * `generation` is the caller's optimistic lock. Eligibility is decided from a
+ * snapshot and applied later, and a claim can be renewed in between, which
+ * writes a new generation onto the same id. A record that no longer matches is
+ * skipped rather than removed: it is no longer the record that was judged.
+ *
+ * @param {{kind: string, id: string, generation: string}[]} entries
+ * @returns {Promise<{ reclaimed: number, skipped: number, remaining: boolean }>}
+ */
+export async function reclaimStateRecords(paths, entries,
+  { root, limit = RECLAIM_BUDGET, deadlineAt } = {}) {
+  if (expired(deadlineAt)) return { reclaimed: 0, skipped: 0, remaining: entries.length > 0 };
+  let spent = 0;
+  let skipped = 0;
+
+  const leftovers = await discardLeftovers(root, limit, deadlineAt);
+  spent += leftovers.spent;
+  if (!leftovers.drained) {
+    return { reclaimed: leftovers.spent, skipped, remaining: true };
+  }
+
+  const doomed = await detachDoomed(root);
+  let condemned = 0;
+  let drained = true;
+  for (const entry of entries) {
+    // Two condemnations per record, asked for together, so a budget never
+    // leaves a record half removed - its markers gone and the record itself
+    // still listed, which would read as a live record whose deletion history
+    // had been erased.
+    if (spent + 2 > limit || expired(deadlineAt)) {
+      drained = false;
+      break;
+    }
+    const filePath = statePath(paths, entry.kind, entry.id);
+    const found = await readJsonIfPresent(filePath, root).catch(() => null);
+    if (found === null || found.value?.generation !== entry.generation) {
+      skipped += 1;
+      continue;
+    }
+    await condemn(filePath, doomed);
+    spent += 1;
+    condemned += 1;
+    try {
+      await condemn(path.join(paths.retained, "state", entry.kind, entry.id), doomed);
+      spent += 1;
+      // Counted only when it moved. Most records own no marker directory at
+      // all, and counting one anyway made `remaining` compare what was
+      // condemned against a number larger than could ever be discarded - so a
+      // finished pass reported work left and asked to be run again forever.
+      condemned += 1;
+    } catch (error) {
+      // A record whose generation was never superseded owns no markers.
+      // Nothing to move is the ordinary case, not a fault.
+      if (error.code !== "ENOENT") throw error;
+    }
+  }
+
+  const removed = await discard(doomed, root, limit - spent, deadlineAt);
+  return { reclaimed: leftovers.spent + removed.spent, skipped,
+    remaining: !drained || !removed.drained || removed.spent < condemned };
+}
+
+/**
+ * Trim the event log to a boundary, reclaiming every event at or below it.
+ *
+ * The floor is raised before a single file moves, and that order is the whole
+ * safety argument. A floor ahead of the trim reports a boundary for events that
+ * are still present, which costs a reader nothing. A trim ahead of the floor is
+ * a silent gap, which is the failure the floor exists to prevent.
+ *
+ * @returns {Promise<{ reclaimed: number, trimmedThrough: string, remaining: boolean }>}
+ */
+export async function trimEventLog(paths, boundary,
+  { root, publish, limit = RECLAIM_BUDGET, deadlineAt } = {}) {
+  assertSequence(boundary, "before");
+  if (expired(deadlineAt)) {
+    return { reclaimed: 0, trimmedThrough: (await readEventFloor(paths, root)) ?? boundary,
+      remaining: true };
+  }
+
+  const names = (await jsonNames(paths.events, root)).sort();
+  const newest = names.at(-1);
+  const allocated = newest === undefined
+    ? await readEventFloor(paths, root)
+    : path.basename(newest, ".json");
+  // A boundary past what this store has ever issued is not a point in its
+  // history. Persisting one would raise the floor above every sequence that can
+  // still be allocated, so a single mistyped cursor would leave a workspace
+  // that can never publish again. Refused rather than clamped: an operator who
+  // named the wrong cursor should hear about it.
+  if (allocated === null || boundary > allocated) {
+    throw new AccError(EXIT.USAGE,
+      "before is past the newest event this workspace has issued",
+      { before: boundary, newest: allocated });
+  }
+  const trimmedThrough = await raiseEventFloor(paths, { ...publish, root }, boundary);
+
+  let spent = 0;
+  let drained = true;
+  const doomed = await detachDoomed(root);
+  for (const name of names) {
+    // Sequences are fixed-width, so comparing the names is comparing the
+    // numbers, and the sorted listing lets this stop at the boundary rather
+    // than walk a log that is mostly newer than it.
+    if (path.basename(name, ".json") > boundary) break;
+    if (spent >= limit || expired(deadlineAt)) {
+      drained = false;
+      break;
+    }
+    await condemn(path.join(paths.events, name), doomed);
+    spent += 1;
+  }
+
+  const removed = await discard(doomed, root, limit - spent, deadlineAt);
+  return { reclaimed: removed.spent, trimmedThrough,
+    remaining: !drained || !removed.drained };
+}

package/node_modules/@agents-can-communicate/storage-filesystem/src/recovery.mjs

@@ -3,22 +3,62 @@ import path from "node:path";
 import { AccError, EXIT, validateRecord } from "@agents-can-communicate/protocol";
 
 import { listDirectoryEntries, listJsonFiles, readJsonIfPresent } from "./atomic-json.mjs";
+import { readEventFloor } from "./event-floor.mjs";
 import { readStoreIdentity } from "./identity.mjs";
 import { readOpenJournals, rollForward } from "./journal.mjs";
 import { assertEventBinding, assertStateBinding } from "./record-id.mjs";
+import { reclaimRetired } from "./reclaim.mjs";
+import { sweepAcceptedStages } from "./stage-sweep.mjs";
 import { storePaths } from "./store.mjs";
 import { withWriterMutex } from "./writer-mutex.mjs";
 
 /** @typedef {{ healthy: boolean, repaired: string[], blocked: string[],
- * corrupt: string[] }} RepairReport */
+ * corrupt: string[], swept: number, staged: number, partials: number, retired: number,
+ * trimmedThrough: string|null }} RepairReport */
 
-const report = ({ repaired = [], blocked = [], corrupt = [] }) => ({
+const report = ({ repaired = [], blocked = [], corrupt = [], swept = 0, staged = 0,
+  partials = 0, retired = 0, trimmedThrough = null }) => ({
   healthy: blocked.length === 0 && corrupt.length === 0,
   repaired: [...repaired].sort(),
   blocked: [...blocked].sort(),
   corrupt: [...corrupt].sort(),
+  swept,
+  staged,
+  partials,
+  retired,
+  trimmedThrough,
 });
 
+// Counting, never touching. A stage held in tmp by an older version is an
+// accepted stage wherever it sits, so both places are counted as staged, while
+// a partial is only ever the unfinished temporary a failed publication left.
+async function countStaging(paths, root) {
+  const entries = async directory => {
+    try {
+      return await listDirectoryEntries(directory, { root });
+    } catch (error) {
+      if (error.code === "ENOENT") return [];
+      throw error;
+    }
+  };
+  const accepted = name => name.endsWith(".published");
+  const staged = (await entries(paths.stage)).filter(entry => entry.isFile()
+    && accepted(entry.name)).length;
+  const inTmp = await entries(paths.tmp);
+  // A retired journal entry is one the store keeps and never reads again, so
+  // counting them is what tells an operator the automatic pass has work left.
+  const completed = new Set((await entries(path.join(paths.retained, "journal")))
+    .filter(entry => entry.isFile()).map(entry => entry.name));
+  const retired = (await entries(paths.journal))
+    .filter(entry => entry.isFile() && completed.has(entry.name)).length;
+  return {
+    staged: staged + inTmp.filter(entry => accepted(entry.name)).length,
+    partials: inTmp.filter(entry => entry.name.endsWith(".tmp")).length,
+    retired,
+    trimmedThrough: await readEventFloor(paths, root),
+  };
+}
+
 async function inspect(root) {
   const paths = storePaths(root);
   const blocked = [];
@@ -82,6 +122,7 @@ export async function diagnoseFilesystemStore({ root }) {
     blocked: state.blocked,
     corrupt: state.corrupt,
     repaired: state.pending.map(entry => entry.transactionId),
+    ...await countStaging(state.paths, root),
   });
 }
 
@@ -92,18 +133,24 @@ export async function repairFilesystemStore({ root, clock }) {
     // would turn an ambiguous store into a confidently wrong one.
     return report({ blocked: state.blocked, corrupt: state.corrupt });
   }
-  if (state.pending.length === 0) return report({});
-
-  const repaired = await withWriterMutex(state.paths, { root, tmpDir: state.paths.tmp, clock },
+  // No early return for an empty journal any more: a store with nothing to roll
+  // forward is exactly the store whose staging directory needs reclaiming.
+  return withWriterMutex(state.paths, { root, tmpDir: state.paths.tmp, clock },
     async () => {
       const completed = [];
       for (const entry of await readOpenJournals(state.paths, root)) {
         await rollForward(state.paths, { root, tmpDir: state.paths.tmp, clock }, entry);
         completed.push(entry.transactionId);
       }
-      return completed;
+      // An operator is waiting on this and a hook is not, so the sweep runs
+      // without the per-pass bound the automatic one obeys.
+      // An operator is waiting on this, and a hook is not, so neither pass is
+      // bounded here.
+      const { swept } = await sweepAcceptedStages(state.paths, { root, limit: Infinity });
+      const reclaimed = await reclaimRetired(state.paths, { root, limit: Infinity });
+      return report({ repaired: completed, swept: swept + reclaimed.reclaimed,
+        ...await countStaging(state.paths, root) });
     });
-  return report({ repaired });
 }
 
 export function assertRepairable(reportValue) {