mixdog 1.3.5 → 1.4.0

package/src/session-runtime/services/agent-tool/worker-index/released-store.mjs

@@ -0,0 +1,123 @@
+// worker-index/released-store.mjs
+// Which agent sessions their owner closed (the agent tool's `close`): session
+// id → when. A closed session stays open and resumable, so a session scan
+// (also after a daemon restart) would list it again; the marker is what keeps
+// its rows `released` until a new turn starts on it. It lives in a file of its
+// own beside the worker index, so a writer that rebuilds the index document
+// (an older daemon) can never drop it. Entries expire after the tombstone TTL,
+// judged on every read, not only when the file changes.
+import { readFileSync, statSync } from 'node:fs';
+
+import { parseJsonText } from '../../../../runtime/shared/atomic-file.mjs';
+import { createIndexWriteQueue } from '../index-write-queue.mjs';
+import { TAG_TOMBSTONE_TTL_MS } from '../worker-rows.mjs';
+
+export const RELEASED_FILE = 'agent-workers-released.json';
+
+/** id → epoch ms of every entry of a raw document (expired ones included). */
+function parseReleased(doc) {
+  const raw = doc?.released && typeof doc.released === 'object' ? doc.released : {};
+  return new Map(
+    Object.entries(raw)
+      .map(([sessionId, at]) => [sessionId, Date.parse(String(at)) || 0])
+      .filter(([sessionId]) => sessionId)
+  );
+}
+
+/** The next document: unexpired entries plus the mutator's changes. */
+function rewriteReleased(cur, mutator) {
+  const cutoff = Date.now() - TAG_TOMBSTONE_TTL_MS;
+  const entries = new Map();
+  for (const [sessionId, at] of parseReleased(cur)) {
+    if (at >= cutoff) entries.set(sessionId, new Date(at).toISOString());
+  }
+  mutator(entries);
+  return { version: 1, released: Object.fromEntries(entries) };
+}
+
+/** `file` may be null (no data dir): reads are empty and writes are no-ops. */
+export function createReleasedStore(file) {
+  let cache = null; // { mtimeMs, size, doc, at }
+  let dirty = true;
+  // Unpersisted writes are read from the queue's projection.
+  let projected = null; // { version, at }
+
+  const queue = createIndexWriteQueue({
+    file,
+    rewrite: rewriteReleased,
+    readDoc: () => {
+      dirty = true;
+      entries();
+      return cache?.doc ?? null;
+    },
+    onPersisted: () => {
+      dirty = true;
+    },
+  });
+
+  /** id → epoch ms, as written (not yet filtered by expiry). */
+  function entries() {
+    const doc = queue.projection();
+    if (doc) {
+      if (!projected || projected.version !== queue.version()) {
+        projected = { version: queue.version(), at: parseReleased(doc) };
+      }
+      return projected.at;
+    }
+    if (!file) return new Map();
+    // Only a missing file is an empty store; any other read error propagates
+    // (queue.write reports it and queues nothing; a reader sees it thrown).
+    let st;
+    try {
+      st = statSync(file);
+    } catch (error) {
+      if (error?.code !== 'ENOENT') throw error;
+      cache = null;
+      return new Map();
+    }
+    if (!dirty && cache && cache.mtimeMs === st.mtimeMs && cache.size === st.size) return cache.at;
+    let parsed = null;
+    let text = null;
+    try {
+      text = readFileSync(file, 'utf8');
+    } catch (error) {
+      if (error?.code !== 'ENOENT') throw error;
+    }
+    if (text !== null) {
+      try {
+        parsed = parseJsonText(text);
+      } catch (error) {
+        // Left on disk and reported; the next save moves it aside to
+        // `<file>.corrupt-<ts>` (atomic-file's damaged-state recovery) instead
+        // of overwriting it.
+        process.stderr.write(`[released-store] ${file} is malformed (${error.message}); markers unreadable\n`);
+      }
+    }
+    cache = { mtimeMs: st.mtimeMs, size: st.size, doc: parsed, at: parseReleased(parsed) };
+    dirty = false;
+    return cache.at;
+  }
+
+  return {
+    /** Whether the owner closed this session less than the TTL ago. */
+    isReleased: (sessionId) => entries().get(sessionId) >= Date.now() - TAG_TOMBSTONE_TTL_MS,
+    mark(sessionIds) {
+      const at = new Date().toISOString();
+      queue.write((entriesById) => {
+        for (const sessionId of sessionIds) entriesById.set(sessionId, at);
+      });
+    },
+    clear(sessionId) {
+      if (!entries().has(sessionId)) return;
+      queue.write((entriesById) => {
+        entriesById.delete(sessionId);
+      });
+    },
+    flush: queue.flush,
+    flushSync: queue.flushSync,
+    hasUnpersisted: queue.hasUnpersisted,
+    invalidate: () => {
+      dirty = true;
+    },
+  };
+}

package/src/session-runtime/services/agent-tool/worker-index/restart-settlement.mjs

@@ -0,0 +1,112 @@
+// worker-index/restart-settlement.mjs
+// Rows whose turn a process exit or restart ended: this process's active rows
+// settled as failed on exit, stale active rows and approval waits settled at
+// startup and on later sweeps, and the owner's restart-failure notification
+// stamped once queued. Owns the keys of the rows this process marked active.
+import { positiveInt } from '../helpers.mjs';
+import { workerRowKey } from '../worker-rows.mjs';
+import { clearAwaitingUser } from './approval-wait.mjs';
+import {
+  createRowLiveness,
+  isActiveWorkerRow,
+  isStaleApprovalWait,
+  isUnnotifiedRestartFailure,
+  PROCESS_RESTART_FAILURE,
+  runtimeDied,
+} from './row-liveness.mjs';
+
+export function createRestartSettlement({ store, cfgMod, mgr }) {
+  const liveness = createRowLiveness({
+    cfgMod,
+    isSessionRunning: (sessionId) => mgr.isSessionTurnInFlight?.(sessionId) === true,
+  });
+  // Rows this process marked active; settled as failed on exit (their turn
+  // dies with the process) so a crash of the host never leaves them
+  // "running" forever.
+  const activeWorkerKeys = new Set();
+
+  /** Track whether the row written under `key` is one this process runs. */
+  function trackActiveRow(key, row) {
+    if (isActiveWorkerRow(row)) activeWorkerKeys.add(key);
+    else activeWorkerKeys.delete(key);
+  }
+
+  function flushActiveWorkerRows() {
+    if (!activeWorkerKeys.size) return;
+    const keys = [...activeWorkerKeys];
+    activeWorkerKeys.clear();
+    const now = Date.now();
+    store.write((byKey) => {
+      for (const key of keys) {
+        const current = byKey.get(key);
+        if (!current || positiveInt(current.runtimePid) !== process.pid) continue;
+        if (isActiveWorkerRow(current)) {
+          byKey.set(key, liveness.failedRow(current, now, PROCESS_RESTART_FAILURE, true));
+        }
+      }
+    });
+  }
+
+  // Startup recovery: stale active rows settle — as failed when another
+  // process ran their turn (it is gone, or is the predecessor still shutting
+  // down: either way the turn never finished), otherwise to idle — and an
+  // approval wait left behind by a runtime that is gone (daemon restart) is
+  // cleared. `deadOnly` (later sweeps) settles only rows whose runtime is
+  // provably dead, never a live process's quiet turn.
+  function recoverStaleWorkerRows({ deadOnly = false } = {}) {
+    const now = Date.now();
+    const rows = store.readAll();
+    const isStale = deadOnly
+      ? (row) => isActiveWorkerRow(row) && runtimeDied(row)
+      : (row) => liveness.isStaleActive(row, now);
+    const staleActive = rows.filter(isStale);
+    const staleWaits = rows.some(isStaleApprovalWait);
+    if (!staleActive.length && !staleWaits) return false;
+    if (staleActive.length) {
+      store.write((byKey) => {
+        for (const row of staleActive) {
+          const key = workerRowKey(row);
+          const current = byKey.get(key);
+          if (!current || !isStale(current)) continue;
+          const ranElsewhere = positiveInt(current.runtimePid) && positiveInt(current.runtimePid) !== process.pid;
+          byKey.set(
+            key,
+            ranElsewhere ? liveness.failedRow(current, now, PROCESS_RESTART_FAILURE) : liveness.idleRow(current, now)
+          );
+        }
+      });
+    }
+    if (staleWaits) store.update((byKey) => clearAwaitingUser(byKey, isStaleApprovalWait));
+    return true;
+  }
+
+  /** Rows whose turn a process restart ended and whose owner has not been told
+   *  yet — whoever settled the row (this process at startup, or the
+   *  predecessor's exit flush). Read-only: the caller enqueues the owner's
+   *  notification durably, THEN calls `markRestartNotified`, so a crash in
+   *  between retries (the notification's stable id makes the retry exactly-once). */
+  function pendingRestartFailures() {
+    return store.readAll().filter(isUnnotifiedRestartFailure);
+  }
+
+  /** Stamp these rows as told, once their notification is durably queued. */
+  function markRestartNotified(rows) {
+    const stamp = new Date().toISOString();
+    store.write((byKey) => {
+      for (const row of rows) {
+        const key = workerRowKey(row);
+        const current = byKey.get(key);
+        if (!current || !isUnnotifiedRestartFailure(current)) continue;
+        byKey.set(key, { ...current, restartNotified: stamp });
+      }
+    });
+  }
+
+  return {
+    trackActiveRow,
+    flushActiveWorkerRows,
+    recoverStaleWorkerRows,
+    pendingRestartFailures,
+    markRestartNotified,
+  };
+}

package/src/session-runtime/services/agent-tool/worker-index/row-reads.mjs

@@ -0,0 +1,36 @@
+// worker-index/row-reads.mjs
+// Scoped reads of the worker rows and tag tombstones: a row a tombstone still
+// blocks never reads as work, and a legacy tombstone without an owner takes
+// the one its session resolves to.
+import { clean, rowMatchesContext } from '../helpers.mjs';
+import { recoverTagTombstoneOwner } from '../worker-rows.mjs';
+import {
+  findTagTombstone,
+  tagTombstoneKey,
+  tombstoneBlocksWork,
+} from '../../../../runtime/shared/agent-reap-state.mjs';
+
+export function createWorkerRowReads(store, mgr) {
+  function readWorkerRows(context = {}) {
+    const { rows, tombstones: savedTombstones } = store.readSnapshot();
+    if (rows.length === 0) return rows;
+    const tombstones = new Map(recoverTombstoneOwners(savedTombstones).map((row) => [tagTombstoneKey(row), row]));
+    return rows.filter(
+      (row) => rowMatchesContext(row, context) && !tombstoneBlocksWork(row, findTagTombstone(row, tombstones))
+    );
+  }
+
+  function readAllTagTombstones() {
+    return recoverTombstoneOwners(store.readTombstones());
+  }
+
+  function recoverTombstoneOwners(rows) {
+    if (!rows.some((row) => !clean(row.parentSessionId || row.ownerSessionId))) return rows;
+    const sessions = typeof mgr.listSessions === 'function' ? mgr.listSessions({ includeClosed: true }) : [];
+    // Resolve legacy ownership against all sessions, never just the caller's
+    // scope: filtering first would conceal an ambiguous sibling owner.
+    return rows.map((row) => recoverTagTombstoneOwner(row, sessions));
+  }
+
+  return { readWorkerRows, readAllTagTombstones };
+}

package/src/session-runtime/services/agent-tool/worker-index/row-shape.mjs

@@ -65,6 +65,8 @@ export function normalizeWorkerRows(value) {
       // Additive and transient: present only while the worker's turn waits on
       // a user approval (worker-index setAwaitingUser).
       ...(row.awaitingUser === 'approval' ? { awaitingUser: 'approval' } : {}),
+      // Present only on an agent its owner closed (the agent tool's `close`).
+      ...(row.released === true ? { released: true } : {}),
     }))
     .filter((row) => !isLeadPoolAgent(row.agent) && keepWorkerRow(row));
 }

package/src/session-runtime/services/agent-tool/worker-index/row-store.mjs

@@ -4,10 +4,11 @@
 // worker index of a process over one file shares one store (acquireWorkerRowStore),
 // so all of them serialize through the same projection and queue.
 import { readFileSync, statSync } from 'node:fs';
-import { resolve } from 'node:path';
+import { dirname, join, resolve } from 'node:path';
 
 import { createIndexWriteQueue } from '../index-write-queue.mjs';
 import { normalizeTagTombstones, tagTombstoneKey, workerRowKey } from '../worker-rows.mjs';
+import { RELEASED_FILE, createReleasedStore } from './released-store.mjs';
 import { settleAwaitingUser } from './approval-wait.mjs';
 import { keepWorkerRow, normalizeWorkerRows } from './row-shape.mjs';
 
@@ -62,6 +63,9 @@ function rewriteIndexDocument(cur, mutator) {
 
 /** `file` may be null (no data dir): reads are empty and writes are no-ops. */
 export function createWorkerRowStore(file) {
+  // The release markers live in their own file beside the index, so a writer
+  // that rebuilds the index document never drops them.
+  const released = createReleasedStore(file ? join(dirname(file), RELEASED_FILE) : null);
   // Mtime-keyed parse cache. A single spawn calls refreshTagsFromSessions /
   // resolveTag / nextTag, which each re-read and re-parse this file; across a
   // parallel fanout that is O(spawns^2) synchronous reads of the same bytes.
@@ -148,18 +152,27 @@ export function createWorkerRowStore(file) {
     readAll,
     readSnapshot,
     readTombstones,
+    isReleased: released.isReleased,
+    markReleased: released.mark,
+    clearReleased: released.clear,
     write,
     /** Conditional field update: `mutator` runs now on the projection and again
      *  on the locked document when persisted, so its condition is re-checked
      *  against what is on disk. The one writer of `awaitingUser`. */
     update: (mutator) => queue.write(mutator, { rerun: true }),
     /** Resolves once every write so far is on disk. */
-    flush: queue.flush,
+    flush: async () => {
+      await Promise.all([queue.flush(), released.flush()]);
+    },
     /** Exit path: persist what is still queued, synchronously. */
-    flushSync: queue.flushSync,
-    hasUnpersisted: queue.hasUnpersisted,
+    flushSync: () => {
+      const flushed = [queue.flushSync(), released.flushSync()];
+      return flushed.every(Boolean);
+    },
+    hasUnpersisted: () => queue.hasUnpersisted() || released.hasUnpersisted(),
     invalidate: () => {
       dirty = true;
+      released.invalidate();
     },
   };
 }

package/src/session-runtime/services/agent-tool/worker-index/row-writes.mjs

@@ -0,0 +1,58 @@
+// worker-index/row-writes.mjs
+// Row upserts (immediate or batched), removal, and the owner's release mark:
+// every upsert projects the row into the tag maps (`bindTag`), tracks whether
+// this process runs it, and brings its approval wait in step.
+import { clean } from '../helpers.mjs';
+import { applyWorkerRowUpsert, workerRowKey } from '../worker-rows.mjs';
+import { isUnnotifiedRestartFailure } from './row-liveness.mjs';
+import { normalizeWorkerRows, workerRowFromSession } from './row-shape.mjs';
+
+export function createWorkerRowWrites({ file, store, batch, mgr, bindTag, settlement, awaitingUser }) {
+  function upsertWorkerRow(row, { defer = false } = {}) {
+    const normalized = normalizeWorkerRows({ workers: [row] })[0];
+    if (!normalized) return false;
+    const key = workerRowKey(normalized);
+    settlement.trackActiveRow(key, normalized);
+    bindTag(normalized);
+    if (defer) {
+      if (!file) return false;
+      awaitingUser.deferReconcile(normalized.sessionId);
+      return batch.queue((byKey) => applyWorkerRowUpsert(byKey, normalized));
+    }
+    store.write((byKey) => {
+      applyWorkerRowUpsert(byKey, normalized);
+    });
+    awaitingUser.reconcileAwaitingUser(normalized.sessionId);
+    return true;
+  }
+
+  // A session its owner closed keeps `released` on every row written for it,
+  // until a turn starts on it.
+  const rowFromSession = (session, fallbackTag = '', extra = {}) => {
+    const row = workerRowFromSession(session, fallbackTag, extra, (sessionId) => mgr.getSessionRuntime?.(sessionId));
+    return row && store.isReleased(row.sessionId) ? { ...row, released: true } : row;
+  };
+
+  /** Persist that the owner closed these sessions (survives a restart). */
+  function markReleased(sessionIds) {
+    store.markReleased(sessionIds.map(clean).filter(Boolean));
+  }
+
+  // A row whose owner is still owed its restart failure stays: closing the
+  // worker (a process exit closes every one) must not lose that result.
+  function removeWorkerRow({ tag = '', sessionId = '' } = {}) {
+    const targetTag = clean(tag);
+    const targetSessionId = clean(sessionId);
+    batch.flush();
+    store.write((byKey) => {
+      for (const [key, row] of [...byKey.entries()]) {
+        if (isUnnotifiedRestartFailure(row)) continue;
+        if ((targetSessionId && row.sessionId === targetSessionId) || (targetTag && row.tag === targetTag)) {
+          byKey.delete(key);
+        }
+      }
+    });
+  }
+
+  return { upsertWorkerRow, rowFromSession, markReleased, removeWorkerRow };
+}