@volter/supercode-orchestrator 0.5.29 → 0.5.31

package/board/dispatch.mjs

@@ -30,7 +30,7 @@ import { homedir, hostname, tmpdir, totalmem } from 'node:os';
 import { basename, dirname, join } from 'node:path';
 import { createHash, randomUUID } from 'node:crypto';
 import { fileURLToPath } from 'node:url';
-import { view, boardCopyOf, boardPath, claimer, copyRefusal, event, json, listBoards, logsDir, now, openBoard, setActor, tx } from './store.mjs';
+import { view, boardCopyOf, boardPath, claimer, claimHolds, copyRefusal, event, json, listBoards, logsDir, newClaim, now, openBoard, setActor, tx } from './store.mjs';
 import { doors, fire, loadWorkflow, resumeOf, runSpec, runWorkingSeconds, requestSessionClose, retireSupersededRuns, refreshCardInputs, refreshRelationAnswers, transitionsFor, scopeOf, statusOf, projectStatus, storedStatus, templateValue, workflowFor } from './workflow.mjs';
 import { checkWorkspace, assertWorkspaceOwner, notePresence, resolveWorkspace } from './workspace.mjs';
 import { capacity, harnessDefaults, holdings } from './capacity.mjs';
@@ -780,7 +780,7 @@ export async function startCard(board, id, { roles = ['manager'], lanes, setting
 
 /** Reconcile durable close intent and ended runs, including old unverified closed_at records.
  * Reads and process I/O stay outside the writer. An active successor owns its pane until it ends. */
-export function closeRequested(board, report, { taskId = null, caller = null } = {}) {
+export function closeRequested(board, report, { taskId = null, caller = null, ttl = null } = {}) {
   const rows = view(board, (db) => db.prepare(`SELECT id, task_id, ended_at, metadata FROM task_runs
     WHERE metadata IS NOT NULL ${taskId ? 'AND task_id = ?' : ''}`).all(...(taskId ? [taskId] : [])));
   // Repair pre-transfer history once: only identities with an older unended reference need a write.
@@ -849,8 +849,19 @@ export function closeRequested(board, report, { taskId = null, caller = null } =
     // every round (each attempt is a process). A deferred close (an offline machine, a run that still owns the pane) waits.
     if (session.close_failed_at && session.close_error && !/^close deferred/.test(session.close_error)) continue;
     const active = view(board, (db) => db.prepare(`SELECT r.id, r.metadata FROM task_runs r WHERE r.ended_at IS NULL OR EXISTS (SELECT 1 FROM tasks t WHERE t.current_run_id = r.id)`).all());
-    // Persist retries even for historical ends that did not have a close_session action.
-    tx(board, (db) => requestSessionClose(db, row.task_id, { runId: row.id }));
+    // Persist retries even for historical ends that did not have a close_session action. The close is claimed in the
+    // same transaction, before the pane or process is touched: of two rounds (or a round and a verb) that read one close,
+    // only the one whose claim committed closes it, and the other never records the pane it finds gone as a failure.
+    const claimed = tx(board, (db) => {
+      const current = json(db.prepare('SELECT metadata FROM task_runs WHERE id = ?').get(row.id)?.metadata, {})?.supercode;
+      if (!current || current.close_confirmed_at || current.close_transferred_to || current.pane !== session.pane || current.session_id !== session.session_id || claimHolds(current.close_claim)) return false;
+      requestSessionClose(db, row.task_id, { runId: row.id });
+      // a close this process already claims, with a minute of its claim left, is tried again without a new claim
+      const own = current.close_claim?.lock === claimer() && Number(current.close_claim.expires) > now() + 60;
+      if (!own) setRunSession(db, row.id, { close_claim: newClaim(ttl ?? boardSettings(workflowFor(board.root).params).ttl) });
+      return true;
+    });
+    if (!claimed) continue;
     let error = null;
     try {
       const machine = session.machine ?? localMachine();
@@ -887,6 +898,8 @@ export function closeRequested(board, report, { taskId = null, caller = null } =
       const current = json(db.prepare('SELECT metadata FROM task_runs WHERE id = ?').get(row.id)?.metadata, {})?.supercode;
       if (!current || current.pane !== session.pane || current.session_id !== session.session_id) return;
       const at = now();
+      // a close confirmed meanwhile (a claim that expired under a slow close) is not undone by this attempt's failure
+      if (error && current.close_confirmed_at) return;
       if (error) {
         const final = !/^close deferred/.test(error);
         setRunSession(db, row.id, { ...current, close_attempted_at: at, close_attempts: Number(current.close_attempts ?? 0) + 1, close_error: error, ...(final ? { close_failed_at: at } : {}) });
@@ -1687,30 +1700,65 @@ export function requestAdoption(board, id, who, { workspace = null, assignee = n
 /** How long a requested adoption waits for its session to appear in the fleet before it fails. */
 const ADOPTION_LISTING_WINDOW_S = 180;
 
-/** Adoptions the manager requested and no tick has answered yet, completed from this tick's fleet reading. */
+/**
+ * The events that answer an adoption request, or hold it: a request is open while it is the card's latest of these, and
+ * a round that acts on it first records `adoption_claimed` (the request, its claim lock and when the claim expires).
+ */
+const ADOPTION_ANSWERS = "('dispatch_held','dispatch_released','session_adopted','adoption_failed','adoption_claimed')";
+const latestAnswer = (db, id) => db.prepare(`SELECT id, kind, payload FROM task_events WHERE task_id = ? AND kind IN ${ADOPTION_ANSWERS} ORDER BY id DESC LIMIT 1`).get(id) ?? null;
+
+/**
+ * Adoptions the manager requested and no tick has answered yet, completed from this tick's fleet reading. A round claims
+ * a request in a transaction that finds it still open, before anything it does outside the board (`adoptSession`), so
+ * of two rounds that read one request, one binds it and the other leaves it; the bind and the failure are each written
+ * only while that claim is the card's latest.
+ */
 export function completeAdoptions(board, fleet, settings, report, lanes = null) {
-  const pending = view(board, db => db.prepare(`SELECT e.task_id, e.payload, e.created_at FROM task_events e WHERE e.kind = 'dispatch_held'
-    AND json_extract(e.payload, '$.reason') = 'adoption requested'
-    AND e.id = (SELECT MAX(id) FROM task_events l WHERE l.task_id = e.task_id AND l.kind IN ('dispatch_held','dispatch_released','session_adopted','adoption_failed'))`).all());
+  const pending = view(board, db => db.prepare(`SELECT e.id, e.task_id, e.kind, e.payload, e.created_at FROM task_events e
+    WHERE ((e.kind = 'dispatch_held' AND json_extract(e.payload, '$.reason') = 'adoption requested') OR e.kind = 'adoption_claimed')
+    AND e.id = (SELECT MAX(id) FROM task_events l WHERE l.task_id = e.task_id AND l.kind IN ${ADOPTION_ANSWERS})`).all());
   for (const row of pending) {
-    const asked = json(row.payload, {}) ?? {};
+    // a request another round has claimed is that round's while its claim holds; a claim whose round died is open again
+    let request = row;
+    if (row.kind === 'adoption_claimed') {
+      const claim = json(row.payload, {}) ?? {};
+      if (claimHolds(claim)) { (report.adoption_skipped ??= []).push([row.task_id, `claimed by ${claim.lock}`]); continue; }
+      request = view(board, db => db.prepare('SELECT id, payload, created_at FROM task_events WHERE id = ?').get(claim.request));
+      if (!request) continue;
+    }
+    const asked = json(request.payload, {}) ?? {};
+    const progress = {};
     try {
-      const got = adoptSession(board, row.task_id, asked.session, { workspace: asked.workspace ?? null, assignee: asked.assignee ?? null, role: asked.role ?? null, ttl: settings.ttl, fleet: fleet?.ok ? fleet : null , lanes});
+      const got = adoptSession(board, row.task_id, asked.session, { request: request.id, open: row.id, progress, workspace: asked.workspace ?? null, assignee: asked.assignee ?? null, role: asked.role ?? null, ttl: settings.ttl, fleet: fleet?.ok ? fleet : null , lanes});
       (report.adopted ??= []).push([row.task_id, got.address]);
     } catch (error) {
       const message = String(error.message).slice(-400);
       // A session opened moments before its adoption is asked may not be in the fleet's published set yet: the
       // card stays held and each tick looks again, until the window closes
-      if (error.notListed && Date.now() / 1000 - Number(row.created_at) < ADOPTION_LISTING_WINDOW_S) {
+      if (error.notListed && Date.now() / 1000 - Number(request.created_at) < ADOPTION_LISTING_WINDOW_S) {
         report.queued.push([row.task_id, `adoption of ${asked.session} waits: ${message}`]);
         continue;
       }
-      tx(board, db => event(db, row.task_id, 'adoption_failed', { session: asked.session, error: message }));
-      report.failed.push([row.task_id, `adoption of ${asked.session}: ${message}`]);
+      // the failure answers the request only while it is still open, or claimed by this round: an adoption that bound and
+      // then failed after (its agent run not recorded) is bound, and a request another round claimed is that round's
+      const answered = !error.superseded && tx(board, db => adoptionOpenTo(db, row.task_id, request.id, row.id) && (event(db, row.task_id, 'adoption_failed', { session: asked.session, error: message }), true));
+      // A request another round answered or claimed first is that round's, not a failure of this one: it is noted where
+      // no round prints it. A failure after this round's own claim (bound, then its agent run not recorded) is one.
+      if (!answered && !progress.claimed) { (report.adoption_skipped ??= []).push([row.task_id, message]); continue; }
+      report.failed.push([row.task_id, `adoption of ${asked.session}${answered ? '' : ' (bound by this round, then failed)'}: ${message}`]);
     }
   }
 }
 
+/** Whether request `request` is still this round's to answer: the card's latest answer is `open`, or this process's claim of it. */
+function adoptionOpenTo(db, id, request, open) {
+  const last = latestAnswer(db, id);
+  if (!last) return false;
+  if (last.id === open) return true;
+  const claim = last.kind === 'adoption_claimed' ? json(last.payload, {}) ?? {} : null;
+  return Boolean(claim) && claim.request === request && claim.lock === claimer();
+}
+
 /**
  * A session on this machine that Teams never published (one outside every synced root: a profile's own directory) is
  * found through this machine's own doors: its panes, and the native session each pane's process runs. Only a full
@@ -1748,7 +1796,7 @@ function adoptedRole(db, board, entry, task, run) {
   return runSpec(entry, t?.target ? String(templateValue(t.target, scope)) : status, task)?.role ?? null;
 }
 
-function adoptSession(board, id, who, { workspace = null, ttl, assignee = null, role = null, fleet = null, lanes = null } = {}) {
+function adoptSession(board, id, who, { request = null, open = request, progress = {}, workspace = null, ttl, assignee = null, role = null, fleet = null, lanes = null } = {}) {
   fleet ??= observeFleet();
   if (!fleet.ok) throw new Error(`supercode could not read the fleet: ${fleet.error}`);
   const found = fleet.sessions.find((s) => s.address === who || s.address.endsWith(`:${who}`) || s.name === who) ?? localListing(who);
@@ -1777,6 +1825,20 @@ function adoptSession(board, id, who, { workspace = null, ttl, assignee = null,
   const agentRole = view(board, (db) => adoptedRole(db, board, entry0, card0, beforeRun)) ?? 'implementer';
   const laneName = view(board, (db) => evaluate(entry0.workflow.roles?.[agentRole] ?? 'card.assignee', scopeOf(db, board, card0, { params: entry0.params }))) ?? assignee ?? card0?.assignee;
   const lane = laneName ? lanes?.[laneName] : null, context = process.env.SUPERCODE_CONTEXT, team = agentScope(context);
+  // The request is claimed before this adoption's first write outside the board (the agent's Teams record, below), in
+  // a transaction that finds it still open and the card's run the one read: a request answered or claimed meanwhile, or
+  // a run that changed, ends it here with nothing written anywhere, and no other round acts on a request claimed here.
+  const stale = (db) => {
+    if (request !== null && !adoptionOpenTo(db, id, request, open)) return Object.assign(new Error(`${id}'s adoption request was answered or claimed meanwhile`), { superseded: true });
+    if ((getTask(db, id)?.current_run_id ?? null) !== (beforeRun?.id ?? null)) return new Error(`${id} changed runs during adoption; retry against its current run`);
+    return null;
+  };
+  if (request !== null) tx(board, (db) => {
+    const refused = stale(db);
+    if (refused) throw refused;
+    event(db, id, 'adoption_claimed', { request, session: who, ...newClaim(ttl) });
+  });
+  progress.claimed = request !== null;
   let agent = null;
   if (lane && !team.offline) {
     try { agent = ensureCardAgent({ root: board.root, org: team.org, server: team.server, profiles: entry0.profiles, profile: lane.name, card: id, role: agentRole, sponsor: card0?.sponsor ?? null, machine: normalMachine(host) === localMachine() ? null : host, context }); }
@@ -1789,7 +1851,9 @@ function adoptSession(board, id, who, { workspace = null, ttl, assignee = null,
   const adopted = tx(board, (db) => {
     const task = getTask(db, id);
     if (!task) throw new Error(`no card ${id}`);
-    if ((task.current_run_id ?? null) !== (beforeRun?.id ?? null)) throw new Error(`${id} changed runs during adoption; retry against its current run`);
+    // the bind is made only while this round's claim is the card's latest and the run is the one read
+    const changed = stale(db);
+    if (changed) throw changed;
     if (beforeRun && assignee && assignee !== task.assignee) throw new Error('rebinding a session does not reassign its active run');
     if (task.machine && normalMachine(task.machine) !== host) throw new Error(`${id} runs on ${task.machine}, and ${found.address} is on ${host}; move the card first`);
     const was = lastSession(db, id);
@@ -2193,17 +2257,22 @@ function fireHooks(board, settings, report) {
       child.unref();
     }
   };
-  tx(board, (db) => {
+  // the events are taken (the cursor moved past them) in one transaction and their hooks run once it commits: a
+  // transaction run again on contention, or a second round, never runs a hook twice
+  const due = tx(board, (db) => {
     db.exec('CREATE TABLE IF NOT EXISTS supercode_board_state (key TEXT PRIMARY KEY, value TEXT)');
     const max = Number(db.prepare('SELECT COALESCE(MAX(id), 0) AS m FROM task_events').get().m);
     const row = db.prepare("SELECT value FROM supercode_board_state WHERE key = 'hooks_cursor'").get();
     const cursor = row ? Number(row.value) : max; // the first tick starts at the present, never replays history
+    const taken = [];
     for (const ev of db.prepare('SELECT e.id, e.task_id, e.run_id, e.kind, e.payload, e.created_at, t.assignee FROM task_events e LEFT JOIN tasks t ON t.id = e.task_id WHERE e.id > ? ORDER BY e.id').all(cursor)) {
       const hook = HOOK_OF[ev.kind];
-      if (hook && hooks[hook]) run(hook, { task_id: ev.task_id, run_id: ev.run_id, event: ev.kind, assignee: ev.assignee, payload: json(ev.payload, null), at: ev.created_at });
+      if (hook && hooks[hook]) taken.push([hook, { task_id: ev.task_id, run_id: ev.run_id, event: ev.kind, assignee: ev.assignee, payload: json(ev.payload, null), at: ev.created_at }]);
     }
     db.prepare("INSERT INTO supercode_board_state (key, value) VALUES ('hooks_cursor', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(String(max));
+    return taken;
   });
+  for (const [hook, payload] of due) run(hook, payload);
   if (hooks.on_kanban_dispatch_tick) run('on_kanban_dispatch_tick', { started: report.started.length, reclaimed: report.reclaimed.length, crashed: report.crashed.length, timed_out: report.timed_out.length, promoted: report.promoted });
 }
 
@@ -2410,7 +2479,14 @@ function capMax(cap, group, entry, lanes) {
 /**
  * One tick over every board of the home at `root`. `lanes` from `lanesOf`, `settings` from `boardSettings`.
  * `only` limits starting to one card (a create that answers whether its card started). Each board is ticked under its
- * version-checked card claims; no dispatcher file lock. Answers one report per board.
+ * version-checked card claims; no dispatcher file lock. Two rounds may run at once (the dispatcher's and a one-shot
+ * `workflow dispatch`). An adoption request and a session close are claimed in a transaction (`newClaim`) before
+ * anything is done for them outside the board; card starts, supervision, review requests, mailed verbs and manual
+ * continuations rely on their per-card transactions. Two rounds on one board at once can still both act on: queued
+ * effects in `runEffects` (send_to_actor, session_notice, merge_reviewed, declare_card_agent), board mail,
+ * subscription notices (a doubled notice_delivered has been seen live), deferred-launch retries (two panes),
+ * launch-receipt work (cancelUnissuedLaunch, bindLaunch), the Teams agent-run sync in reconcileLaunches, and
+ * board-only writes that do not re-check (a doubled agent_profile_recorded). Answers one report per board.
  */
 export async function tick({ root, lanes, settings, only = null, slugs = null, dispatch = true, params = {}, deploymentReceipt = false }) {
   eventFleetRoot = root;
@@ -2451,7 +2527,7 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
       readMailbox(board, boardAddress(board.label), report); phase('mailbox');
       await refreshRelationAnswers(board); phase('answers');
       const effects = [];
-      completeAdoptions(board, fleet, settings, report, lanes); reconnectContinuedSessions(board, fleet, entry, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); await runEffects(board, effects.splice(0), { retryNotices: true }); closeRequested(board, report); phase('supervise');
+      completeAdoptions(board, fleet, settings, report, lanes); reconnectContinuedSessions(board, fleet, entry, report); reconcileLaunches(board, fleet, report, entry, effects, settings, lanes); supervise(board, fleet, settings, report, entry, effects); await runEffects(board, effects.splice(0), { retryNotices: true }); closeRequested(board, report, { ttl: settings.ttl }); phase('supervise');
       report.unsettled = [];
       report.promoted = recomputeReady(board, effects, report.unsettled);
       tx(board,db=>auditEndedRuns(db,board,entry));

package/board/store.mjs

@@ -259,6 +259,28 @@ export const newTaskId = () => `t_${randomBytes(4).toString('hex')}`;
 /** The claim lock this process writes, in Hermes's `host:pid` form. */
 export const claimer = () => `${hostname()}:${process.pid}`;
 
+/**
+ * A claim on one request a round acts on outside the board (an adoption request, a session close): written in the
+ * transaction that finds the request still open, before anything outside the board is done for it, so of two rounds
+ * that read one request only the one whose claim committed acts on it. It is this process's claim lock and an expiry
+ * one claim TTL out, as a card's claim is.
+ */
+export const newClaim = (ttl) => ({ lock: claimer(), expires: now() + (Number(ttl) > 0 ? Number(ttl) : 15 * 60) });
+
+/**
+ * Whether a claim keeps the caller off its request: another process's, unexpired, and that process alive where its pid
+ * can be read (this machine). A claim whose process died is open again at once here, and at its expiry from another
+ * machine; the caller's own claim never keeps it off (its rounds run one after another).
+ */
+export function claimHolds(claim) {
+  if (!claim?.lock || claim.lock === claimer() || Number(claim.expires) <= now()) return false;
+  const lock = String(claim.lock), at = lock.lastIndexOf(':');
+  if (lock.slice(0, at) !== hostname()) return true;
+  const pid = Number(lock.slice(at + 1));
+  if (!(pid > 0)) return false;
+  try { process.kill(pid, 0); return true; } catch (error) { return error.code === 'EPERM'; }
+}
+
 export function json(text, fallback = null) {
   if (text === null || text === undefined || text === '') return fallback;
   try { return JSON.parse(text); } catch { return fallback; }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.29",
+  "version": "0.5.31",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {