@volter/supercode-orchestrator 0.4.140 → 0.4.142

package/board/cli.mjs

@@ -385,7 +385,7 @@ export async function runWorkflow(argv) {
       const card = withBoard((b) => {
         const got = engine.showCard(b.db, id);
         got.runs = engine.runs(b.db, id, { stateType: args['state-type'] ?? null, stateName: args['state-name'] ?? null });
-        got.diagnostics = engine.diagnose(b.db, got.task, { failureLimit: settings.failureLimit });
+        got.diagnostics = engine.diagnose(b.db, got.task, { failureLimit: settings.failureLimit, launchWindowSeconds: settings.launchWindow });
         got.stage = statusOf(b.db, got.task);
         got.time = scopeOf(b.db, b, got.task, { params: workflowFor(root).params }).card.time;
         return got;
@@ -644,7 +644,7 @@ export async function runWorkflow(argv) {
       const order = ['warning', 'error', 'critical'];
       const found = withBoard((b) => {
         const cards = args.task ? [engine.getTask(b.db, args.task)].filter(Boolean) : engine.listCards(b.db, {});
-        return cards.flatMap((t) => engine.diagnose(b.db, t, { failureLimit: settings.failureLimit, fleet }).map((d) => ({ task: t.id, title_of_task: t.title, ...d })))
+        return cards.flatMap((t) => engine.diagnose(b.db, t, { failureLimit: settings.failureLimit, fleet, launchWindowSeconds: settings.launchWindow }).map((d) => ({ task: t.id, title_of_task: t.title, ...d })))
           .filter((d) => !args.severity || order.indexOf(d.severity) >= order.indexOf(args.severity));
       });
       if (args.json) { say(args, '', found); return 0; }

package/board/dispatch.mjs

@@ -21,7 +21,7 @@ import { spawn as spawnChild, spawnSync } from 'node:child_process';
 import { existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
 import { hostname, tmpdir } from 'node:os';
 import { basename, dirname, join } from 'node:path';
-import { randomUUID } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import { fileURLToPath } from 'node:url';
 import { view, boardPath, claimer, event, json, listBoards, logsDir, now, openBoard, setActor, tx } from './store.mjs';
 import { doors, fire, loadWorkflow, resumeOf, runSpec, requestSessionClose, retireSupersededRuns, refreshRelationAnswers, transitionsFor, scopeOf, statusOf, templateValue, workflowFor } from './workflow.mjs';
@@ -195,6 +195,7 @@ export function boardSettings(params = {}) {
     failureLimit: pos(params.failure_limit) ?? DEFAULT_FAILURE_LIMIT,
     ttl: claimTtl(params.claim_ttl_seconds),
     grace: pos(params.session_grace_seconds) ?? 120,
+    launchWindow: pos(params.session_launch_window_seconds) ?? 300,
     interval: Math.max(Number(params.dispatch_interval_seconds) || 60, 1),
     idleQuiet: pos(params.session_idle_quiet_seconds) ?? 600,
     logRotateBytes: pos(params.worker_log_rotate_bytes) ?? 2 * 1024 * 1024,
@@ -258,10 +259,6 @@ function supervise(board, fleet, settings, report, entry, effects) {
       if (!task || (entry.workflow.run ? !task.current_run_id : task.status !== 'running')) return;
       const runRow = currentRun(db, id);
       let session = runRow?.metadata?.supercode ?? null;
-      if (session?.launch_pending || session?.launch_deferred) {
-        extendClaim(db, id, { ttl: settings.ttl, reason: 'launch_receipt_pending' });
-        return; // An uncertain launch owns this run until a machine receipt settles it.
-      }
       if (session?.pane) {
         let found;
         try { found = continuationPane(fleet, session, observed.get(session.machine)); }
@@ -286,6 +283,20 @@ function supervise(board, fleet, settings, report, entry, effects) {
         }
       };
       const limits = limitsOf(db, board, entry, task, session);
+      if (runRow && (session?.launch_pending || session?.launch_deferred || (!session?.session_id && !session?.pid))) {
+        const age = t - Number(session?.launched_at ?? runRow.started_at);
+        if (age >= settings.launchWindow) {
+          const reason = session?.launch_deferred_error ?? session?.launch_error ?? 'no session or process was bound to the dispatched run';
+          const error = `launch window ${settings.launchWindow}s expired after ${age}s: ${reason}`;
+          if (!session?.launch_expired_at) {
+            setRunSession(db, runRow.id, { launch_expired_at: t, launch_error: error });
+            event(db, id, 'session_launch_expired', { error, age_seconds: age, window_seconds: settings.launchWindow, key: session?.launch_key ?? null, pane: session?.pane ?? null }, runRow.id);
+          }
+          observe('spawn_failed', { error, launch_window_seconds: settings.launchWindow }, null);
+          report.failed.push([id, error]);
+        } else extendClaim(db, id, { ttl: settings.ttl, reason: 'launch_receipt_pending' });
+        return;
+      }
       // Availability is the same retained linked/unlinked feed subscribers hear.
       // A failed stream is unknown, not evidence that every remote machine left.
       if (session?.session_id && session.surface !== 'headless') {
@@ -681,7 +692,7 @@ export function closeRequested(board, report, { taskId = null, caller = null } =
       if (observed.error) throw new Error(observed.error);
       if (observed.live.has(session.pane)) {
         const pendingNotice = view(board, (db) => db.prepare("SELECT id FROM supercode_effects WHERE json_extract(effect, '$.kind')='session_notice' AND json_extract(effect, '$.runId')=? LIMIT 1").get(row.id));
-        if (pendingNotice || session.lifecycle_notice_pending) throw new Error(`close deferred: lifecycle notice ${session.lifecycle_notice_pending} has not been delivered: ${session.lifecycle_notice_error ?? 'pending'}`);
+        if (pendingNotice || session.lifecycle_notice_pending) throw new Error(`close deferred: lifecycle notice ${pendingNotice?.id ?? session.lifecycle_notice_pending} has not been delivered: ${session.lifecycle_notice_error ?? 'pending'}`);
         const protectedNow = view(board, (db) => db.prepare(`SELECT metadata FROM task_runs r WHERE r.ended_at IS NULL OR EXISTS (SELECT 1 FROM tasks t WHERE t.current_run_id = r.id)`).all())
           .some((r) => { const s = json(r.metadata, {})?.supercode; return s?.pane === session.pane && normalMachine(s.machine) === normalMachine(session.machine); });
         if (protectedNow) throw new Error(`close deferred: pane ${session.pane} is owned by an unended run`);
@@ -767,6 +778,12 @@ function observeIdle(board, fleet, settings, report, entry, effects) {
   }
 }
 
+function effectMessageKey(effect) {
+  if (effect.messageKey) return effect.messageKey;
+  const identity = JSON.stringify([effect.kind, effect.id, effect.runId, effect.sourceEvent, effect.durableId, effect.verb, effect.text]);
+  return `board-effect-${createHash('sha256').update(identity).digest('hex')}`;
+}
+
 /** The verb drains this durable notice immediately; the dispatcher retries after a failure. */
 export async function sessionNotice(board, e) {
   const prior = view(board, (db) => json(db.prepare('SELECT metadata FROM task_runs WHERE id=?').get(e.runId)?.metadata, {})?.supercode);
@@ -781,7 +798,7 @@ export async function sessionNotice(board, e) {
   let error = null, receipt = null;
   try {
     if (!e.address) throw new Error('lifecycle notice awaits the owned launch session identity');
-    receipt = await send(boardAddress(board.label), board.label, e.address, e.text, e.messageKey ?? `board-run-notice-${e.id}-${e.durableId}`);
+    receipt = await send(boardAddress(board.label), board.label, e.address, e.text, effectMessageKey(e));
     if (!receipt.delivered) error = receipt.reason ?? 'notice queued; native transcript confirmation pending'; }
   catch (failure) { error = String(failure.message).slice(-500); }
   tx(board, (db) => {
@@ -802,7 +819,7 @@ export async function sessionEffect(board, e) {
     let receipt;
     try {
       if (!address) throw new Error('actor session identity unavailable');
-      receipt = await send(boardAddress(board.label), board.label, address, e.text, `board-actor-${e.id}-${e.durableId}`);
+      receipt = await send(boardAddress(board.label), board.label, address, e.text, effectMessageKey(e));
     } catch (error) { receipt = { delivered: false, reason: error.message }; }
     const kind = receipt.delivered ? 'actor_notice_delivered' : receipt.queued ? 'actor_notice_queued' : 'actor_notice_failed';
     tx(board, db => {
@@ -951,8 +968,9 @@ export function reconcileCallerLaunch(board, sid) {
 }
 
 function reconcileLaunches(board, fleet, report, entry, effects, settings, lanes) {
-  const deferred=view(board,db=>db.prepare("SELECT t.id FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
-  for(const {id} of deferred) {
+  const deferred=view(board,db=>db.prepare("SELECT t.id,r.started_at FROM tasks t JOIN task_runs r ON r.id=t.current_run_id WHERE r.ended_at IS NULL AND json_extract(r.metadata,'$.supercode.launch_deferred')=1").all());
+  for(const {id,started_at} of deferred) {
+    if (now() - started_at >= settings.launchWindow) continue;
     try {
       const task=getTaskNow(board,id),status=view(board,db=>statusOf(db,task)),spec=runSpec(entry,status,task);
       const profile=spec&&view(board,db=>evaluate(entry.workflow.roles?.[spec.role]??'card.assignee',scopeOf(db,board,task,{params:entry.params})));

package/board/engine.mjs

@@ -710,11 +710,21 @@ const hours = (s) => s / 3600;
  * same thing can go wrong, and supercode's own where only its sessions can): each `{ kind, severity, title, data,
  * actions }`, worst first.
  */
-export function diagnose(db, task, { failureLimit = DEFAULT_FAILURE_LIMIT, strandedSeconds = 30 * 60, blockedHours = 24, fleet = null } = {}) {
+export function diagnose(db, task, { failureLimit = DEFAULT_FAILURE_LIMIT, strandedSeconds = 30 * 60, blockedHours = 24, fleet = null, launchWindowSeconds = 300 } = {}) {
   const out = [];
   const t = now();
   const events = all(db, 'SELECT kind, payload, created_at FROM task_events WHERE task_id = ? ORDER BY id', task.id);
   const failures = Number(task.consecutive_failures || 0);
+  const active = task.current_run_id ? one(db, 'SELECT id,started_at,metadata FROM task_runs WHERE id=?', task.current_run_id) : null;
+  const session = json(active?.metadata, {})?.supercode;
+  if (active && (session?.launch_pending || session?.launch_deferred || (!session?.session_id && !session?.pid))) {
+    const age = t - Number(session?.launched_at ?? active.started_at);
+    const reason = session?.launch_deferred_error ?? session?.launch_error ?? 'no session or process has been bound';
+    out.push({ kind: 'session_launch_pending', severity: age >= launchWindowSeconds ? 'error' : 'warning',
+      title: `run ${active.id} has no confirmed session after ${age}s: ${reason}`,
+      data: { run_id: active.id, age_seconds: age, window_seconds: launchWindowSeconds, reason, launch_key: session?.launch_key ?? null, pane: session?.pane ?? null },
+      actions: [`supercode workflow log ${task.id}`] });
+  }
   if (failures >= Math.max(1, (task.max_retries ?? failureLimit) - 1) && task.status !== 'done') {
     out.push({ kind: 'repeated_failures', severity: task.status === 'blocked' ? 'error' : 'warning', title: `${failures} consecutive failures`,
       data: { failures, last_error: task.last_failure_error }, actions: [`supercode workflow reclaim ${task.id}`, `supercode workflow reassign ${task.id} <profile>`] });

package/board/rows.mjs

@@ -5,7 +5,7 @@ const quote = s => `"${s.replaceAll('"', '""')}"`;
 const RUNTIME = new Set(['claim_lock','claim_expires','worker_pid','last_heartbeat_at','current_run_id','consecutive_failures','last_failure_error','block_recurrences','session_id','workspace_path']);
 const OMIT = new Set(['tasks','supercode_cards','task_links','sqlite_sequence']);
 export class RuntimeRows {
-  constructor(db) { this.db=db; this.revision=null; this.tables=new Map(); this.runtime={}; this.clocks={}; this.cachedTasks={}; this.cachedCards={}; }
+  constructor(db) { this.db=db; this.revision=null; this.tables=new Map(); this.runtime={}; this.clocks={}; this.sequences={}; this.cachedTasks={}; this.cachedCards={}; }
   normalized() { return Boolean(this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='runtime_rows'").get()); }
   watch(mem) {
     mem.exec('CREATE TEMP TABLE IF NOT EXISTS _runtime_dirty (name TEXT, key TEXT)');
@@ -40,16 +40,24 @@ export class RuntimeRows {
       const value=json(row.value,null);
       if(row.name==='@runtime') { if(value)this.runtime[row.key]=value;else delete this.runtime[row.key]; }
       else if(row.name==='@clock') { if(value)this.clocks[row.key]=value;else delete this.clocks[row.key]; }
+      else if(row.name==='@sequence') { if(value)this.sequences[row.key]=value;else delete this.sequences[row.key]; }
       else if(row.name==='@task_cache') { if(value)this.cachedTasks[row.key]=value;else delete this.cachedTasks[row.key]; }
       else if(row.name==='@card_cache') { if(value)this.cachedCards[row.key]=value;else delete this.cachedCards[row.key]; }
       else this.insert(mem,row.name,value,row.key);
     }
+    if (mem.prepare("SELECT 1 FROM sqlite_master WHERE name='sqlite_sequence'").get()) {
+      for (const [name, value] of Object.entries(this.sequences)) {
+        const sequence = mem.prepare('SELECT seq FROM sqlite_sequence WHERE name=?').get(name);
+        if (sequence) mem.prepare('UPDATE sqlite_sequence SET seq=MAX(seq,?) WHERE name=?').run(value.seq,name);
+        else mem.prepare('INSERT INTO sqlite_sequence(name,seq) VALUES(?,?)').run(name,value.seq);
+      }
+    }
     this.revision=Number(this.db.prepare("SELECT value FROM state WHERE key='rows_revision'").get()?.value??0);
     return true;
   }
   changes(mem,{all=false}={}) {
     const dirty=all?[...this.tables.values()].flatMap(t=>mem.prepare(`SELECT ${t.keys.map(quote).join(',')} FROM ${quote(t.name)}`).all().map(row=>({name:t.name,key:JSON.stringify(t.keys.map(k=>row[k]))}))):mem.prepare('SELECT DISTINCT name,key FROM _runtime_dirty').all();
-    return dirty.map(({name,key})=>{
+    const changes = dirty.map(({name,key})=>{
       const table=this.tables.get(name);
       const row=mem.prepare(`SELECT * FROM ${quote(name)} WHERE ${table.keys.map(k=>`${quote(k)} IS ?`).join(' AND ')}`).get(...JSON.parse(key))??null;
       if(name==='tasks')return {name:'@runtime',key:JSON.parse(key)[0],value:row&&Object.fromEntries(Object.entries(row).filter(([k])=>RUNTIME.has(k))),source:{name,key,row}};
@@ -72,6 +80,12 @@ export class RuntimeRows {
       if(name==='@clock')return JSON.stringify(source.row)!==JSON.stringify(this.cachedCards[key]??null);
       return false;
     });
+    if (mem.prepare("SELECT 1 FROM sqlite_master WHERE name='sqlite_sequence'").get()) {
+      for (const row of mem.prepare('SELECT name,seq FROM sqlite_sequence').all()) {
+        if (Number(this.sequences[row.name]?.seq) !== Number(row.seq)) changes.push({name:'@sequence',key:row.name,value:row,source:{name:'sqlite_sequence',key:JSON.stringify([row.name]),row}});
+      }
+    }
+    return changes;
   }
   mirror(changes, revision) {
     const previous=Number(this.db.prepare("SELECT value FROM state WHERE key='read_revision'").get()?.value??-1);
@@ -108,6 +122,7 @@ export class RuntimeRows {
       }
       if(name==='@runtime') { if(value)this.runtime[key]=value;else delete this.runtime[key]; }
       if(name==='@clock') { if(value)this.clocks[key]=value;else delete this.clocks[key]; }
+      if(name==='@sequence') { if(value)this.sequences[key]=value;else delete this.sequences[key]; }
     }
     for(const {name,sql} of this.tables.values())if(!OMIT.has(name))this.db.prepare('INSERT OR IGNORE INTO runtime_schema VALUES(?,?)').run(name,sql);
     this.db.prepare("INSERT INTO state VALUES('rows_revision',?) ON CONFLICT(key) DO UPDATE SET value=excluded.value").run(String(revision));

package/board/workflow.mjs

@@ -338,7 +338,7 @@ function queueTransitionNotice(db, board, entry, task, from, name, payload, befo
   const scope = scopeOf(db, board, after, { params: entry.params, event: { ...payload, verb: name, from, to }, extra: { run: { id: beforeRun.id } } });
   const text = render(entry.workflow.notify?.worker?.run_changed ?? '{{card.id}}: {{event.verb}} changed run {{run.id}} ({{event.from}} → {{event.to}}). Stop work under the previous run; follow the board’s current state. {{event.reason ?? ""}}', scope);
   const source = one(db, 'SELECT MAX(id) AS id FROM task_events WHERE task_id = ?', task.id)?.id;
-  const effect = { kind: 'session_notice', id: task.id, runId: beforeRun.id, address, text, sourceEvent: source, verb: name };
+  const effect = { kind: 'session_notice', id: task.id, runId: beforeRun.id, address, text, sourceEvent: source, verb: name, messageKey: `board-run-notice-${randomBytes(16).toString('hex')}` };
   const inserted = run(db, 'INSERT INTO supercode_effects (effect) VALUES (?)', JSON.stringify(effect));
   effects.push({ ...effect, durableId: Number(inserted.lastInsertRowid) });
   const current = json(one(db, 'SELECT metadata FROM task_runs WHERE id = ?', beforeRun.id)?.metadata, {}) ?? {};
@@ -791,7 +791,7 @@ const ACTIONS = {
     ctx.effects.push({ kind: 'board_message', id: ctx.id });
   },
   send_to_actor(ctx, text) {
-    const effect = { kind: 'send_to_actor', id: ctx.id, runId: ctx.runId, text };
+    const effect = { kind: 'send_to_actor', id: ctx.id, runId: ctx.runId, text, messageKey: `board-actor-${randomBytes(16).toString('hex')}` };
     const inserted = run(ctx.db, 'INSERT INTO supercode_effects (effect) VALUES (?)', JSON.stringify(effect));
     ctx.effects.push({ ...effect, durableId: Number(inserted.lastInsertRowid) });
   },

package/board/ztrack.mjs

@@ -625,6 +625,7 @@ class ZtrackBoard {
         if (id) sources.set(`tasks:${JSON.stringify([id])}`, {name:'tasks',key:JSON.stringify([id]),row:this.mem.prepare('SELECT * FROM tasks WHERE id=?').get(id)});
       }
       for (const {name,key,row:source} of sources.values()) {
+        if (name === 'sqlite_sequence') continue;
         if (!compat.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?").get(name)) continue;
         const table = this.rows.tables.get(name);
         if (name === 'task_events') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.4.140",
+  "version": "0.4.142",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {