@volter/supercode-orchestrator 0.5.37 → 0.5.39

package/agents.mjs

@@ -38,6 +38,11 @@ function store(root){
   CREATE TABLE IF NOT EXISTS agent_end_outbox(org TEXT NOT NULL,agent_id TEXT NOT NULL,reason TEXT NOT NULL,created_at INTEGER NOT NULL,PRIMARY KEY(org,agent_id));
   CREATE TABLE IF NOT EXISTS agent_runtime_sessions(org TEXT NOT NULL,runtime_ref TEXT NOT NULL,session_ref TEXT NOT NULL,PRIMARY KEY(org,runtime_ref));
   CREATE TABLE IF NOT EXISTS agent_runs(org TEXT NOT NULL,agent_id TEXT NOT NULL,session_ref TEXT NOT NULL,machine TEXT,run_id TEXT,profile_json TEXT,published INTEGER NOT NULL DEFAULT 0,ended_at INTEGER,end_reason TEXT,main INTEGER NOT NULL DEFAULT 1,starter_session TEXT,usage_json TEXT,PRIMARY KEY(org,session_ref));`);
+  // A declaration Teams refused is asked again a bounded number of times (synchronizeLocalAgents), then left failed.
+  const columns=new Set(db.prepare('PRAGMA table_info(agent_intents)').all().map(c=>c.name));
+  for(const [column,type] of Object.entries({declare_attempts:'INTEGER NOT NULL DEFAULT 0',declare_next_at:'INTEGER NOT NULL DEFAULT 0',declare_failed_at:'INTEGER',declare_error:'TEXT'}))if(!columns.has(column))db.exec(`ALTER TABLE agent_intents ADD COLUMN ${column} ${type}`);
+  // An end is claimed by the pass sending it (synchronizeAgentEnds).
+  if(!new Set(db.prepare('PRAGMA table_info(agent_end_outbox)').all().map(c=>c.name)).has('claimed_at'))db.exec('ALTER TABLE agent_end_outbox ADD COLUMN claimed_at INTEGER');
   return db;
 }
 const digest=value=>createHash('sha256').update(value).digest('hex');
@@ -86,13 +91,16 @@ export function synchronizeAgentEnds(root,org,{context}={}){
   try{
     for(const row of db.prepare('SELECT * FROM agent_end_outbox WHERE org=? ORDER BY created_at LIMIT 100').all(org)){
       if(db.prepare("SELECT 1 FROM agent_assignments WHERE org=? AND agent_id=? AND state='active'").get(org,row.agent_id)){db.prepare('DELETE FROM agent_end_outbox WHERE org=? AND agent_id=?').run(org,row.agent_id);continue;}
-      if(db.prepare('SELECT 1 FROM agent_runs WHERE org=? AND agent_id=? AND published=0').get(org,row.agent_id)){pending.push({agent:row.agent_id,error:'session ownership publication remains pending'});continue;}
+      if(db.prepare('SELECT 1 FROM agent_runs WHERE org=? AND agent_id=? AND published<=0').get(org,row.agent_id)){pending.push({agent:row.agent_id,error:'session ownership publication remains pending'});continue;}
+      // claimed by compare-and-set on its claimed_at (free, or a claim older than SYNC_CLAIM_MS from a pass that died);
+      // deleted only after Teams ended the agent (the end is idempotent by its digest), handed back when it failed
+      if(db.prepare('UPDATE agent_end_outbox SET claimed_at=? WHERE org=? AND agent_id=? AND (claimed_at IS NULL OR claimed_at<?)').run(Date.now(),org,row.agent_id,Date.now()-SYNC_CLAIM_MS).changes!==1)continue;
       try{
         agentDoor('end',[row.agent_id,row.reason,digest(`${org}/${row.agent_id}/end/${row.created_at}`)],{context});
         db.prepare('UPDATE agent_runs SET ended_at=COALESCE(ended_at,?),end_reason=COALESCE(end_reason,?) WHERE org=? AND agent_id=?').run(Date.now(),row.reason,org,row.agent_id);
         db.prepare('DELETE FROM agent_end_outbox WHERE org=? AND agent_id=?').run(org,row.agent_id);
       }
-      catch(error){pending.push({agent:row.agent_id,error:error.message});}
+      catch(error){db.prepare('UPDATE agent_end_outbox SET claimed_at=NULL WHERE org=? AND agent_id=?').run(org,row.agent_id);pending.push({agent:row.agent_id,error:error.message});}
     }
     return pending;
   }finally{db.close();}
@@ -349,21 +357,36 @@ function adoptLocalOrg(root,scope){
     db.exec('COMMIT');
   }catch(error){try{db.exec('ROLLBACK');}catch{}throw error;}finally{db.close();}
 }
+const DECLARE_ATTEMPTS=6;
+// A sync claim outlasts one Teams call chain (each door call bounded at 25-60 s).
+const SYNC_CLAIM_MS=600000;
 /** Declare every locally minted identity once Teams is reachable, under the id its sessions already carry. */
 export function synchronizeLocalAgents({root,profiles,context,machine=null,declarations={}}){
   const scope=agentScope(context);if(scope.offline)return [];
   adoptLocalOrg(root,scope);
   const db=store(root);let rows;
-  try{rows=db.prepare('SELECT * FROM agent_intents WHERE org=? AND declared=0 AND declaration_json IS NOT NULL ORDER BY created_at').all(scope.org);}finally{db.close();}
+  try{rows=db.prepare('SELECT * FROM agent_intents WHERE org=? AND declared=0 AND declaration_json IS NOT NULL AND declare_failed_at IS NULL AND declare_next_at<=? ORDER BY created_at').all(scope.org,Date.now());}finally{db.close();}
   const pending=[],base={root,org:scope.org,server:scope.server,profiles,context,machine};
   for(const row of rows){
     const declaration=JSON.parse(row.declaration_json);
+    // claimed by compare-and-set on its own next attempt time, set ahead only while it is due: a second pass skips it
+    const claim=store(root);let claimed;
+    try{claimed=claim.prepare('UPDATE agent_intents SET declare_next_at=? WHERE org=? AND id=? AND declared=0 AND declare_failed_at IS NULL AND declare_next_at<=?').run(Date.now()+SYNC_CLAIM_MS,row.org,row.id,Date.now()).changes===1;}finally{claim.close();}
+    if(!claimed)continue;
     try{
       if(declaration.kind==='root')ensureRootAgent({...base,profile:declaration.profile});
       else if(declaration.kind==='card')ensureCardAgent({...base,profile:declaration.profile,card:declaration.card,role:declaration.role,sponsor:declaration.sponsor,starterSession:declaration.starterSession});
       else if(declaration.kind==='profile')ensureProfileAgent({...base,profile:declaration.profile});
       else if(declaration.kind==='declared')ensureProfileAgent({...base,profile:declaration.profile,agent:declaration.agent,declarations});
-    }catch(error){if(!teamsUnavailable(error))pending.push({agent:row.id,error:error.message});else return [...pending,{agent:row.id,error:error.message}];}
+    }catch(error){
+      // Teams unreachable is no answer about this declaration: the rest wait for the next pass, and no attempt is counted.
+      if(teamsUnavailable(error)){const db=store(root);try{db.prepare('UPDATE agent_intents SET declare_next_at=? WHERE org=? AND id=?').run(row.declare_next_at??0,row.org,row.id);}finally{db.close();}return [...pending,{agent:row.id,error:error.message}];}
+      // A refusal is this declaration's own: asked again after a doubling wait (one minute, at most an hour), and after
+      // DECLARE_ATTEMPTS refusals left failed, said once, and never asked again.
+      const made=Number(row.declare_attempts??0)+1,failed=made>=DECLARE_ATTEMPTS,db=store(root);
+      try{db.prepare('UPDATE agent_intents SET declare_attempts=?,declare_next_at=?,declare_failed_at=?,declare_error=? WHERE org=? AND id=?').run(made,Date.now()+Math.min(3600000,60000*2**(made-1)),failed?Date.now():null,String(error.message).slice(0,500),row.org,row.id);}finally{db.close();}
+      pending.push({agent:row.id,error:error.message,attempts:made,failed});
+    }
   }
   return pending;
 }
@@ -377,7 +400,7 @@ export function recordNativeAgentSession(root,runtimeRef,session) {
     db.exec('BEGIN IMMEDIATE');
     db.prepare('INSERT INTO agent_runtime_sessions(org,runtime_ref,session_ref) VALUES (?,?,?) ON CONFLICT(org,runtime_ref) DO UPDATE SET session_ref=excluded.session_ref').run(owner.org,runtimeRef,session);
     // A published native session remains history; an unpublished lease-only row is not a session.
-    db.prepare('DELETE FROM agent_runs WHERE org=? AND session_ref=? AND published=0').run(owner.org,runtimeRef);
+    db.prepare('DELETE FROM agent_runs WHERE org=? AND session_ref=? AND published<=0').run(owner.org,runtimeRef);
     db.exec('COMMIT');
   }catch(error){try{db.exec('ROLLBACK');}catch{}throw error;}finally{db.close();}
   return agentRunForSession(root,[runtimeRef]);
@@ -421,7 +444,13 @@ export function recordAgentRun(root,org,id,{session,machine=null,run,profile=nul
     db.exec('COMMIT');
   }catch(error){try{db.exec('ROLLBACK');}catch{}throw error;}finally{db.close();}
 }
-export function pendingAgentRuns(root,org){const db=store(root);try{return db.prepare('SELECT * FROM agent_runs WHERE org=? AND published=0 ORDER BY rowid LIMIT 100').all(org).filter(row=>!(retryOf(root,org,row)?.next_at>Date.now()));}finally{db.close();}}
+// A run waiting to be joined has published=0; a pass joining it holds published=-(the time it claimed it), by
+// compare-and-set, and a claim older than SYNC_CLAIM_MS (a pass that died) is free again.
+export function pendingAgentRuns(root,org){const db=store(root);try{return db.prepare('SELECT * FROM agent_runs WHERE org=? AND (published=0 OR (published<0 AND -published<?)) ORDER BY rowid LIMIT 100').all(org,Date.now()-SYNC_CLAIM_MS).filter(row=>!(retryOf(root,org,row)?.next_at>Date.now()));}finally{db.close();}}
+/** Claim one waiting run for this pass; false when another pass holds it. */
+export function claimAgentRun(root,org,row){const db=store(root);try{return db.prepare('UPDATE agent_runs SET published=? WHERE org=? AND session_ref=? AND published=?').run(-Date.now(),org,row.session_ref,row.published).changes===1;}finally{db.close();}}
+/** Give a claimed run back, unjoined. */
+export function releaseAgentRun(root,org,row){const db=store(root);try{db.prepare('UPDATE agent_runs SET published=0 WHERE org=? AND session_ref=? AND published<0').run(org,row.session_ref);}finally{db.close();}}
 /** Session references come from the bound runtime, never from a model's requested agent id. */
 export function agentRunForSession(root, references) {
   const db = store(root);
@@ -468,7 +497,7 @@ export function runBudgetRefusal(root,agent,budgets,now){
   return null;
 }
 /** A run whose session Teams never had and that no longer runs here can never be owned there: it leaves the sync. */
-export function abandonAgentRun(root,org,session){const db=store(root);try{db.prepare('UPDATE agent_runs SET published=2 WHERE org=? AND session_ref=? AND published=0').run(org,session);db.prepare('DELETE FROM agent_run_retries WHERE org=? AND session_ref=?').run(org,session);}finally{db.close();}}
+export function abandonAgentRun(root,org,session){const db=store(root);try{db.prepare('UPDATE agent_runs SET published=2 WHERE org=? AND session_ref=? AND published<=0').run(org,session);db.prepare('DELETE FROM agent_run_retries WHERE org=? AND session_ref=?').run(org,session);}finally{db.close();}}
 export function publishAgentRun(root,org,session){const db=store(root);try{db.prepare('UPDATE agent_runs SET published=1 WHERE org=? AND session_ref=?').run(org,session);}finally{db.close();}}
 export function agentPreamble(agent,{main=true,previous=[]}={}){
   const {identity:a,intended,assignments=[]}=agent;

package/bin/machine-fleet.mjs

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+// One fleet reading for a synchronous board process: each named machine's panes and the harness session under each live
+// pane, every machine read at once through its own door, each bounded by its own deadline. Prints one row per machine:
+// { machine, panes, activities } when its door answered, { machine, error, code } when it did not.
+import { openMachine } from '@volter/teams/machine-door';
+import { StreamRpc, harnessTransport } from '@volter/teams/client';
+import { SupercodeHarnessClient } from '@volter/supercode-harness-sdk';
+let input = ''; for await (const chunk of process.stdin) input += chunk;
+const { machines, deadlineMs } = JSON.parse(input);
+const within = (promise, ms) => Promise.race([promise, new Promise((_, reject) => setTimeout(() => reject(Object.assign(new Error(`no answer within ${ms} ms`), { code: 'timeout' })), ms))]);
+async function read({ name, at }) {
+  let channel;
+  try {
+    channel = await openMachine({ machine: at });
+    const panes = await new StreamRpc(await channel.open('node')).call('harness.v1.panes.list', {});
+    const live = (Array.isArray(panes) ? panes : []).filter((pane) => !pane.dead && pane.pid);
+    let activities = [];
+    if (live.length) {
+      const client = new SupercodeHarnessClient({ transport: harnessTransport(channel) });
+      try { await client.start?.(); activities = (await client.request('harness.v1.sessions.activity_under', { pids: live.map((pane) => pane.pid) }))?.activities ?? []; }
+      finally { await client.close?.(); }
+    }
+    return { machine: name, panes, activities };
+  } finally { channel?.close(); }
+}
+const rows = await Promise.all(machines.map((machine) => within(read(machine), deadlineMs)
+  .catch((error) => ({ machine: machine.name, error: String(error?.message ?? error).slice(0, 300), code: error?.code ?? 'unavailable' }))));
+process.stdout.write(`${JSON.stringify(rows)}\n`);
+process.exit(0);

package/board/cli.mjs

@@ -15,7 +15,7 @@ import { actorOf, view, backingPaths, layOut, backingOf, boardExists, boardPath,
 import { isLocalWorkspace } from './workspace.mjs';
 import { openZtrackBoard } from './ztrack.mjs';
 import * as engine from './engine.mjs';
-import { requestAdoption, alertManagers, boardAddress, boardLabel, boardSettings, callerSession, commandEnv, reconcileCallerLaunch, closeRequested, lanesOf, localMachine, observeFleet, paused, plan, startCard, stopDispatching, supercodeBin, tick } from './dispatch.mjs';
+import { requestAdoption, alertManagers, boardAddress, boardLabel, boardSettings, callerSession, commandEnv, reconcileCallerLaunch, closeRequested, lanesOf, localMachine, observeFleet, paused, plan, startCard, stopDispatching, supercodeBin, tick, deliver, startDelivery } from './dispatch.mjs';
 import { watchBoard } from './wake.mjs';
 import { streamBoardPublications } from './event-stream.mjs';
 import { completionOwnerOf, isLatestReviewer, loadWorkflow, requestSessionClose, scopeOf, statusOf, useWorkflow, workflowEntry, workflowFor } from './workflow.mjs';
@@ -950,6 +950,14 @@ export async function runWorkflow(argv) {
       process.stdout.write(`${lines.join('\n')}\n`);
       return 0;
     }
+    case 'deliver': {
+      // the delivery pass: due effects, board mail, subscription notices and the agent records' Teams sync (serve
+      // starts it after each round in a process of its own; never part of a round)
+      const reports = await deliver({ root, slugs: args.board ? [slug] : null });
+      if (args.json) say(args, '', reports);
+      else for (const r of reports) process.stdout.write(`${r.board}: ${r.error ? `error: ${r.error}` : `${r.effects_waiting ?? 0} effect(s) waiting, ${r.effects_failed ?? 0} failed for good, ${r.notified.length} notified`}\n`);
+      return reports.some((r) => r.error) ? 1 : 0;
+    }
     case 'daemon': case 'serve': return serve(root, args);
 
     case 'gc': {
@@ -1440,7 +1448,7 @@ async function serve(root, args = {}) {
   const running = installed();
   log(`dispatching the boards of ${root} from ${localMachine()} (orchestrator ${running ?? 'unknown'})`);
   events = watchBoard({ root, binary: supercodeBin(), env: commandEnv(), log });
-  log('dispatch wakes on board writes, document edits, board mail and session catalogue events; interval is a safety sweep');
+  log('dispatch wakes on board writes, document edits and board mail; the interval sweep reads each machine\'s own door');
   const stuck = {};
   const warned = {};
   let pausedSaid = false;
@@ -1488,6 +1496,8 @@ async function serve(root, args = {}) {
         interval = settings.interval;
         const reports = await tick({ root, lanes: state.lanes, settings, params, deploymentReceipt: true });
         round += 1;
+        // what the round recorded is delivered by a pass of its own, which this loop never waits for
+        if (!stopped) startDelivery(root, { log: (level, line) => { if (level === 'warn' || args.verbose) log(line); } });
         if (!stopped) await watchRounds(reports);
         for (const r of reports) {
           nextMailAt = Math.min(nextMailAt, r.next_mail_at ?? Infinity);