@volter/supercode-teams 0.2.129 → 0.2.131

package/bin/workspace.mjs

@@ -13,6 +13,8 @@ import { moveWorkspace } from '../workspace-move.mjs';
 import { TeamsConnector, readSyncStatus } from '../connector/index.mjs';
 import { syncStatusLines } from './workspace/sync-status.mjs';
 import { CONTINUATION_FIDELITY, CONTINUATION_HARNESSES } from '../connector/native-continuation.mjs';
+import { MachineEvents } from '../machine-events.mjs';
+import { openLocalMachine, nodeRpc } from '../client.mjs';
 import { TeamsMachine } from '../machine.mjs';
 import { teamsHome, paths } from '@volter/teams/home';
 import { loadOrCreateKeyPair, publicKeyId } from '@volter/teams/identity';
@@ -564,6 +566,12 @@ async function sourcesCommand(sub, command) {
 }
 
 async function machinesCommand(sub, { flags, rest, output, client, context, team }) {
+  if (['subscribe','unsubscribe','events'].includes(sub)) {
+    requireArgs(rest,sub==='events'?0:1,sub==='events'?0:1,`teams machines ${sub} ${sub==='events'?'':'ADDRESS'}`);
+    const channel=await openLocalMachine(paths(teamsHome()).socket);
+    try { const rpc=await nodeRpc(channel); return output(await rpc.request(`machine.events.${sub}`,{team_id:context.team_id,address:rest[0]})); }
+    finally { channel.close(); }
+  }
   if (sub === 'list') return output(await team.machines.list());
   if (sub === 'roots') { requireArgs(rest, 1, 1, 'teams machines roots MACHINE'); return output(await client.request('GET', teamPath(context, `/machines/${rest[0]}/roots`))); }
   if (sub === 'authorize-continuation') {
@@ -617,7 +625,7 @@ async function machinesCommand(sub, { flags, rest, output, client, context, team
       return output(await client.request('POST', machineBase, { body: { subject_id: subject, caps, targets, expires_at: Date.now() + ttl } }));
     }
   }
-  usage('teams machines list|roots|access|authorize-continuation');
+  usage('teams machines list|roots|access|authorize-continuation|subscribe|unsubscribe|events');
 }
 
 async function syncCommand(sub, { flags, rest, output, client, context, team, home }) {
@@ -745,7 +753,9 @@ async function connectorCommand(verb, { flags, rest, output, client, context, te
     const liveCore = new SupercodeHarnessClient({ command: binary, args: ['harness', 'serve'] });
     const connector = new TeamsConnector({ server: context.origin, teamId: context.team_id, machineId: saved.machine.id, sourceId: saved.source.id, credential: saved.credential.token, stateDir: state.directory, core, largeCore, slowCore, liveCore, hostKeyId });
     // A connected machine is a person's: each pane it launches opens a window on its screen (SUPERCODE_POPUPS=0: none).
+    const machineEvents = new MachineEvents({home:daemonHome,server:context.origin,teamId:context.team_id,credential:saved.credential.token,machineName:saved.machine.name,binary});
     const machine = new TeamsMachine({ home: daemonHome, supercode: binary, popups: process.env.SUPERCODE_POPUPS !== '0', ...(flags.tmux_socket ? { tmuxSocket: String(flags.tmux_socket) } : {}), link: { server: context.origin, teamId: context.team_id, machineId: saved.machine.id, credential: saved.credential.token, incarnationId: connector.state.incarnation_id, configuration: () => connector.configuration() }, onTurn: (turn) => connector.noteTurn(turn), onTurnWithdrawn: (turn) => connector.withdrawTurn(turn) });
+    machine.machineEvents = machineEvents;
     machine.on('link', (event) => { if (event.event === 'replaced') process.stderr.write(`supercode teams: ${event.error}. Two connectors were running for this machine; stop one of them (\`supercode teams disconnect\` where it should not run).\n`); });
     // The link's state for the watch after start: down until it is up, down again (since when) when it drops. A
     // `replaced` link is left alone: another connector for this machine owns it, by design.
@@ -758,12 +768,12 @@ async function connectorCommand(verb, { flags, rest, output, client, context, te
     });
     // The connector is the service's daemon and keeps serving; a daemon that took its socket path is the one to stop.
     machine.on('superseded', (path) => process.stderr.write(`supercode teams: another machine daemon replaced this connector's socket at ${path}; stop it (supercode teams status)\n`));
-    const stopAll = async () => { await machine.stop().catch(() => {}); await connector.stop(); await core.close(); await largeCore.close(); await slowCore.close(); await liveCore.close(); await lifecycle.close(); state.update(current => current ? ({ ...current, process: undefined }) : null); };
+    const stopAll = async () => { machineEvents.stop(); await machine.stop().catch(() => {}); await connector.stop(); await core.close(); await largeCore.close(); await slowCore.close(); await liveCore.close(); await lifecycle.close(); state.update(current => current ? ({ ...current, process: undefined }) : null); };
     // A daemon with no server link holding the socket is a session's auto-start while this service was between
     // restarts (from a supercode old enough not to wait for the service): it yields to the service's connector, which
     // otherwise failed on EADDRINUSE at every restart and left this machine unlinked.
     if (saved.service) await retireLinklessDaemon(paths(daemonHome).machine);
-    try { await connector.start({ roots: () => saved.roots }); await machine.start(); } catch (error) { await stopAll(); throw error; }
+    try { await machine.start(); await connector.start({ roots: () => saved.roots }); machineEvents.start(); } catch (error) { await stopAll(); throw error; }
     output({ version: 1, context: scope(context), data: { machine: saved.machine, source: saved.source, roots: saved.roots.map(({ path: _path, ...item }) => item), socket: machine.socketPath, foreground: true } });
     // Run by a service, the connector moves to an upgrade on its own: its readers would otherwise answer from the code
     // they loaded until something restarted it (after 0.5.51 this Mac's went on reading a Codex turn waiting on an
@@ -792,6 +802,9 @@ async function connectorCommand(verb, { flags, rest, output, client, context, te
     if (saved.service) {
       const linkWatch = setInterval(() => {
         if (linkReplaced || linkDownSince === null || Date.now() - linkDownSince < LINK_LOST_MS) return;
+        // A server outage is retried in place. Restart only a broken link while
+        // HTTP heartbeats still prove that server reachable, not every two minutes of an outage.
+        if (!connector.lastHeartbeatAt || Date.now() - connector.lastHeartbeatAt > 60_000) return;
         clearInterval(linkWatch);
         process.stderr.write(`supercode teams: this machine has had no link to its Teams server for ${Math.round((Date.now() - linkDownSince) / 1000)} s; restarting the connector\n`);
         void (async () => { await stopConnector?.(); process.exit(75); })();
@@ -993,7 +1006,7 @@ function printHelp() {
   members invite|list|remove|role
   groups list|create|remove|add-member|remove-member
   sources create|list|status|pause|revoke|credentials|config
-  machines list|roots|access|authorize-continuation
+  machines list|roots|access|authorize-continuation|subscribe|unsubscribe|events
   sync add|list|show|pause|resume|remove
   sessions list|search|show|follow|access|rename|delete|url|attach|export|continue|bind|unbind
 Use --help with a command for its required arguments and options.

package/connector/index.mjs

@@ -141,7 +141,7 @@ export class TeamsConnector {
   clearSessionProblem(harness,nativeSessionId){
     const status=this.state.sync_status;if(!status?.problems?.some(item=>item.harness===harness&&item.session_id===nativeSessionId))return;
     status.problems=status.problems.filter(item=>!(item.harness===harness&&item.session_id===nativeSessionId));this.saveQuietly();
-  } async request(method,path,body){const json=body===undefined?undefined:JSON.stringify(body),mutation=['POST','PATCH','DELETE'].includes(method);let response;for(let attempt=0;;attempt++){response=await this.fetch(this.server+`/api/v1/teams/${this.teamId}`+path,{method,redirect:'manual',headers:{Authorization:`Bearer ${this.credential}`,'Content-Type':'application/json',...(mutation?{'Idempotency-Key':hash(`${method}\n${path}\n${json??''}`)}:{})},body:json,signal:this.abort.signal});
+  } async request(method,path,body){const json=body===undefined?undefined:JSON.stringify(body),mutation=['POST','PATCH','DELETE'].includes(method);let response;for(let attempt=0;;attempt++){response=await this.fetch(this.server+`/api/v1/teams/${this.teamId}`+path,{method,redirect:'manual',headers:{Authorization:`Bearer ${this.credential}`,'Content-Type':'application/json',...(mutation?{'Idempotency-Key':hash(`${method}\n${path}\n${json??''}`)}:{})},body:json,signal:AbortSignal.any([this.abort.signal, AbortSignal.timeout(30_000)])});
     // A rate limit is waited out (the server says how long; the same idempotency key makes the retry safe), so a
     // large first sync slows down instead of failing its pass and restarting the connector.
     if(response.status!==429||attempt>=5)break;// The server names the wait in error.details (wire.mjs validationFailure); a Retry-After header is the fallback.
@@ -159,7 +159,8 @@ export class TeamsConnector {
     if(pruned)this.save();
   }
   async start({bindings=()=>[],roots=()=>[]}={}) {
-    await this.configuration();
+    // Remote outages must not prevent the local machine doors from opening.
+    // Configuration and heartbeat failures are retried by their existing timers.
     // Heartbeats are serialized (the server takes each sequence number once) and run on their own
     // timer: a long sync pass (a large first capture) never makes a live machine read as offline.
     let beating = Promise.resolve();
@@ -174,6 +175,7 @@ export class TeamsConnector {
           // What each synced session's harness is doing (its turn: working, idle, …), from the last discovery.
           ...(this.reportedActivity ? { activity: this.reportedActivity } : {}),
         });
+        this.lastHeartbeatAt = this.clock.nowMs();
         // The sessions someone is following now (a viewer or channel open on them) skip the capture floors.
         if (Array.isArray(reply?.watched)) this.watched = new Set(reply.watched.map(item => `${item.harness}:${item.native_id}`));
         this.save();
@@ -188,7 +190,6 @@ export class TeamsConnector {
       await this.pollEffects();
     };
     this.localRoots = await roots();
-    await heartbeat();
     // The first sync pass runs behind `start`, so the machine's link opens without waiting for it.
     let running = false;
     const syncPass = () => {

package/machine-events.mjs

@@ -0,0 +1,79 @@
+// One retained server stream per connector. Subscriptions and pending deliveries
+// live beside the connector, so neither a restart nor a lost reply duplicates mail.
+import { mkdirSync, readFileSync, writeFileSync, renameSync } from 'node:fs';
+import { join } from 'node:path';
+import { spawn } from 'node:child_process';
+import { createHash } from 'node:crypto';
+
+export class MachineEvents {
+  constructor({home,server,teamId,credential,machineName,binary,env=process.env}) {
+    Object.assign(this,{server,teamId,credential,binary,env});
+    this.machine=machineName.split('.')[0].toLowerCase().replace(/[^a-z0-9_-]/g,'-');
+    const key=createHash('sha256').update(`${server}\0${teamId}`).digest('hex').slice(0,20);
+    const dir=join(home,'machine-events');mkdirSync(dir,{recursive:true,mode:0o700});this.file=join(dir,`${key}.json`);
+    try{this.state=JSON.parse(readFileSync(this.file,'utf8'));}catch(error){if(error.code!=='ENOENT')throw error;this.state={cursor:null,items:{},subscriptions:[],pending:[]};}
+    this.abort=new AbortController();this.connected=false;
+  }
+  save(){const temp=`${this.file}.${process.pid}.tmp`;writeFileSync(temp,JSON.stringify(this.state)+'\n',{mode:0o600});renameSync(temp,this.file);}
+  snapshot(){return {team_id:this.teamId,connected:this.connected,cursor:this.state.cursor,items:Object.values(this.state.items),subscriptions:this.state.subscriptions,pending:this.state.pending.length,error:this.error??null};}
+  subscribe(address,remove=false){
+    if(!/^sc:[^:\s]+:[^:\s]+:[^\s]+$/.test(address??'')||address.split(':')[1]!==this.machine)throw new Error('subscribe a full session or board address on this connector machine');
+    this.state.subscriptions=this.state.subscriptions.filter(value=>value!==address);
+    if(!remove)this.state.subscriptions.push(address);
+    else this.state.pending=this.state.pending.filter(item=>item.to!==address);
+    this.save();return this.snapshot();
+  }
+  start(){void this.follow();this.deliveryTimer=setInterval(()=>void this.flush(),5000);this.deliveryTimer.unref?.();}
+  stop(){this.abort.abort();clearInterval(this.deliveryTimer);}
+  async follow(){
+    while(!this.abort.signal.aborted){
+      const streamAbort=new AbortController();
+      let deadline=setTimeout(()=>streamAbort.abort(new Error('machine event stream timed out')),45000);
+      try{
+        const suffix=this.state.cursor==null?'':`?after=${this.state.cursor}`;
+        const response=await fetch(`${this.server}/api/v1/teams/${this.teamId}/machines/events${suffix}`,{headers:{Authorization:`Bearer ${this.credential}`,Accept:'text/event-stream'},redirect:'error',signal:AbortSignal.any([this.abort.signal,streamAbort.signal])});
+        if(!response.ok||!response.headers.get('content-type')?.includes('text/event-stream'))throw new Error(`machine events returned HTTP ${response.status}, expected event stream`);
+        const reader=response.body.getReader();const decoder=new TextDecoder();let buffer='';
+        try{for(;;){const {value,done}=await reader.read();if(done)break;clearTimeout(deadline);deadline=setTimeout(()=>streamAbort.abort(new Error('machine event stream timed out')),45000);buffer+=decoder.decode(value,{stream:true});if(buffer.length>2*1024*1024)throw new Error('machine event exceeded stream bound');let boundary;while((boundary=buffer.indexOf('\n\n'))>=0){const frame=buffer.slice(0,boundary);buffer=buffer.slice(boundary+2);const data=frame.split('\n').filter(line=>line.startsWith('data:')).map(line=>line.slice(5).trimStart()).join('\n');if(data)await this.receive(JSON.parse(data));}}}finally{await reader.cancel().catch(()=>{});}
+      }catch(error){if(!this.abort.signal.aborted){if(this.error!==error.message)process.stderr.write(`supercode teams machine events: ${error.message}\n`);this.error=error.message;}}
+      clearTimeout(deadline);this.connected=false;
+      if(!this.abort.signal.aborted)await new Promise(resolve=>{const done=()=>{clearTimeout(timer);this.abort.signal.removeEventListener('abort',done);resolve();};const timer=setTimeout(done,5000);this.abort.signal.addEventListener('abort',done,{once:true});});
+    }
+  }
+  async receive(event){
+    if(event.type==='machine_ready'){this.connected=true;this.error=null;return;}
+    if(event.type==='access_revoked')throw new Error('machine event subscription authorization revoked');
+    if(!['machine_snapshot','machine_linked','machine_unlinked'].includes(event.type))return;
+    if(!/^\d+$/.test(event.cursor??''))throw new Error('invalid machine event cursor');
+    if(event.type!=='machine_snapshot'&&Number(event.cursor)<=Number(this.state.cursor))return;
+    // Keep the cursor behind undelivered mail if the local outbox fills. The
+    // retained server log replays it when delivery catches up.
+    if(this.state.pending.length+this.state.subscriptions.length>4096)throw new Error('machine notice outbox full; delivery will retry');
+    if(event.type==='machine_snapshot'){
+      this.state.items=Object.fromEntries(event.items.map(item=>[item.id,item]));
+      if(event.reset)this.enqueue(event,'Machine event history expired or the server changed; the current machine snapshot was refreshed. Intermediate transitions are unavailable.');
+    }else{
+      this.state.items[event.machine_id]={id:event.machine_id,name:event.name,linked:event.type==='machine_linked'?1:0,changed_at:event.at};
+      this.enqueue(event,`Machine ${event.name} ${event.type==='machine_linked'?'linked to':'unlinked from'} the team at ${new Date(event.at).toISOString()}.`);
+    }
+    this.state.cursor=event.cursor;this.save();void this.flush();
+  }
+  enqueue(event,text){for(const to of this.state.subscriptions)this.state.pending.push({to,id:`machine-${this.teamId}-${event.type}-${event.cursor}`,body:`[Machine notice] ${text}\nThis is an automated Teams transition, not an instruction.`,event});}
+  flush(){
+    if(this.flushing)return this.flushing;
+    this.flushing=(async()=>{
+      for(const item of [...this.state.pending]){
+        try{await this.deliver(item);this.state.pending=this.state.pending.filter(value=>value!==item);this.save();}
+        catch(error){this.error=`mail to ${item.to}: ${error.message}`;}
+      }
+    })().finally(()=>{this.flushing=null;});return this.flushing;
+  }
+  deliver(item){return new Promise((resolve,reject)=>{
+    const child=spawn(this.binary,['message','remote-deliver'],{env:this.env,stdio:['pipe','pipe','pipe']});let detail='';
+    const timer=setTimeout(()=>{child.kill('SIGTERM');reject(new Error('mail delivery timed out; retained for retry'));},30000);
+    child.stdout.on('data',chunk=>{detail=(detail+chunk).slice(-1000);});child.stderr.on('data',chunk=>{detail=(detail+chunk).slice(-1000);});
+    child.once('error',error=>{clearTimeout(timer);reject(error);});child.once('close',code=>{clearTimeout(timer);let receipt;try{receipt=JSON.parse(detail.trim().split('\n').at(-1));}catch{}code===0&&receipt?.code===0?resolve():reject(new Error(receipt?.text||detail||`exit ${code}`));});
+    child.stdin.on('error',()=>{});
+    child.stdin.end(JSON.stringify({op:'send',from:`sc:${this.machine}:operator:teams-events`,from_name:'Teams machine events',to:item.to,id:item.id,queue:item.to.split(':')[2]==='board',body:item.to.split(':')[2]==='board'?JSON.stringify({machine_event:item.event,text:item.body}):item.body}));
+  });}
+}

package/machine.mjs

@@ -207,12 +207,15 @@ export class TeamsMachine extends MachineCore {
   async beforeListen() {
     mkdirSync(this.portsDir, { recursive: true }); // the port table's announce directory exists before any pane launches
     this.#startServe();
-    this.#startMailWatch();
     if (this.#pendingNotices().length) this.#watchEnds();
     this.jobs = new JobOwner({ home: this.home, env: this.env, log: (row) => this.log.append({ principal: null, cap: 'exec', target: `job:${row.job}`, outcome: row.outcome, detail: row.reason }) });
     this.jobs.reapOrphans();
   }
 
+  async afterListen() {
+    this.#startMailWatch();
+  }
+
   async beforeClose() {
     clearInterval(this.endWatch); this.endWatch = null;
     // A launch still waiting for its pane to be ready is abandoned, not hurried:
@@ -1291,6 +1294,13 @@ export class TeamsMachine extends MachineCore {
         if (!operator) throw Object.assign(new Error('listing machine voice packs requires operator authority'), { code: 403 });
         return [...this.voiceProcesses].map(([session, child]) => ({ session, pid: child.pid }));
       }
+      case 'machine.events.events':
+      case 'machine.events.subscribe':
+      case 'machine.events.unsubscribe': {
+        if (!operator) throw Object.assign(new Error('machine event subscriptions require operator authority'), {code:403});
+        if (!this.machineEvents || params.team_id!==this.machineEvents.teamId) throw new Error('this connector is not serving that team');
+        return method==='machine.events.events'?this.machineEvents.snapshot():this.machineEvents.subscribe(params.address,method.endsWith('.unsubscribe'));
+      }
       case 'describe':
         return this.describe();
       case 'panes.present': {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.2.129",
+  "version": "0.2.131",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",
@@ -62,7 +62,7 @@
     "@volter/supercode-harness-sdk": "^0.3.43",
     "@volter/supercode-terminal": "^0.2.30",
     "@volter/identity": "^0.4.1",
-    "@volter/teams": "^0.1.53",
+    "@volter/teams": "^0.1.54",
     "@xterm/addon-serialize": "^0.14.0",
     "@xterm/headless": "^6.0.0",
     "protobufjs": "8.8.0",

package/worker/index.mjs

@@ -134,7 +134,14 @@ export class TeamsServer extends DurableObject {
 }
 
 export default {
-  fetch(request, env) {
-    return env.TEAMS_SERVER.get(env.TEAMS_SERVER.idFromName('server')).fetch(request);
+  async fetch(request, env) {
+    try { return await env.TEAMS_SERVER.get(env.TEAMS_SERVER.idFromName('server')).fetch(request); }
+    catch (error) {
+      // Failures before the HTTP handler (object initialization, migrations or
+      // storage) otherwise become a platform HTML page, not our API contract.
+      const requestId = crypto.randomUUID();
+      console.error(`Teams request ${requestId} failed before response: ${error.message}`);
+      return Response.json({ version: 1, error: { code: 'unavailable', message: 'Teams server is temporarily unavailable', request_id: requestId, retryable: true } }, { status: 503, headers: { 'Retry-After': '5', 'X-Request-ID': requestId } });
+    }
   },
 };