@volter/supercode-teams 0.2.22 → 0.2.24

package/bin/workspace/sync-status.mjs

@@ -0,0 +1,20 @@
+// How `supercode teams status` and `supercode auth status` show what this machine's connector could not sync
+// (connector/index.mjs readSyncStatus): the sessions not syncing and why, most recent first, and its last error.
+const SHOWN = 10;
+const when = at => Number.isFinite(at) ? new Date(at).toISOString().replace(/\.\d{3}Z$/, 'Z') : 'unknown time';
+const reason = item => `${item.code ? `${item.code}: ` : ''}${item.message}`;
+
+/** Human lines for a connector's sync status, each without a trailing newline; `indent` prefixes the detail lines. */
+export function syncStatusLines(sync, { indent = '  ' } = {}) {
+  if (!sync) return [];
+  const lines = [];
+  const problems = sync.not_syncing ?? [];
+  if (problems.length) {
+    lines.push(`Not syncing (${problems.length} session${problems.length === 1 ? '' : 's'}, most recent first):`);
+    for (const item of problems.slice(0, SHOWN)) lines.push(`${indent}${item.harness} ${item.session_id}${item.rule_id ? ` · ${item.rule_id}` : ''} · ${when(item.at)} · ${reason(item)}`);
+    if (problems.length > SHOWN) lines.push(`${indent}… and ${problems.length - SHOWN} more (--json lists them all)`);
+  } else lines.push('Not syncing  none');
+  if (sync.queued_captures) lines.push(`Queued   ${sync.queued_captures} capture${sync.queued_captures === 1 ? '' : 's'} waiting for the server`);
+  if (sync.last_error) lines.push(`Last error ${when(sync.last_error.at)} · ${reason(sync.last_error)}`);
+  return lines;
+}

package/bin/workspace.mjs

@@ -9,7 +9,8 @@ import { WorkspaceContexts, fromContext, loginContext } from '../context.mjs';
 import { productionIds } from '@volter/teams/server/core';
 import { makeToken } from '@volter/teams/server/auth';
 import { moveWorkspace } from '../workspace-move.mjs';
-import { TeamsConnector } from '../connector/index.mjs';
+import { TeamsConnector, readSyncStatus } from '../connector/index.mjs';
+import { syncStatusLines } from './workspace/sync-status.mjs';
 import { CONTINUATION_FIDELITY, CONTINUATION_HARNESSES } from '../connector/native-continuation.mjs';
 import { TeamsMachine } from '../machine.mjs';
 import { teamsHome, paths } from '@volter/teams/home';
@@ -205,7 +206,7 @@ async function authCommand({ flags, sub = 'status', rest, home, contexts, output
     if (account.context) {
       const command = authTeamCommand(home, account.context, flags, output, env);
       const enrollment = command.state().read();
-      account.machine = enrollment?.machine ? { id: enrollment.machine.id, name: enrollment.machine.name, folder: enrollment.roots?.[0]?.path ?? null, service: enrollment.service?.kind ?? null, connection: null } : null;
+      account.machine = enrollment?.machine ? { id: enrollment.machine.id, name: enrollment.machine.name, folder: enrollment.roots?.[0]?.path ?? null, service: enrollment.service?.kind ?? null, connection: null, sync: readSyncStatus(command.state().directory) ?? { not_syncing: [], last_error: null, queued_captures: 0 } } : null;
       try { await command.client.me(); account.reachable = true; } catch { account.reachable = false; }
       if (account.machine && account.reachable) account.machine.connection = (await command.team.machines.get(account.machine.id).catch(() => null))?.data?.connection ?? null;
     }
@@ -214,6 +215,9 @@ async function authCommand({ flags, sub = 'status', rest, home, contexts, output
     if (!account.context) return say(account.others.length ? `Team     none current · saved: ${account.others.join(', ')} (supercode auth switch NAME)` : 'Team     none (supercode login)');
     say(`Team     ${account.team} · ${new URL(account.origin).host}${account.reachable ? '' : ' · unreachable'}${account.others.length ? ` · also: ${account.others.join(', ')} (supercode auth switch)` : ''}`);
     say(account.machine ? `Machine  ${account.machine.name} · ${account.machine.connection ?? 'unknown'} · sharing ${account.machine.folder ?? '?'}${account.machine.service ? ` (${account.machine.service})` : ' (runs while `teams connect` does)'}` : 'Machine  not connected (supercode teams connect)');
+    // Only trouble (a session not syncing, an error in the last day) is worth lines here; `teams status` shows the rest.
+    const sync = account.machine?.sync;
+    if (sync?.not_syncing.length || sync?.last_error?.at > Date.now() - 86_400_000) for (const line of syncStatusLines(sync)) say(line);
     return;
   }
   if (sub === 'switch') {
@@ -625,7 +629,19 @@ async function syncCommand(sub, { flags, rest, output, client, context, team, ho
 
 async function connectorCommand(verb, { flags, rest, output, client, context, team, contextName }, home) {
   requireArgs(rest, 0, 0, `teams ${verb}`); const state = new WorkspaceContexts(home).connector(context.server_id, context.team_id);
-  if (verb === 'status') { let reachable = true; let account = null; try { account = await client.me(); } catch { reachable = false; } const saved = state.read(); const running = saved?.process?.socket ? await activeSocket(saved.process.socket) : false; return output({ version: 1, context: scope(context), data: { reachable, account, connector: saved ? { enrolled: true, running, machine: saved.machine, source: saved.source } : { enrolled: false, running: false }, sources: reachable ? (await team.sources.list()).items : [] } }); }
+  if (verb === 'status') {
+    let reachable = true; let account = null; try { account = await client.me(); } catch { reachable = false; }
+    const saved = state.read(); const running = saved?.process?.socket ? await activeSocket(saved.process.socket) : false;
+    // What the connector could not sync, as it last wrote it to its state (it is written while it runs).
+    const sync = saved ? readSyncStatus(state.directory) ?? { not_syncing: [], last_error: null, queued_captures: 0 } : null;
+    const value = { version: 1, context: scope(context), data: { reachable, account, connector: saved ? { enrolled: true, running, machine: saved.machine, source: saved.source, sync } : { enrolled: false, running: false }, sources: reachable ? (await team.sources.list()).items : [] } };
+    if (flags.json) return output(value);
+    const lines = [`Team       ${context.team_name ?? context.team_id} · ${new URL(context.origin).host}${reachable ? '' : ' · unreachable'}`];
+    lines.push(saved?.machine ? `Connector  ${saved.machine.name ?? saved.machine.id} · ${running ? 'running' : 'not running'}${saved.service?.kind ? ` (${saved.service.kind})` : ''}` : 'Connector  not connected (supercode teams connect)');
+    lines.push(...syncStatusLines(sync));
+    process.stdout.write(`${lines.join('\n')}\n`);
+    return;
+  }
   if (verb === 'connect') {
     let saved = state.read();
     const path = realpathSync(resolve(flags.cwd ?? '.'));

package/connector/index.d.ts

@@ -57,10 +57,17 @@ export interface ConnectorConfigurationValue {
   authorization_revision: number;
 }
 
+export interface SyncErrorSummary { code: string | null; message: string; at: number }
+export interface SyncProblem extends SyncErrorSummary { harness: string; session_id: string; rule_id: string | null }
+/** What `supercode teams status` shows: sessions not syncing (most recent first) and the last error; null without state. */
+export declare function readSyncStatus(stateDir: string): { not_syncing: SyncProblem[]; last_error: SyncErrorSummary | null; queued_captures: number } | null;
+
 export declare class TeamsConnector {
   constructor(options: TeamsConnectorOptions);
   readonly lastError?: unknown;
   configuration(): Promise<ConnectorConfigurationValue>;
+  /** Resolves when the first sync pass after `start` ends (rejects if it failed; the connector keeps running). `start` returns after the first heartbeat, without waiting for it. */
+  firstPass?: Promise<void> | null;
   start(options?: { bindings?: () => unknown | Promise<unknown>; roots?: () => Array<{ id: string; machine_id: string; name: string; harnesses: HarnessId[]; path?: string; workspace?: string; homes?: unknown }> | Promise<Array<{ id: string; machine_id: string; name: string; harnesses: HarnessId[]; path?: string; workspace?: string; homes?: unknown }>> }): Promise<ConnectorConfigurationValue>;
   capture(options: CaptureOptions): Promise<CaptureReceipt | null>;
   flush(): Promise<CaptureReceipt | null>;

package/connector/index.mjs

@@ -5,6 +5,49 @@ import { canonicalJson } from '../contracts/index.mjs';
 import { EffectConsumer } from './effects.mjs';
 import { NativeEffects } from './native-effects.mjs';
 
+const SYNC_PROBLEM_LIMIT=20;
+// Live follows are bounded. The harness SDK lets at most 64 subscription opens await replies at once (sdk/typescript
+// core.mjs MAX_PENDING_RUNTIME_OPENS; more reject with SUBSCRIPTION_OPEN_CAPACITY) and every live follow is a follower
+// the service polls, while a subtree rule can cover hundreds of sessions. Only the most recently active sessions (at
+// most FOLLOW_LIMIT, half the SDK's limit for headroom, active within FOLLOW_ACTIVE_WINDOW_MS) are followed live; the
+// rest are covered by each beat's stamp-based capture. Followers rotate as activity changes.
+const FOLLOW_LIMIT=32,FOLLOW_ACTIVE_WINDOW_MS=30*60*1000,FOLLOW_RETRY_MS=60*1000;
+const CAPACITY_CODE='SUBSCRIPTION_OPEN_CAPACITY';
+const capacityError=error=>error?.code===CAPACITY_CODE||/Too many unresolved subscriptions/.test(String(error?.message??''));
+// An error of the connection itself (the outbox, authentication, the server), not of one session.
+const connectionError=error=>Boolean(error?.outbox||error?.status===401||error?.status===403||error?.status>=500);
+const errorSummary=(error,at)=>error==null?null:{code:typeof error.code==='string'?error.code:null,message:String(error.message??error).slice(0,500),at};
+/** The sync status a connector keeps in its state directory (see TeamsConnector#syncStatus); null when it has none. */
+export function readSyncStatus(stateDir){
+  let state;try{state=JSON.parse(readFileSync(join(stateDir,'connector.json'),'utf8'));}catch{return null;}
+  const status=state?.sync_status;
+  return {not_syncing:Array.isArray(status?.problems)?status.problems:[],last_error:status?.last_error??null,queued_captures:Array.isArray(state?.outbox)?state.outbox.length:0};
+}
+// A tool call still running when the core reads a session has no result yet; the core answers it with a synthesized
+// placeholder (crates/interchange session/helpers.rs ensure_tool_results_paired: a `tool` message for that call id,
+// right after the assistant message that made the call, marked `sc.tool_error`, whose only distinct trait is its
+// text), and the real result replaces it at the same position once it is written. So a `none` rule that first sees
+// a session mid tool call keeps back history whose last positions will change. Rather than match the placeholder's
+// text, the kept-back history is identified by its shape: the trailing run of tool results and the call ids they
+// answer (recorded when the rule starts). A later rewrite confined to that run, answering the same calls of the
+// unchanged assistant message before it, is the same history settling: it stays kept back and is never published.
+const trailingToolResults=messages=>{
+  let from=messages.length;while(from>0&&messages[from-1].value?.role==='tool'&&typeof messages[from-1].value.tool_call_id==='string')from--;
+  return from===messages.length||from===0?null:{from,call_ids:messages.slice(from).map(message=>message.value.tool_call_id)};
+};
+// The kept-back digests after such a settled rewrite, or null when the kept-back history changed in any other way.
+// A baseline recorded before its tool tail was (baseline_tool_tail absent) takes the tail from the current messages.
+function settledToolTail(previous,messages){
+  const baseline=previous.baseline,count=baseline.length,digestOf=message=>hash(canonicalJson(message.value));
+  if(messages.length<count||baseline.some((native_key,index)=>messages[index].native_key!==native_key))return null;
+  const tail=previous.baseline_tool_tail??trailingToolResults(messages.slice(0,count));
+  if(!tail||!Number.isSafeInteger(tail.from)||tail.from<1||tail.from+tail.call_ids.length!==count)return null;
+  for(let index=0;index<tail.from;index++)if(digestOf(messages[index])!==previous.baseline_digests?.[index])return null;
+  const run=messages.slice(tail.from,count),ids=run.map(message=>message.value?.role==='tool'?message.value.tool_call_id:null);
+  const calls=new Set((messages[tail.from-1].value?.tool_calls??[]).map(call=>call?.id).filter(id=>typeof id==='string'));
+  if(ids.some(id=>typeof id!=='string'||!calls.has(id))||new Set(ids).size!==ids.length||[...ids].sort().join('\0')!==[...tail.call_ids].sort().join('\0'))return null;
+  return messages.slice(0,count).map(digestOf);
+}
 const hash=value=>createHash('sha256').update(value).digest('hex');
 const recordedAt=entry=>{if(Number.isSafeInteger(entry.recorded_at))return entry.recorded_at;if(Number.isSafeInteger(entry.timestamp))return entry.timestamp;const value=entry.metadata?.timestamp??entry.value?.metadata?.timestamp??entry.timestamp;if(typeof value!=='string')return null;const parsed=Date.parse(value);return Number.isSafeInteger(parsed)?parsed:null;};
 const serverOrigin=value=>{const url=new URL(value),loopback=url.hostname==='localhost'||url.hostname==='::1'||url.hostname==='[::1]'||/^127(?:\.\d{1,3}){3}$/.test(url.hostname);if(!['http:','https:'].includes(url.protocol)||url.username||url.password||url.search||url.hash||(url.pathname!==''&&url.pathname!=='/')||(url.protocol==='http:'&&!loopback))throw new TypeError('server must be an HTTPS origin (HTTP is loopback-only)');return url.origin;};
@@ -24,8 +67,24 @@ function atomic(path, value) {
   }
 }
 export class TeamsConnector {
-  constructor({server,teamId,machineId,sourceId,credential,stateDir,core,hostKeyId=null,effectExecutor=null,clock={nowMs:()=>Date.now()},fetch:fetchImpl=globalThis.fetch}){Object.assign(this,{server:serverOrigin(server),teamId,machineId,sourceId,credential,stateDir,core,hostKeyId,effectExecutor,clock,fetch:fetchImpl});mkdirSync(stateDir,{recursive:true,mode:0o700});chmodSync(stateDir,0o700);this.statePath=join(stateDir,'connector.json');this.state=existsSync(this.statePath)?JSON.parse(readFileSync(this.statePath,'utf8')):{version:1,sequence:0,origins:{},outbox:[]};this.state.effects??={};this.state.incarnation_id=`inc_${randomBytes(16).toString('hex')}`;this.abort=new AbortController();this.work=Promise.resolve();this.nativeEffects=new NativeEffects(this);this.effects=new EffectConsumer(this,effectExecutor?{prepare:async effect=>({execute:()=>effectExecutor(effect,{core:this.core,signal:this.abort.signal})})}:this.nativeEffects);this.save();}
-  save(){atomic(this.statePath,this.state);} async request(method,path,body){const json=body===undefined?undefined:JSON.stringify(body),mutation=['POST','PATCH','DELETE'].includes(method);let response;for(let attempt=0;;attempt++){response=await this.fetch(this.server+`/api/v1/teams/${this.teamId}`+path,{method,redirect:'manual',headers:{Authorization:`Bearer ${this.credential}`,'Content-Type':'application/json',...(mutation?{'Idempotency-Key':hash(`${method}\n${path}\n${json??''}`)}:{})},body:json,signal:this.abort.signal});
+  constructor({server,teamId,machineId,sourceId,credential,stateDir,core,hostKeyId=null,effectExecutor=null,clock={nowMs:()=>Date.now()},fetch:fetchImpl=globalThis.fetch}){Object.assign(this,{server:serverOrigin(server),teamId,machineId,sourceId,credential,stateDir,core,hostKeyId,effectExecutor,clock,fetch:fetchImpl});mkdirSync(stateDir,{recursive:true,mode:0o700});chmodSync(stateDir,0o700);this.statePath=join(stateDir,'connector.json');this.state=existsSync(this.statePath)?JSON.parse(readFileSync(this.statePath,'utf8')):{version:1,sequence:0,origins:{},outbox:[]};this.state.effects??={};if(Array.isArray(this.state.sync_status?.problems))this.state.sync_status.problems=this.state.sync_status.problems.filter(item=>item?.code!==CAPACITY_CODE);this.state.incarnation_id=`inc_${randomBytes(16).toString('hex')}`;this.abort=new AbortController();this.work=Promise.resolve();this.nativeEffects=new NativeEffects(this);this.effects=new EffectConsumer(this,effectExecutor?{prepare:async effect=>({execute:()=>effectExecutor(effect,{core:this.core,signal:this.abort.signal})})}:this.nativeEffects);this.save();}
+  save(){atomic(this.statePath,this.state);}
+  // What `supercode teams status` shows about this connector: its last error, and the sessions it could not capture
+  // (most recent first, bounded; an entry clears when that session next captures). They are kept in connector.json,
+  // so another process reads them while this one runs; the error itself stays in memory as lastError.
+  get lastError(){return this.lastErrorValue;}
+  set lastError(error){this.lastErrorValue=error;this.syncStatus().last_error=errorSummary(error,this.clock.nowMs());this.saveQuietly();}
+  syncStatus(){const status=this.state.sync_status??={problems:[],last_error:null};status.problems??=[];return status;}
+  saveQuietly(){try{this.save();}catch{}}
+  noteSessionProblem({harness,nativeSessionId,ruleId=null},error){
+    const status=this.syncStatus(),at=this.clock.nowMs();
+    status.problems=[{harness,session_id:nativeSessionId,rule_id:ruleId,...errorSummary(error,at)},...status.problems.filter(item=>!(item.harness===harness&&item.session_id===nativeSessionId))].slice(0,SYNC_PROBLEM_LIMIT);
+    this.lastError=error;
+  }
+  clearSessionProblem(harness,nativeSessionId){
+    const status=this.state.sync_status;if(!status?.problems?.some(item=>item.harness===harness&&item.session_id===nativeSessionId))return;
+    status.problems=status.problems.filter(item=>!(item.harness===harness&&item.session_id===nativeSessionId));this.saveQuietly();
+  } async request(method,path,body){const json=body===undefined?undefined:JSON.stringify(body),mutation=['POST','PATCH','DELETE'].includes(method);let response;for(let attempt=0;;attempt++){response=await this.fetch(this.server+`/api/v1/teams/${this.teamId}`+path,{method,redirect:'manual',headers:{Authorization:`Bearer ${this.credential}`,'Content-Type':'application/json',...(mutation?{'Idempotency-Key':hash(`${method}\n${path}\n${json??''}`)}:{})},body:json,signal:this.abort.signal});
     // A rate limit is waited out (the server says how long; the same idempotency key makes the retry safe), so a
     // large first sync slows down instead of failing its pass and restarting the connector.
     if(response.status!==429||attempt>=5)break;// The server names the wait in error.details (wire.mjs validationFailure); a Retry-After header is the fallback.
@@ -37,34 +96,53 @@ export class TeamsConnector {
     if(!Array.isArray(configuration?.rules))return;
     const active=new Set(configuration.rules.filter(rule=>rule.state==='active').map(rule=>rule.id));let pruned=false;
     for(const origin of Object.values(this.state.origins??{}))for(const ruleId of Object.keys(origin?.capture_stamps??{}))if(!active.has(ruleId)){delete origin.capture_stamps[ruleId];pruned=true;}
+    // A session that stopped syncing under a rule no longer active is no longer that rule's problem.
+    const problems=this.state.sync_status?.problems;
+    if(problems?.some(item=>item.rule_id&&!active.has(item.rule_id))){this.state.sync_status.problems=problems.filter(item=>!item.rule_id||active.has(item.rule_id));pruned=true;}
     if(pruned)this.save();
   }
   async start({bindings=()=>[],roots=()=>[]}={}) {
     await this.configuration();
-    const heartbeat = async () => {
-      this.state.sequence += 1;
-      await this.request('POST', `/machines/${this.machineId}/heartbeat`, {
-        incarnation_id: this.state.incarnation_id, sequence: this.state.sequence,
-        bindings: [...this.nativeEffects.published(), ...await bindings()],
-        roots: this.localRoots.map(({id,machine_id,name,harnesses})=>({id,machine_id,name,harnesses})),
-        ...(this.hostKeyId ? { host_key_id: this.hostKeyId } : {}),
+    // Heartbeats are serialized (the server takes each sequence number once) and run on their own
+    // timer: a long sync pass (a large first capture) never makes a live machine read as offline.
+    let beating = Promise.resolve();
+    const heartbeat = () => {
+      const next = beating.then(async () => {
+        this.state.sequence += 1;
+        await this.request('POST', `/machines/${this.machineId}/heartbeat`, {
+          incarnation_id: this.state.incarnation_id, sequence: this.state.sequence,
+          bindings: [...this.nativeEffects.published(), ...await bindings()],
+          roots: this.localRoots.map(({id,machine_id,name,harnesses})=>({id,machine_id,name,harnesses})),
+          ...(this.hostKeyId ? { host_key_id: this.hostKeyId } : {}),
+        });
+        this.save();
       });
-      this.save();
+      beating = next.catch(() => {});
+      return next;
     };
-    const beat = async () => {
-      this.localRoots = await roots();
-      await heartbeat();
+    const pass = async () => {
       const configuration = await this.configuration();
       await this.syncConfiguredSessions(configuration, this.localRoots);
       await heartbeat();
       await this.pollEffects();
     };
-    await beat();
+    this.localRoots = await roots();
+    await heartbeat();
+    // The first sync pass runs behind `start`, so the machine's link opens without waiting for it.
     let running = false;
-    this.timer = setInterval(() => {
-      if (running || this.abort.signal.aborted) return;
+    const syncPass = () => {
+      if (running || this.abort.signal.aborted) return null;
       running = true;
-      void beat().catch(error => { this.lastError = error; }).finally(() => { running = false; });
+      const run = (async () => { this.localRoots = await roots(); await pass(); })();
+      void run.catch(error => { this.lastError = error; }).finally(() => { running = false; });
+      return run;
+    };
+    /** The first sync pass, for a caller that waits on it; the connector keeps running if it fails. */
+    this.firstPass = syncPass();
+    this.timer = setInterval(syncPass, 15000);
+    this.heartbeatTimer = setInterval(() => {
+      if (this.abort.signal.aborted) return;
+      void heartbeat().catch(error => { this.lastError = error; });
     }, 15000);
     return this.configurationValue;
   }
@@ -90,14 +168,14 @@ export class TeamsConnector {
   pollEffects() { return this.effects.poll(); }
   async syncConfiguredSessions(configuration,roots){
     if(typeof this.core.discover!=='function')return;
-    this.followers??=new Map();
+    this.followers??=new Map();this.followRetry??=new Map();this.stoppingFollowers??=new Set();
     this.nativeEffects.beginDiscovery();
     const activeRules=new Map(configuration.rules.filter(item=>item.state==='active').map(rule=>[rule.id,rule]));
-    for(const [key,follower] of this.followers){const rule=activeRules.get(follower.ruleId);if(!rule||rule.revision!==follower.ruleRevision){follower.controller.abort();this.followers.delete(key);}}
+    for(const [key,follower] of this.followers){const rule=activeRules.get(follower.ruleId);if(!rule||rule.revision!==follower.ruleRevision){this.stopFollower(key);}}
     // A rule covers its root folder and every folder under it. Core discovery filters that subtree before it
     // reads any session's tail, so a beat costs one bounded scan per root, not one over every session on the
     // machine; rules over the same root and harnesses share the scan.
-    const discoveries=new Map();
+    const discoveries=new Map(),followCandidates=[];
     for(const rule of activeRules.values()){
       const root=roots.find(item=>item.id===rule.selector.root_id);
       if(!root)continue;
@@ -125,22 +203,65 @@ export class TeamsConnector {
         // the pass. The session keeps its native binding either way, so it can still be attached to.
         let captured = true;
         if (this.sessionChanged(options)) try { await this.capture(options); } catch (error) {
-          if (error.outbox || error.status === 401 || error.status === 403 || error.status >= 500 || this.abort.signal.aborted) throw error;
-          captured = false; this.lastError = error;
+          if (connectionError(error) || this.abort.signal.aborted) throw error;
+          captured = false; this.noteSessionProblem(options, error);
           process.stderr.write(`supercode teams: ${harness} session ${nativeSessionId} not synced: ${error.message}\n`);
         }
         try { await this.nativeEffects.discover(descriptor, rule.id); } catch (error) { this.lastError = error; }
         if (!captured) continue;
-        const key=`${rule.id}:${harness}:${nativeSessionId}`;
-        if(this.followers.has(key)||typeof this.core.session(locator).follow!=='function')continue;
-        const controller=new AbortController(),stop=()=>controller.abort(this.abort.signal.reason);this.abort.signal.addEventListener('abort',stop,{once:true});
-        // A follow event may land between beats, after the rule changed: it reads the configuration afresh.
-        const followed={...options,configuration:null,stamp:undefined};
-        const task=(async()=>{for await(const _event of this.core.session(locator).follow({signal:controller.signal})){if(controller.signal.aborted)break;await this.capture(followed);}})().catch(error=>{if(!controller.signal.aborted)this.lastError=error;}).finally(()=>{this.abort.signal.removeEventListener('abort',stop);this.followers.delete(key);});
-        this.followers.set(key,{ruleId:rule.id,ruleRevision:rule.revision,controller,task});
+        followCandidates.push({key:`${rule.id}:${harness}:${nativeSessionId}`,activity:this.sessionActivity(options),options});
       }
     }
     this.nativeEffects.endDiscovery();
+    this.rotateFollowers(followCandidates);
+  }
+  // When a session was last active: its descriptor's updated_at_ms, else its native file's modification time.
+  sessionActivity({locator,descriptor}){
+    let activity=Number.isSafeInteger(descriptor?.updated_at_ms)?descriptor.updated_at_ms:null;
+    const path=locator?.storage?.kind==='file'&&typeof locator.storage.path==='string'?locator.storage.path:null;
+    if(path)try{activity=Math.max(activity??0,statSync(path).mtimeMs);}catch{}
+    return activity;
+  }
+  // Follows the FOLLOW_LIMIT most recently active candidates (active within the window) and stops every other
+  // follower. A capacity refusal is not a session's failure: the connector backs off opening follows for a while.
+  rotateFollowers(candidates){
+    const now=this.clock.nowMs(),seen=new Set(),wanted=[];
+    for(const candidate of candidates.filter(item=>item.activity!==null&&now-item.activity<=FOLLOW_ACTIVE_WINDOW_MS).sort((a,b)=>b.activity-a.activity)){
+      if(seen.has(candidate.key)||typeof this.core.session(candidate.options.locator).follow!=='function')continue;
+      seen.add(candidate.key);wanted.push(candidate);if(wanted.length>=FOLLOW_LIMIT)break;
+    }
+    for(const key of this.followers.keys())if(!seen.has(key))this.stopFollower(key);
+    for(const [key,until] of this.followRetry)if(until<=now)this.followRetry.delete(key);
+    for(const {key,options} of wanted){
+      if(this.followers.has(key)||this.followRetry.has(key))continue;
+      // A stopped follower holds its slot until it has closed, so live follows never exceed the bound.
+      if((this.followBackoffUntil??0)>now||this.followers.size+this.stoppingFollowers.size>=FOLLOW_LIMIT)break;
+      this.follow(key,options);
+    }
+  }
+  stopFollower(key){
+    const follower=this.followers.get(key);if(!follower)return;
+    follower.controller.abort();this.followers.delete(key);
+    this.stoppingFollowers.add(follower.task);void follower.task.finally(()=>this.stoppingFollowers.delete(follower.task));
+  }
+  follow(key,options){
+    const {locator}=options,controller=new AbortController(),stop=()=>controller.abort(this.abort.signal.reason);this.abort.signal.addEventListener('abort',stop,{once:true});
+    // A follow event may land between beats, after the rule changed: it reads the configuration afresh.
+    const followed={...options,configuration:null,stamp:undefined};
+    const task=(async()=>{
+      for await(const _event of this.core.session(locator).follow({signal:controller.signal})){
+        if(controller.signal.aborted)break;
+        // A capture that fails is that session's problem, as on a beat; the follow goes on.
+        try{await this.capture(followed);}catch(error){if(controller.signal.aborted)break;if(connectionError(error))this.lastError=error;else this.noteSessionProblem(followed,error);}
+      }
+    })().catch(error=>{
+      if(controller.signal.aborted)return;
+      // The follow itself failed: the beat's capture still covers the session, so it is not listed as not syncing.
+      // A capacity refusal pauses every new follow for a while; any other failure retries this one after a while.
+      if(capacityError(error))this.followBackoffUntil=this.clock.nowMs()+FOLLOW_RETRY_MS;
+      else{this.followRetry.set(key,this.clock.nowMs()+FOLLOW_RETRY_MS);this.lastError=error;}
+    }).finally(()=>{this.abort.signal.removeEventListener('abort',stop);if(this.followers.get(key)?.controller===controller)this.followers.delete(key);});
+    this.followers.set(key,{ruleId:options.ruleId,ruleRevision:options.ruleRevision,controller,task});
   }
   // What a session's native store looked like when discovery listed it: its file's size and modification
   // time (read before any load, so a write after it reads as a change), those of the file's SQLite write-ahead
@@ -170,7 +291,7 @@ export class TeamsConnector {
     this.save();
   }
   capture(options) {
-    const work=this.work.then(()=>this.captureSerial(options));
+    const work=this.work.then(()=>this.captureSerial(options)).then(receipt=>{this.clearSessionProblem(options.harness,options.nativeSessionId);return receipt;});
     this.work=work.catch(()=>{});
     return work;
   }
@@ -186,7 +307,7 @@ export class TeamsConnector {
     let messages=(loaded.messages??loaded.turns??[]).map((entry,index)=>{const value=entry.value??Object.fromEntries(Object.entries(entry).filter(([key])=>!['id','native_key','recorded_at','timestamp'].includes(key)));return{native_key:entry.id??entry.native_key??String(index),recorded_at:recordedAt(entry),value};});
     if(previous?.last_rule_revision===ruleRevision&&previous?.backfill_mode===backfillPolicy.mode&&previous?.observed_keys?.length===messages.length&&messages.every((message,index)=>message.native_key===previous.observed_keys[index]&&hash(canonicalJson(message.value))===previous.observed_digests?.[index])){this.recordStamp(key,ruleId,ruleRevision,backfillPolicy.mode,stamp);return null;}
     let effectivePrevious=previous;const clearBaseline=backfillPolicy.mode==='all';
-    if(!previous&&backfillPolicy.mode==='none'){this.state.origins[key]={generation:1,sequence:0,digest:null,baseline:messages.map(x=>x.native_key),baseline_digests:messages.map(x=>hash(canonicalJson(x.value))),observed_keys:messages.map(x=>x.native_key),observed_digests:messages.map(x=>hash(canonicalJson(x.value))),last_rule_revision:ruleRevision,backfill_mode:backfillPolicy.mode};this.recordStamp(key,ruleId,ruleRevision,backfillPolicy.mode,stamp);return null;}
+    if(!previous&&backfillPolicy.mode==='none'){const toolTail=trailingToolResults(messages);this.state.origins[key]={generation:1,sequence:0,digest:null,baseline:messages.map(x=>x.native_key),baseline_digests:messages.map(x=>hash(canonicalJson(x.value))),...(toolTail?{baseline_tool_tail:toolTail}:{}),observed_keys:messages.map(x=>x.native_key),observed_digests:messages.map(x=>hash(canonicalJson(x.value))),last_rule_revision:ruleRevision,backfill_mode:backfillPolicy.mode};this.recordStamp(key,ruleId,ruleRevision,backfillPolicy.mode,stamp);return null;}
     if(!previous&&backfillPolicy.mode==='since'){
       if(!Number.isSafeInteger(backfillPolicy.since))throw Object.assign(new Error('since backfill requires an integer timestamp'),{code:'invalid_request'});
       const excluded=messages.filter(message=>message.recorded_at==null||message.recorded_at<backfillPolicy.since);
@@ -202,7 +323,10 @@ export class TeamsConnector {
     // timestamps; a `none` rule cannot tell old history from new, so that session waits for an explicit
     // `supercode teams sync resume RULE --backfill all` (or a since window), which republishes it whole.
     if(!clearBaseline&&effectivePrevious?.baseline?.length&&effectivePrevious.baseline.every(x=>/^\d+$/.test(x))){
-      const baselineMoved=effectivePrevious.baseline.some((native_key,index)=>messages[index]?.native_key!==native_key||hash(canonicalJson(messages[index].value))!==effectivePrevious.baseline_digests?.[index]);
+      let baselineMoved=effectivePrevious.baseline.some((native_key,index)=>messages[index]?.native_key!==native_key||hash(canonicalJson(messages[index].value))!==effectivePrevious.baseline_digests?.[index]);
+      // A rule that started mid tool call kept back the core's placeholder result; the real result replaces it.
+      const settled=baselineMoved&&backfillPolicy.mode==='none'?settledToolTail(effectivePrevious,messages):null;
+      if(settled){baselineMoved=false;effectivePrevious={...effectivePrevious,baseline_digests:settled};this.state.origins[key]={...this.state.origins[key],baseline_digests:settled};this.save();}
       if(baselineMoved&&backfillPolicy.mode==='since'&&Number.isSafeInteger(backfillPolicy.since)){
         const excluded=messages.filter(message=>message.recorded_at==null||message.recorded_at<backfillPolicy.since);
         effectivePrevious={...effectivePrevious,baseline:excluded.map(message=>message.native_key),baseline_digests:excluded.map(message=>hash(canonicalJson(message.value)))};
@@ -257,6 +381,6 @@ export class TeamsConnector {
     if(!Array.isArray(configuration?.rules))return false;
     return !configuration.rules.some(rule=>rule.id===item.begin.rule_id&&rule.revision===item.begin.rule_revision&&rule.state==='active');
   }
-  async flushOutbox(){let latest=null;while(this.state.outbox.length){const item=this.state.outbox[0];let receipt;try{receipt=await this.sendCapture(item);}catch(error){if(!await this.ruleRejected(item,error))throw error;this.lastError=error;process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id}; its sync rule changed or was removed: ${error.message}\n`);this.state.outbox.shift();this.save();continue;}const key=`${item.begin.harness}:${item.begin.native_session_id}`,prior=this.state.origins[key]??{},next={...prior,generation:item.begin.generation,sequence:item.begin.sequence,digest:item.commit.digest,observed_keys:item.observed?.keys??prior.observed_keys,observed_digests:item.observed?.digests??prior.observed_digests,last_rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode};if(item.clearBaseline){delete next.baseline;delete next.baseline_digests;}if(item.stamp&&item.begin.rule_id)next.capture_stamps={...prior.capture_stamps,[item.begin.rule_id]:{stamp:item.stamp,rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode}};this.state.origins[key]=next;this.state.outbox.shift();this.save();latest=receipt;}return latest;}
-  stop(){if(this.timer)clearInterval(this.timer);this.abort.abort();for(const follower of this.followers?.values()??[])follower.controller.abort();return this.nativeEffects.close();}
+  async flushOutbox(){let latest=null;while(this.state.outbox.length){const item=this.state.outbox[0];let receipt;try{receipt=await this.sendCapture(item);}catch(error){if(!await this.ruleRejected(item,error))throw error;this.lastError=error;process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id}; its sync rule changed or was removed: ${error.message}\n`);this.state.outbox.shift();this.save();continue;}const key=`${item.begin.harness}:${item.begin.native_session_id}`,prior=this.state.origins[key]??{},next={...prior,generation:item.begin.generation,sequence:item.begin.sequence,digest:item.commit.digest,observed_keys:item.observed?.keys??prior.observed_keys,observed_digests:item.observed?.digests??prior.observed_digests,last_rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode};if(item.clearBaseline){delete next.baseline;delete next.baseline_digests;delete next.baseline_tool_tail;}if(item.stamp&&item.begin.rule_id)next.capture_stamps={...prior.capture_stamps,[item.begin.rule_id]:{stamp:item.stamp,rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode}};this.state.origins[key]=next;this.state.outbox.shift();this.save();latest=receipt;}return latest;}
+  stop(){if(this.timer)clearInterval(this.timer);if(this.heartbeatTimer)clearInterval(this.heartbeatTimer);this.abort.abort();for(const follower of this.followers?.values()??[])follower.controller.abort();return this.nativeEffects.close();}
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.2.22",
+  "version": "0.2.24",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",