@volter/supercode-teams 0.2.24 → 0.2.25

package/bin/workspace.mjs

@@ -865,4 +865,9 @@ Use --help with a command for its required arguments and options.
 `);
 }
 export function errorExit(error) { return error.exitCode ?? ERROR_EXITS[error.code] ?? 1; }
-if (import.meta.url === new URL(`file://${process.argv[1]}`).href) workspaceMain().catch(error => { process.stderr.write(`${JSON.stringify({ version: 1, error: { code: error.code ?? 'invalid_request', message: error.message } })}\n`); process.exit(errorExit(error)); });
+if (import.meta.url === new URL(`file://${process.argv[1]}`).href) workspaceMain().catch(error => { process.stderr.write(`${JSON.stringify({ version: 1, error: { code: error.code ?? 'invalid_request', message: error.message } })}\n`); process.exitCode = errorExit(error);
+  // Drain rather than process.exit(): on Windows, exiting while a just-used fetch connection is still closing aborts
+  // Node inside libuv (src\win\async.c, `!(handle->flags & UV_HANDLE_CLOSING)`) and loses the exit code. Whatever
+  // still holds the event loop open gets a moment, then the exit is forced.
+  setTimeout(() => process.exit(process.exitCode), 2_000).unref();
+});

package/connector/index.d.ts

@@ -36,6 +36,12 @@ export interface TeamsConnectorOptions {
   effectExecutor?: (effect: NativeEffect, context: {core: ConnectorCore; signal: AbortSignal}) => Promise<{native_session_id?: string | null; runtime_id?: string | null} | void>;
   clock?: ConnectorClock;
   fetch?: typeof globalThis.fetch;
+  /** Milliseconds between heartbeats, sent on their own timer independent of the capture pass (default 15000). */
+  heartbeatMs?: number;
+  /** A native file at least nativeIdleBytes long gets its native bundle only after this long without a write (default 60000). */
+  nativeIdleMs?: number;
+  /** Size from which a native file's bundle waits for it to be idle (default 8 MiB). */
+  nativeIdleBytes?: number;
 }
 
 export interface CaptureOptions {

package/connector/index.mjs

@@ -1,4 +1,4 @@
-import { mkdirSync, readFileSync, writeFileSync, renameSync, existsSync, chmodSync, realpathSync, statSync, openSync, closeSync, fsyncSync, unlinkSync } from 'node:fs';
+import { mkdirSync, readFileSync, writeFileSync, renameSync, existsSync, chmodSync, realpathSync, statSync, openSync, closeSync, fsyncSync, unlinkSync, readSync, readdirSync } from 'node:fs';
 import { basename, dirname, join } from 'node:path';
 import { randomBytes, createHash } from 'node:crypto';
 import { canonicalJson } from '../contracts/index.mjs';
@@ -10,12 +10,15 @@ const SYNC_PROBLEM_LIMIT=20;
 // core.mjs MAX_PENDING_RUNTIME_OPENS; more reject with SUBSCRIPTION_OPEN_CAPACITY) and every live follow is a follower
 // the service polls, while a subtree rule can cover hundreds of sessions. Only the most recently active sessions (at
 // most FOLLOW_LIMIT, half the SDK's limit for headroom, active within FOLLOW_ACTIVE_WINDOW_MS) are followed live; the
-// rest are covered by each beat's stamp-based capture. Followers rotate as activity changes.
-const FOLLOW_LIMIT=32,FOLLOW_ACTIVE_WINDOW_MS=30*60*1000,FOLLOW_RETRY_MS=60*1000;
+// rest are covered by each beat's stamp-based capture. Followers rotate as activity changes, with hysteresis: a
+// follower is kept while it stays in the window and ranks within FOLLOW_KEEP_RANK, so sessions near the cutoff do not
+// stop and reopen (each reopen a full load and capture) on alternate beats; new follows only fill up to FOLLOW_LIMIT.
+const FOLLOW_LIMIT=32,FOLLOW_KEEP_RANK=40,FOLLOW_ACTIVE_WINDOW_MS=30*60*1000,FOLLOW_RETRY_MS=60*1000;
 const CAPACITY_CODE='SUBSCRIPTION_OPEN_CAPACITY';
 const capacityError=error=>error?.code===CAPACITY_CODE||/Too many unresolved subscriptions/.test(String(error?.message??''));
 // An error of the connection itself (the outbox, authentication, the server), not of one session.
 const connectionError=error=>Boolean(error?.outbox||error?.status===401||error?.status===403||error?.status>=500);
+const sameError=(a,b)=>a!=null&&b!=null&&a.code===b.code&&a.message===b.message;
 const errorSummary=(error,at)=>error==null?null:{code:typeof error.code==='string'?error.code:null,message:String(error.message??error).slice(0,500),at};
 /** The sync status a connector keeps in its state directory (see TeamsConnector#syncStatus); null when it has none. */
 export function readSyncStatus(stateDir){
@@ -49,6 +52,15 @@ function settledToolTail(previous,messages){
   return messages.slice(0,count).map(digestOf);
 }
 const hash=value=>createHash('sha256').update(value).digest('hex');
+const CHUNK_BYTES=1024*1024;
+// A native file at least NATIVE_IDLE_BYTES long gets its native bundle only once it has gone NATIVE_IDLE_MS
+// without a write. Translating reads the whole file, takes minutes for a 100 MB transcript, and holds the
+// harness server's only request reader while it runs; a live Claude Code transcript is appended to on every
+// message and tool result, so translating it mid-turn is wasted (the file has moved by the time it finishes).
+// A minute is four beats with no write: the turn is over, or waiting on the user or one long tool call.
+// Smaller files translate in well under a second and keep getting a bundle with every capture.
+const NATIVE_IDLE_MS=60000,NATIVE_IDLE_BYTES=8*1024*1024;
+const fileStamp=stat=>`${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeMs}`;
 const recordedAt=entry=>{if(Number.isSafeInteger(entry.recorded_at))return entry.recorded_at;if(Number.isSafeInteger(entry.timestamp))return entry.timestamp;const value=entry.metadata?.timestamp??entry.value?.metadata?.timestamp??entry.timestamp;if(typeof value!=='string')return null;const parsed=Date.parse(value);return Number.isSafeInteger(parsed)?parsed:null;};
 const serverOrigin=value=>{const url=new URL(value),loopback=url.hostname==='localhost'||url.hostname==='::1'||url.hostname==='[::1]'||/^127(?:\.\d{1,3}){3}$/.test(url.hostname);if(!['http:','https:'].includes(url.protocol)||url.username||url.password||url.search||url.hash||(url.pathname!==''&&url.pathname!=='/')||(url.protocol==='http:'&&!loopback))throw new TypeError('server must be an HTTPS origin (HTTP is loopback-only)');return url.origin;};
 function atomic(path, value) {
@@ -67,19 +79,24 @@ function atomic(path, value) {
   }
 }
 export class TeamsConnector {
-  constructor({server,teamId,machineId,sourceId,credential,stateDir,core,hostKeyId=null,effectExecutor=null,clock={nowMs:()=>Date.now()},fetch:fetchImpl=globalThis.fetch}){Object.assign(this,{server:serverOrigin(server),teamId,machineId,sourceId,credential,stateDir,core,hostKeyId,effectExecutor,clock,fetch:fetchImpl});mkdirSync(stateDir,{recursive:true,mode:0o700});chmodSync(stateDir,0o700);this.statePath=join(stateDir,'connector.json');this.state=existsSync(this.statePath)?JSON.parse(readFileSync(this.statePath,'utf8')):{version:1,sequence:0,origins:{},outbox:[]};this.state.effects??={};if(Array.isArray(this.state.sync_status?.problems))this.state.sync_status.problems=this.state.sync_status.problems.filter(item=>item?.code!==CAPACITY_CODE);this.state.incarnation_id=`inc_${randomBytes(16).toString('hex')}`;this.abort=new AbortController();this.work=Promise.resolve();this.nativeEffects=new NativeEffects(this);this.effects=new EffectConsumer(this,effectExecutor?{prepare:async effect=>({execute:()=>effectExecutor(effect,{core:this.core,signal:this.abort.signal})})}:this.nativeEffects);this.save();}
+  constructor({server,teamId,machineId,sourceId,credential,stateDir,core,hostKeyId=null,effectExecutor=null,clock={nowMs:()=>Date.now()},fetch:fetchImpl=globalThis.fetch,heartbeatMs=15000,nativeIdleMs=NATIVE_IDLE_MS,nativeIdleBytes=NATIVE_IDLE_BYTES}){Object.assign(this,{server:serverOrigin(server),teamId,machineId,sourceId,credential,stateDir,core,hostKeyId,effectExecutor,clock,fetch:fetchImpl,heartbeatMs,nativeIdleMs,nativeIdleBytes});mkdirSync(stateDir,{recursive:true,mode:0o700});chmodSync(stateDir,0o700);this.statePath=join(stateDir,'connector.json');this.state=existsSync(this.statePath)?JSON.parse(readFileSync(this.statePath,'utf8')):{version:1,sequence:0,origins:{},outbox:[]};this.state.effects??={};if(Array.isArray(this.state.sync_status?.problems))this.state.sync_status.problems=this.state.sync_status.problems.filter(item=>item?.code!==CAPACITY_CODE);this.state.native_cache??={};this.state.incarnation_id=`inc_${randomBytes(16).toString('hex')}`;this.abort=new AbortController();this.work=Promise.resolve();this.nativeEffects=new NativeEffects(this);this.effects=new EffectConsumer(this,effectExecutor?{prepare:async effect=>({execute:()=>effectExecutor(effect,{core:this.core,signal:this.abort.signal})})}:this.nativeEffects);this.save();}
   save(){atomic(this.statePath,this.state);}
   // What `supercode teams status` shows about this connector: its last error, and the sessions it could not capture
   // (most recent first, bounded; an entry clears when that session next captures). They are kept in connector.json,
   // so another process reads them while this one runs; the error itself stays in memory as lastError.
   get lastError(){return this.lastErrorValue;}
-  set lastError(error){this.lastErrorValue=error;this.syncStatus().last_error=errorSummary(error,this.clock.nowMs());this.saveQuietly();}
+  // The same error (code and message) again changes nothing on disk, so it is not saved again: an error that repeats
+  // every beat does not rewrite connector.json every beat. Its time stays that of its first occurrence in the run.
+  set lastError(error){this.lastErrorValue=error;const status=this.syncStatus(),summary=errorSummary(error,this.clock.nowMs());if(sameError(status.last_error,summary))return;status.last_error=summary;this.saveQuietly();}
   syncStatus(){const status=this.state.sync_status??={problems:[],last_error:null};status.problems??=[];return status;}
   saveQuietly(){try{this.save();}catch{}}
   noteSessionProblem({harness,nativeSessionId,ruleId=null},error){
-    const status=this.syncStatus(),at=this.clock.nowMs();
-    status.problems=[{harness,session_id:nativeSessionId,rule_id:ruleId,...errorSummary(error,at)},...status.problems.filter(item=>!(item.harness===harness&&item.session_id===nativeSessionId))].slice(0,SYNC_PROBLEM_LIMIT);
-    this.lastError=error;
+    const status=this.syncStatus(),at=this.clock.nowMs(),summary=errorSummary(error,at);
+    // The same session failing the same way on every beat keeps its entry where it is.
+    const listed=status.problems.some(item=>item.harness===harness&&item.session_id===nativeSessionId&&item.rule_id===ruleId&&sameError(item,summary));
+    if(!listed)status.problems=[{harness,session_id:nativeSessionId,rule_id:ruleId,...summary},...status.problems.filter(item=>!(item.harness===harness&&item.session_id===nativeSessionId))].slice(0,SYNC_PROBLEM_LIMIT);
+    this.lastErrorValue=error;const known=sameError(status.last_error,summary);if(!known)status.last_error=summary;
+    if(!listed||!known)this.saveQuietly();
   }
   clearSessionProblem(harness,nativeSessionId){
     const status=this.state.sync_status;if(!status?.problems?.some(item=>item.harness===harness&&item.session_id===nativeSessionId))return;
@@ -143,7 +160,7 @@ export class TeamsConnector {
     this.heartbeatTimer = setInterval(() => {
       if (this.abort.signal.aborted) return;
       void heartbeat().catch(error => { this.lastError = error; });
-    }, 15000);
+    }, this.heartbeatMs);
     return this.configurationValue;
   }
   async effectFetch(method, path, {body, contentType='application/json', raw=false, freshKey=false}={}) {
@@ -175,7 +192,7 @@ export class TeamsConnector {
     // A rule covers its root folder and every folder under it. Core discovery filters that subtree before it
     // reads any session's tail, so a beat costs one bounded scan per root, not one over every session on the
     // machine; rules over the same root and harnesses share the scan.
-    const discoveries=new Map(),followCandidates=[];
+    const discoveries=new Map(),followCandidates=[],discovered=new Set();
     for(const rule of activeRules.values()){
       const root=roots.find(item=>item.id===rule.selector.root_id);
       if(!root)continue;
@@ -195,6 +212,7 @@ export class TeamsConnector {
       for(const descriptor of descriptors){
         const locator=descriptor.locator,nativeSessionId=locator?.session_id,harness=locator?.harness;
         if(!locator||!nativeSessionId||!harness)continue;
+        discovered.add(`${harness}:${nativeSessionId}`);
         const options={locator,descriptor,harness,nativeSessionId,ruleId:rule.id,ruleRevision:rule.revision,backfill:rule.backfill,rootPath:workspace,configuration};
         // A session whose native store has not changed since this rule last captured it is not reloaded.
         // One session that cannot be captured (a transcript that rewrote its history needs an explicit resync)
@@ -213,6 +231,7 @@ export class TeamsConnector {
       }
     }
     this.nativeEffects.endDiscovery();
+    this.pruneNativeCache(discovered);
     this.rotateFollowers(followCandidates);
   }
   // When a session was last active: its descriptor's updated_at_ms, else its native file's modification time.
@@ -222,14 +241,16 @@ export class TeamsConnector {
     if(path)try{activity=Math.max(activity??0,statSync(path).mtimeMs);}catch{}
     return activity;
   }
-  // Follows the FOLLOW_LIMIT most recently active candidates (active within the window) and stops every other
-  // follower. A capacity refusal is not a session's failure: the connector backs off opening follows for a while.
+  // Follows the most recently active candidates (active within the window): a follower is kept while it ranks
+  // within FOLLOW_KEEP_RANK, every other one stops, and the FOLLOW_LIMIT most recent open new follows while fewer
+  // than FOLLOW_LIMIT are live. A capacity refusal is not a session's failure: the connector backs off for a while.
   rotateFollowers(candidates){
-    const now=this.clock.nowMs(),seen=new Set(),wanted=[];
+    const now=this.clock.nowMs(),seen=new Set(),ranked=[];
     for(const candidate of candidates.filter(item=>item.activity!==null&&now-item.activity<=FOLLOW_ACTIVE_WINDOW_MS).sort((a,b)=>b.activity-a.activity)){
       if(seen.has(candidate.key)||typeof this.core.session(candidate.options.locator).follow!=='function')continue;
-      seen.add(candidate.key);wanted.push(candidate);if(wanted.length>=FOLLOW_LIMIT)break;
+      seen.add(candidate.key);ranked.push(candidate);if(ranked.length>=FOLLOW_KEEP_RANK)break;
     }
+    const wanted=ranked.slice(0,FOLLOW_LIMIT);
     for(const key of this.followers.keys())if(!seen.has(key))this.stopFollower(key);
     for(const [key,until] of this.followRetry)if(until<=now)this.followRetry.delete(key);
     for(const {key,options} of wanted){
@@ -282,8 +303,77 @@ export class TeamsConnector {
     if(stamp===null||this.state.outbox.length)return true;
     const rule=options.configuration?.rules?.find(item=>item.id===options.ruleId&&item.revision===options.ruleRevision&&item.state==='active');if(!rule)return true;
     const configured=rule.backfill??options.backfill??'all',mode=typeof configured==='string'?configured:configured.mode;
-    const seen=this.state.origins[`${options.harness}:${options.nativeSessionId}`]?.capture_stamps?.[options.ruleId];
-    return !(seen&&seen.stamp===stamp&&seen.rule_revision===options.ruleRevision&&seen.backfill_mode===mode);
+    const key=`${options.harness}:${options.nativeSessionId}`,origin=this.state.origins[key],seen=origin?.capture_stamps?.[options.ruleId];
+    if(!(seen&&seen.stamp===stamp&&seen.rule_revision===options.ruleRevision&&seen.backfill_mode===mode))return true;
+    // Published without its native bundle while it was live: captured again once it has been idle long enough.
+    if(origin?.native_deferred&&options.locator?.storage?.kind==='file')try{return this.nativeReady(key,statSync(realpathSync(options.locator.storage.path)));}catch{return false;}
+    return false;
+  }
+  // Whether a capture may attach the native bundle of a file in this state: its translation is cached (and did
+  // not fail), or it is small, or it has been idle for nativeIdleMs.
+  nativeReady(key,stat){
+    const cached=this.state.native_cache[key];
+    if(cached?.stamp===fileStamp(stat))return !cached.failed;
+    return stat.size<this.nativeIdleBytes||this.clock.nowMs()-stat.mtimeMs>=this.nativeIdleMs;
+  }
+  // Native artifact bytes are spilled to content-addressed files under the state directory, never into
+  // connector.json: a 129 MB transcript was ~172 MB of base64 held in the outbox and rewritten on every save().
+  spillDir(){return join(this.stateDir,'artifacts');}
+  spill(bytes){
+    const sha256=hash(bytes),name=`${sha256}.bin`,path=join(this.spillDir(),name);
+    if(!existsSync(path)){
+      mkdirSync(this.spillDir(),{recursive:true,mode:0o700});
+      const temporary=`${path}.${process.pid}.${randomBytes(12).toString('hex')}.tmp`,fd=openSync(temporary,'wx',0o600);
+      try{writeFileSync(fd,bytes);fsyncSync(fd);}catch(error){closeSync(fd);try{unlinkSync(temporary);}catch{}throw error;}
+      closeSync(fd);renameSync(temporary,path);
+    }
+    const chunk_sha256=[];for(let at=0;at<bytes.length||chunk_sha256.length===0;at+=CHUNK_BYTES)chunk_sha256.push(hash(bytes.subarray(at,Math.min(bytes.length,at+CHUNK_BYTES))));
+    return {spill:name,sha256,byte_length:bytes.length,chunk_sha256};
+  }
+  openSpill(name){return openSync(join(this.spillDir(),name),'r');}
+  // Each chunk of a spilled artifact, read back one at a time; null when the file is gone or no longer matches.
+  *spilledChunks(artifact){
+    let fd;try{fd=this.openSpill(artifact.spill.file);}catch{yield null;return;}
+    try{
+      for(let index=0;index<artifact.manifest.chunks;index++){
+        const length=Math.max(0,Math.min(CHUNK_BYTES,artifact.manifest.byte_length-index*CHUNK_BYTES)),bytes=Buffer.alloc(length);
+        const read=length?readSync(fd,bytes,0,length,index*CHUNK_BYTES):0;
+        if(read!==length||hash(bytes)!==artifact.spill.chunk_sha256[index]){yield null;return;}
+        yield {upload_id:artifact.manifest.upload_id,index,sha256:artifact.spill.chunk_sha256[index],bytes_base64:bytes.toString('base64')};
+      }
+    }finally{closeSync(fd);}
+  }
+  // Spill files no queued capture and no cached translation refers to are removed.
+  pruneSpill(){
+    const kept=new Set();
+    for(const item of this.state.outbox)for(const artifact of item.artifacts??[])if(artifact.spill)kept.add(artifact.spill.file);
+    for(const cached of Object.values(this.state.native_cache))for(const file of cached.files??[])kept.add(file.spill);
+    let names=[];try{names=readdirSync(this.spillDir());}catch{return;}
+    for(const name of names)if(!kept.has(name)&&!name.endsWith('.tmp'))try{unlinkSync(join(this.spillDir(),name));}catch{}
+  }
+  // A cached translation of a session no active rule discovers any more (deleted, moved, its rule gone) is dropped
+  // with its spill files, so a deleted 129 MB transcript's bundle does not stay on disk for good.
+  pruneNativeCache(discovered){
+    // A beat that discovers nothing while translations are cached (a home on a drive that is briefly offline) says
+    // nothing about those sessions, so it keeps them rather than re-translating every large session later.
+    if(!discovered.size&&Object.keys(this.state.native_cache).length)return;
+    const stale=Object.keys(this.state.native_cache).filter(key=>!discovered.has(key));if(!stale.length)return;
+    for(const key of stale)delete this.state.native_cache[key];
+    this.save();this.pruneSpill();
+  }
+  // A translated bundle, spilled, in the form the cache keeps: one entry per file, primary first or not.
+  spillTranslated(translated,defaultName){
+    const files=translated.files?.length?translated.files:[{path:translated.suggested_filename??defaultName,content:translated.content,role:'primary'}];
+    if(files.filter(file=>file.role==='primary').length!==1)throw Object.assign(new Error('native codec bundle must contain exactly one primary file'),{code:'fidelity_unavailable'});
+    let total=0;const spilled=[];
+    for(const file of files){const bytes=Buffer.from(file.content,'utf8');total+=bytes.length;if(total>256*1024*1024)throw Object.assign(new Error('native artifact bundle exceeds 256 MiB'),{code:'request_too_large'});spilled.push({path:file.path,role:file.role,...this.spill(bytes)});}
+    return {fidelity:translated.fidelity,residue:translated.residue??[],files:spilled};
+  }
+  // The outbox artifacts and native_bundle of one capture from a spilled bundle; every capture gets fresh upload ids.
+  bundleArtifacts(bundle,harness){
+    const artifacts=[],files=[];
+    for(const file of bundle.files){const uploadId=`upload_${randomBytes(16).toString('hex')}`;artifacts.push({manifest:{upload_id:uploadId,relative_name:file.path,sha256:file.sha256,byte_length:file.byte_length,chunks:file.chunk_sha256.length,media_type:'application/octet-stream',role:file.role==='source_recovery'?'sidecar':'native'},spill:{file:file.spill,chunk_sha256:file.chunk_sha256}});files.push({upload_id:uploadId,role:file.role});}
+    return {artifacts,nativeBundle:{id:files.find(file=>file.role==='primary').upload_id,harness,fidelity:bundle.fidelity,residue:bundle.residue,files}};
   }
   recordStamp(key,ruleId,ruleRevision,backfillMode,stamp){
     const origin=this.state.origins[key];
@@ -305,7 +395,11 @@ export class TeamsConnector {
     const loaded=await this.core.session(locator).load(),configuredBackfill=rule.backfill??backfill??'all',backfillPolicy=typeof configuredBackfill==='string'?{mode:configuredBackfill,since:null}:configuredBackfill;
     const key=`${harness}:${nativeSessionId}`,previous=this.state.origins[key]??null;
     let messages=(loaded.messages??loaded.turns??[]).map((entry,index)=>{const value=entry.value??Object.fromEntries(Object.entries(entry).filter(([key])=>!['id','native_key','recorded_at','timestamp'].includes(key)));return{native_key:entry.id??entry.native_key??String(index),recorded_at:recordedAt(entry),value};});
-    if(previous?.last_rule_revision===ruleRevision&&previous?.backfill_mode===backfillPolicy.mode&&previous?.observed_keys?.length===messages.length&&messages.every((message,index)=>message.native_key===previous.observed_keys[index]&&hash(canonicalJson(message.value))===previous.observed_digests?.[index])){this.recordStamp(key,ruleId,ruleRevision,backfillPolicy.mode,stamp);return null;}
+    // A session published without its native bundle is captured again, with the same messages, once the bundle can be built.
+    const nativeWanted=backfillPolicy.mode==='all'&&typeof this.core.translateSession==='function';
+    const retryNative=Boolean(previous?.native_deferred&&nativeWanted&&artifactBefore?.isFile()&&!previous?.baseline&&this.nativeReady(key,artifactBefore));
+    const unchanged=previous?.last_rule_revision===ruleRevision&&previous?.backfill_mode===backfillPolicy.mode&&previous?.observed_keys?.length===messages.length&&messages.every((message,index)=>message.native_key===previous.observed_keys[index]&&hash(canonicalJson(message.value))===previous.observed_digests?.[index]);
+    if(unchanged&&!retryNative){this.recordStamp(key,ruleId,ruleRevision,backfillPolicy.mode,stamp);return null;}
     let effectivePrevious=previous;const clearBaseline=backfillPolicy.mode==='all';
     if(!previous&&backfillPolicy.mode==='none'){const toolTail=trailingToolResults(messages);this.state.origins[key]={generation:1,sequence:0,digest:null,baseline:messages.map(x=>x.native_key),baseline_digests:messages.map(x=>hash(canonicalJson(x.value))),...(toolTail?{baseline_tool_tail:toolTail}:{}),observed_keys:messages.map(x=>x.native_key),observed_digests:messages.map(x=>hash(canonicalJson(x.value))),last_rule_revision:ruleRevision,backfill_mode:backfillPolicy.mode};this.recordStamp(key,ruleId,ruleRevision,backfillPolicy.mode,stamp);return null;}
     if(!previous&&backfillPolicy.mode==='since'){
@@ -330,7 +424,9 @@ export class TeamsConnector {
       if(baselineMoved&&backfillPolicy.mode==='since'&&Number.isSafeInteger(backfillPolicy.since)){
         const excluded=messages.filter(message=>message.recorded_at==null||message.recorded_at<backfillPolicy.since);
         effectivePrevious={...effectivePrevious,baseline:excluded.map(message=>message.native_key),baseline_digests:excluded.map(message=>hash(canonicalJson(message.value)))};
-        this.state.origins[key]={...this.state.origins[key],baseline:effectivePrevious.baseline,baseline_digests:effectivePrevious.baseline_digests};
+        // A tool tail recorded for the old baseline describes positions that no longer hold; it goes with it.
+        delete effectivePrevious.baseline_tool_tail;
+        this.state.origins[key]={...this.state.origins[key],baseline:effectivePrevious.baseline,baseline_digests:effectivePrevious.baseline_digests};delete this.state.origins[key].baseline_tool_tail;
       }else if(baselineMoved)throw Object.assign(new Error(`the history kept back when rule ${ruleId} started was rewritten; run \`supercode teams sync resume ${ruleId} --backfill all\` (or a since window) to republish this session`),{code:'resync_required'});
     }
     if(effectivePrevious?.baseline&&!clearBaseline)messages=messages.filter(x=>!effectivePrevious.baseline.includes(x.native_key));
@@ -338,29 +434,55 @@ export class TeamsConnector {
     const captureId=`cap_${randomBytes(16).toString('hex')}`,generation=effectivePrevious?.generation??1,sequence=(effectivePrevious?.sequence??0)+1;
     const begin={native_session_id:nativeSessionId,harness,generation,sequence,capture_id:captureId,previous_digest:effectivePrevious?.digest??null,visibility,rule_id:ruleId,rule_revision:ruleRevision};
     const parts=[];for(let index=0;index<messages.length;index+=500)parts.push({capture_id:captureId,index:parts.length,messages:messages.slice(index,index+500)});
-    const artifacts=[];
+    let artifacts=[];
     // Native stores normally live in harness homes outside the repository. The locator is
     // accepted only when it came from this configured core discovery, and raw bytes are safe
     // to publish only when the complete native history is in the permitted projection.
-    let nativeBundle=null;
+    let nativeBundle=null,nativeDeferred=false;
     // A session in a shared SQLite store (OpenCode, Goose) has no file of its own to stat; it is
     // stable when the core reads back the same messages this capture publishes.
     const sqliteSource=locator?.storage?.kind==='sqlite';
-    if((artifactSource||sqliteSource)&&backfillPolicy.mode==='all'&&!effectivePrevious?.baseline&&typeof this.core.translateSession==='function'){
-      if(artifactSource&&!artifactBefore.isFile())throw Object.assign(new Error('native artifact locator is not a regular file'),{code:'invalid_request'});
-      const translated=(await this.core.translateSession({locator,target_harness:harness})).artifact;
-      const changed=new Error('native session changed while it was being captured');
-      if(artifactSource){const after=statSync(artifactSource);if(artifactBefore.dev!==after.dev||artifactBefore.ino!==after.ino||artifactBefore.size!==after.size||artifactBefore.mtimeMs!==after.mtimeMs)throw Object.assign(changed,{code:'resync_required'});}
-      else{const digest=entries=>hash(canonicalJson((entries.messages??entries.turns??[]).map(entry=>entry.value??entry)));if(digest(await this.core.session(locator).load())!==digest(loaded))throw Object.assign(changed,{code:'resync_required'});}
-      const translatedFiles=translated.files?.length?translated.files:[{path:translated.suggested_filename??(artifactSource?basename(artifactSource):`${nativeSessionId}.json`),content:translated.content,role:'primary'}];
-      if(translatedFiles.filter(file=>file.role==='primary').length!==1)throw Object.assign(new Error('native codec bundle must contain exactly one primary file'),{code:'fidelity_unavailable'});
-      let total=0;const bundleFiles=[];
-      for(const file of translatedFiles){const bytes=Buffer.from(file.content,'utf8');total+=bytes.length;if(total>256*1024*1024)throw Object.assign(new Error('native artifact bundle exceeds 256 MiB'),{code:'request_too_large'});const uploadId=`upload_${randomBytes(16).toString('hex')}`,chunks=[];for(let at=0,index=0;at<bytes.length||index===0;at+=1024*1024,index++){const chunk=bytes.subarray(at,Math.min(bytes.length,at+1024*1024));chunks.push({upload_id:uploadId,index,sha256:hash(chunk),bytes_base64:chunk.toString('base64')});if(bytes.length===0)break;}artifacts.push({manifest:{upload_id:uploadId,relative_name:file.path,sha256:hash(bytes),byte_length:bytes.length,chunks:chunks.length,media_type:'application/octet-stream',role:file.role==='source_recovery'?'sidecar':'native'},chunks});bundleFiles.push({upload_id:uploadId,role:file.role});}
-      nativeBundle={id:bundleFiles.find(file=>file.role==='primary').upload_id,harness,fidelity:translated.fidelity,residue:translated.residue??[],files:bundleFiles};
+    // Translating reads the whole native file, so its time grows with the file: a 129 MB Claude Code transcript
+    // outlasted the client's 30 s default on every beat and never synced. About 2 s per MB, from 30 s to 10 min.
+    const translateTimeoutMs=Math.min(600000,Math.max(30000,Math.ceil((artifactBefore?.size??0)/1e6)*2000));
+    if(artifactSource&&nativeWanted){
+      if(!artifactBefore.isFile())throw Object.assign(new Error('native artifact locator is not a regular file'),{code:'invalid_request'});
+      // A file source's bundle is best effort: when it cannot be attached (the file is live, it moved while it was
+      // read, the rule's kept-back baseline is still being cleared, or translating it failed) the messages are
+      // published without it and nativeDeferred brings the session back once the file is idle. Coverage speaks
+      // of the message history, which is whole either way, so it is unchanged; the capture simply names no
+      // artifacts, and the server refuses a native export or continue of that revision (fidelity_unavailable).
+      const onDisk=fileStamp(artifactBefore);let bundle=null;
+      if(!effectivePrevious?.baseline&&this.nativeReady(key,artifactBefore)){
+        const cached=this.state.native_cache[key];
+        if(cached?.stamp===onDisk&&cached.files.every(file=>existsSync(join(this.spillDir(),file.spill))))bundle=cached;
+        else try{
+          const translated=(await this.core.translateSession({locator,target_harness:harness},{timeoutMs:translateTimeoutMs})).artifact;
+          if(fileStamp(statSync(artifactSource))===onDisk){
+            bundle={stamp:onDisk,...this.spillTranslated(translated,basename(artifactSource))};
+            // Only a translation that took real time is worth keeping; a small file translates again in no time.
+            if(artifactBefore.size>=this.nativeIdleBytes){this.state.native_cache[key]=bundle;this.save();this.pruneSpill();}
+          }
+        }catch(error){
+          if(this.abort.signal.aborted)throw error;
+          this.lastError=error;process.stderr.write(`supercode teams: ${harness} session ${nativeSessionId} published without its native bundle: ${error.message}\n`);
+          // The same file is not translated again; a write to it clears this.
+          this.state.native_cache[key]={stamp:onDisk,failed:true,files:[]};this.save();this.pruneSpill();
+        }
+      }
+      // The messages were loaded after artifactBefore was read, so they match the bundle only if the file is still that file.
+      if(bundle&&fileStamp(statSync(artifactSource))!==onDisk)bundle=null;
+      if(bundle)({artifacts,nativeBundle}=this.bundleArtifacts(bundle,harness));else nativeDeferred=true;
+      // A retry that still has no bundle would republish the same messages for nothing.
+      if(unchanged&&!nativeBundle){this.recordStamp(key,ruleId,ruleRevision,backfillPolicy.mode,stamp);return null;}
+    }else if(sqliteSource&&nativeWanted&&!effectivePrevious?.baseline){
+      const translated=(await this.core.translateSession({locator,target_harness:harness},{timeoutMs:translateTimeoutMs})).artifact;
+      const digest=entries=>hash(canonicalJson((entries.messages??entries.turns??[]).map(entry=>entry.value??entry)));if(digest(await this.core.session(locator).load())!==digest(loaded))throw Object.assign(new Error('native session changed while it was being captured'),{code:'resync_required'});
+      ({artifacts,nativeBundle}=this.bundleArtifacts(this.spillTranslated(translated,`${nativeSessionId}.json`),harness));
     }
     const recordedTimes=messages.map(message=>message.recorded_at).filter(Number.isSafeInteger),commit={capture_id:captureId,parts:parts.length,digest:'',title:descriptor?.title??null,repository:null,reported_user:null,recorded_activity_at:recordedTimes.length?Math.max(...recordedTimes):null,parent_native_id:descriptor?.parent_session_id??null,coverage:effectivePrevious?.baseline&&!clearBaseline?{...coverage,completeness:'partial',reasons:[...new Set([...coverage.reasons,'prefix_not_captured'])]}:coverage,artifacts:artifacts.map(item=>item.manifest),...(nativeBundle?{native_bundle:nativeBundle}:{})};
     const digestCommit={...commit};delete digestCommit.digest;commit.digest=hash(canonicalJson({begin,parts,commit:digestCommit}));
-    this.state.outbox.push({begin,parts,commit,artifacts,clearBaseline,backfillMode:backfillPolicy.mode,stamp,observed:{keys:(loaded.messages??loaded.turns??[]).map((entry,index)=>entry.id??entry.native_key??String(index)),digests:(loaded.messages??loaded.turns??[]).map(entry=>hash(canonicalJson(entry.value??Object.fromEntries(Object.entries(entry).filter(([key])=>!['id','native_key','recorded_at','timestamp'].includes(key))))))}});this.save();return this.flushSerial();
+    this.state.outbox.push({begin,parts,commit,artifacts,clearBaseline,backfillMode:backfillPolicy.mode,stamp,nativeDeferred,observed:{keys:(loaded.messages??loaded.turns??[]).map((entry,index)=>entry.id??entry.native_key??String(index)),digests:(loaded.messages??loaded.turns??[]).map(entry=>hash(canonicalJson(entry.value??Object.fromEntries(Object.entries(entry).filter(([key])=>!['id','native_key','recorded_at','timestamp'].includes(key))))))}});this.save();return this.flushSerial();
   }
   flush() {
     const work=this.work.then(()=>this.flushSerial());
@@ -369,7 +491,49 @@ export class TeamsConnector {
   }
   // Every outbox error is the connection's, not one session's: a capture's own catch passes it on (syncConfiguredSessions).
   async flushSerial(){try{return await this.flushOutbox();}catch(error){throw Object.assign(error,{outbox:true});}}
-  async sendCapture(item){const begun=await this.request('POST',`/sources/${this.sourceId}/captures`,item.begin);if(begun.state==='committed')return begun.receipt;for(const part of item.parts)await this.request('PUT',`/sources/${this.sourceId}/captures/${item.begin.capture_id}/parts/${part.index}`,part);for(const artifact of item.artifacts??[])for(const chunk of artifact.chunks)await this.request('PUT',`/sources/${this.sourceId}/uploads/${chunk.upload_id}/chunks/${chunk.index}`,chunk);return this.request('POST',`/sources/${this.sourceId}/captures/${item.begin.capture_id}/commit`,item.commit);}
+  // A spill file is hashed once per version on disk (its stat stamp) and only stat'ed after that. It is lost only
+  // when it is missing, the wrong size or damaged; any other failure to read it (a Windows antivirus lock) is thrown,
+  // which keeps the capture queued with its bundle for the next flush.
+  // A read error that never clears (a lock never released, a lasting EACCES) is retried on five flushes, then the
+  // file counts as lost: the outbox is flushed in order, so one unreadable file would otherwise stop every session.
+  spillLost(artifact){
+    const failures=this.spillReadFailures??=new Map();
+    try{const lost=this.spillLostOnce(artifact);failures.delete(artifact.spill.file);return lost;}
+    catch(error){
+      const count=(failures.get(artifact.spill.file)??0)+1;failures.set(artifact.spill.file,count);
+      if(count<5)throw error;
+      failures.delete(artifact.spill.file);
+      process.stderr.write(`supercode teams: a spilled native artifact could not be read on ${count} flushes (${error.message}); treating it as lost\n`);
+      return true;
+    }
+  }
+  spillLostOnce(artifact){
+    const path=join(this.spillDir(),artifact.spill.file),missing=error=>error?.code==='ENOENT'||error?.code==='ENOTDIR';
+    let stat;try{stat=statSync(path);}catch(error){if(missing(error))return true;throw error;}
+    if(!stat.isFile()||stat.size!==artifact.manifest.byte_length)return true;
+    const stamp=fileStamp(stat),verified=this.verifiedSpills??=new Map();
+    if(verified.get(artifact.spill.file)===stamp)return false;
+    let fd;try{fd=this.openSpill(artifact.spill.file);}catch(error){if(missing(error))return true;throw error;}
+    try{
+      for(let index=0;index<artifact.manifest.chunks;index++){
+        const length=Math.max(0,Math.min(CHUNK_BYTES,artifact.manifest.byte_length-index*CHUNK_BYTES)),bytes=Buffer.alloc(length);
+        if((length?readSync(fd,bytes,0,length,index*CHUNK_BYTES):0)!==length||hash(bytes)!==artifact.spill.chunk_sha256[index])return true;
+      }
+    }finally{closeSync(fd);}
+    verified.set(artifact.spill.file,stamp);return false;
+  }
+  // A queued capture whose spilled artifact is gone or damaged on disk would never flush: it is sent without
+  // its native bundle instead (its digest resealed), and the session is captured again for one.
+  dropLostArtifacts(item){
+    if(!(item.artifacts??[]).some(artifact=>artifact.spill&&this.spillLost(artifact)))return;
+    process.stderr.write(`supercode teams: a queued capture of ${item.begin.harness} session ${item.begin.native_session_id} lost its native artifact on disk; sending it without one\n`);
+    item.artifacts=[];item.commit.artifacts=[];delete item.commit.native_bundle;item.nativeDeferred=true;
+    const digestCommit={...item.commit};delete digestCommit.digest;item.commit.digest=hash(canonicalJson({begin:item.begin,parts:item.parts,commit:digestCommit}));this.save();
+  }
+  async sendCapture(item){const begun=await this.request('POST',`/sources/${this.sourceId}/captures`,item.begin);if(begun.state==='committed')return begun.receipt;for(const part of item.parts)await this.request('PUT',`/sources/${this.sourceId}/captures/${item.begin.capture_id}/parts/${part.index}`,part);
+    // A capture queued by an older connector holds its chunks inline; a current one reads them from its spill file.
+    for(const artifact of item.artifacts??[])for(const chunk of artifact.spill?this.spilledChunks(artifact):artifact.chunks){if(!chunk)throw Object.assign(new Error('a spilled native artifact changed on disk while it was sent'),{code:'resync_required'});await this.request('PUT',`/sources/${this.sourceId}/uploads/${chunk.upload_id}/chunks/${chunk.index}`,chunk);}
+    return this.request('POST',`/sources/${this.sourceId}/captures/${item.begin.capture_id}/commit`,item.commit);}
   // True when the server refused a queued capture because its sync rule was removed, paused or revised since it
   // was queued: the capture can never be accepted, so it must not hold every later capture behind it. The
   // refusal is confirmed against a fresh configuration; any other refusal (a revoked credential, a paused
@@ -381,6 +545,6 @@ export class TeamsConnector {
     if(!Array.isArray(configuration?.rules))return false;
     return !configuration.rules.some(rule=>rule.id===item.begin.rule_id&&rule.revision===item.begin.rule_revision&&rule.state==='active');
   }
-  async flushOutbox(){let latest=null;while(this.state.outbox.length){const item=this.state.outbox[0];let receipt;try{receipt=await this.sendCapture(item);}catch(error){if(!await this.ruleRejected(item,error))throw error;this.lastError=error;process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id}; its sync rule changed or was removed: ${error.message}\n`);this.state.outbox.shift();this.save();continue;}const key=`${item.begin.harness}:${item.begin.native_session_id}`,prior=this.state.origins[key]??{},next={...prior,generation:item.begin.generation,sequence:item.begin.sequence,digest:item.commit.digest,observed_keys:item.observed?.keys??prior.observed_keys,observed_digests:item.observed?.digests??prior.observed_digests,last_rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode};if(item.clearBaseline){delete next.baseline;delete next.baseline_digests;delete next.baseline_tool_tail;}if(item.stamp&&item.begin.rule_id)next.capture_stamps={...prior.capture_stamps,[item.begin.rule_id]:{stamp:item.stamp,rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode}};this.state.origins[key]=next;this.state.outbox.shift();this.save();latest=receipt;}return latest;}
+  async flushOutbox(){let latest=null;while(this.state.outbox.length){const item=this.state.outbox[0];let receipt;this.dropLostArtifacts(item);try{receipt=await this.sendCapture(item);}catch(error){if(!await this.ruleRejected(item,error))throw error;this.lastError=error;process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id}; its sync rule changed or was removed: ${error.message}\n`);this.state.outbox.shift();this.save();continue;}const key=`${item.begin.harness}:${item.begin.native_session_id}`,prior=this.state.origins[key]??{},next={...prior,generation:item.begin.generation,sequence:item.begin.sequence,digest:item.commit.digest,observed_keys:item.observed?.keys??prior.observed_keys,observed_digests:item.observed?.digests??prior.observed_digests,last_rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode};if(item.clearBaseline){delete next.baseline;delete next.baseline_digests;delete next.baseline_tool_tail;}if(item.nativeDeferred)next.native_deferred=true;else delete next.native_deferred;if(item.stamp&&item.begin.rule_id)next.capture_stamps={...prior.capture_stamps,[item.begin.rule_id]:{stamp:item.stamp,rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode}};this.state.origins[key]=next;this.state.outbox.shift();this.save();if(item.artifacts?.length)this.pruneSpill();latest=receipt;}return latest;}
   stop(){if(this.timer)clearInterval(this.timer);if(this.heartbeatTimer)clearInterval(this.heartbeatTimer);this.abort.abort();for(const follower of this.followers?.values()??[])follower.controller.abort();return this.nativeEffects.close();}
 }

package/connector/native-continuation.mjs

@@ -1,5 +1,5 @@
 import { readFileSync } from 'node:fs';
-import { join, resolve } from 'node:path';
+import { join, resolve, sep } from 'node:path';
 
 // A continuation materializes a new native session (new ids), so value_lossless is the only
 // fidelity it delivers.
@@ -19,7 +19,7 @@ export function createNativeContinuationAdapter({ core }) {
       if (signal?.aborted) throw signal.reason ?? new Error('native continuation was aborted');
       const files = ticket.artifact.files.map(file => {
         const path = resolve(directory, file.relative_name);
-        if (!path.startsWith(`${resolve(directory)}/`)) throw unsupported('native continuation artifact escapes its verified directory');
+        if (!path.startsWith(`${resolve(directory)}${sep}`)) throw unsupported('native continuation artifact escapes its verified directory');
         return { path: file.relative_name, content: readFileSync(path, 'utf8'), role: file.role };
       });
       const primary = files.filter(file => file.role === 'primary');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.2.24",
+  "version": "0.2.25",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",
@@ -61,7 +61,7 @@
     "@volter/supercode-frontend": "^0.2.2",
     "@volter/supercode-harness-sdk": "^0.3.43",
     "@volter/supercode-terminal": "^0.2.30",
-    "@volter/identity": "^0.4.0",
+    "@volter/identity": "^0.4.1",
     "@volter/teams": "^0.1.8",
     "@xterm/addon-serialize": "^0.14.0",
     "@xterm/headless": "^6.0.0",

package/server/otlp/wire-codec.mjs

@@ -46,6 +46,10 @@ class Reader {
 const signed64 = (value) => BigInt.asIntN(64, value);
 const zigzag64 = (value) => (value >> 1n) ^ -(value & 1n);
 
+// As `json: true`: ProtoJSON spells non-finite floats and doubles "NaN", "Infinity" and "-Infinity"
+// (JSON has no such numbers; JSON.stringify would turn them into null).
+const protoJsonDouble = (value) => Number.isFinite(value) ? value : String(value);
+
 function scalar(type, wire, reader) {
   switch (type) {
     case 'string': return utf8.decode(reader.take(Number(reader.varint())));
@@ -59,10 +63,10 @@ function scalar(type, wire, reader) {
     case 'sint64': return zigzag64(reader.varint()).toString();
     case 'fixed32': return reader.fixed32();
     case 'sfixed32': return reader.fixed32() | 0;
-    case 'float': { const at = reader.pos; reader.take(4); return reader.view.getFloat32(at, true); }
+    case 'float': { const at = reader.pos; reader.take(4); return protoJsonDouble(reader.view.getFloat32(at, true)); }
     case 'fixed64': return reader.fixed64().toString();
     case 'sfixed64': return signed64(reader.fixed64()).toString();
-    case 'double': { const at = reader.pos; reader.take(8); return reader.view.getFloat64(at, true); }
+    case 'double': { const at = reader.pos; reader.take(8); return protoJsonDouble(reader.view.getFloat64(at, true)); }
     default: throw new TypeError(`unsupported protobuf scalar ${type} (wire ${wire})`);
   }
 }