@volter/supercode-teams 0.2.78 → 0.2.79

package/connector/index.mjs

@@ -52,6 +52,20 @@ function settledToolTail(previous,messages){
   return messages.slice(0,count).map(digestOf);
 }
 const hash=value=>createHash('sha256').update(value).digest('hex');
+// A capture part is one request, and the server refuses a body over 8 MiB (volter-teams request-json.mjs): a part
+// holds at most PART_MESSAGES messages (the contract's bound) and PART_BYTES of them, so a run of large tool results
+// splits across parts instead of failing the whole session. A single message larger than that goes alone.
+const PART_MESSAGES=500,PART_BYTES=6*1024*1024;
+function captureParts(captureId,messages){
+  const parts=[];let current=[],bytes=0;
+  for(const message of messages){
+    const size=Buffer.byteLength(JSON.stringify(message));
+    if(current.length&&(current.length>=PART_MESSAGES||bytes+size>PART_BYTES)){parts.push({capture_id:captureId,index:parts.length,messages:current});current=[];bytes=0;}
+    current.push(message);bytes+=size;
+  }
+  if(current.length)parts.push({capture_id:captureId,index:parts.length,messages:current});
+  return parts;
+}
 const CHUNK_BYTES=1024*1024;
 // A native file at least NATIVE_IDLE_BYTES long gets its native bundle only once it has gone NATIVE_IDLE_MS
 // without a write. Translating reads the whole file, takes minutes for a 100 MB transcript, and holds the
@@ -565,7 +579,7 @@ export class TeamsConnector {
     const carried=this.carriedPrefix(effectivePrevious,messages,{ruleRevision,backfillMode:backfillPolicy.mode,clearBaseline});
     if(carried)begin.carry={count:carried,digest:carryDigest(messages.slice(0,carried))};
     const sent=carried?messages.slice(carried):messages;
-    const parts=[];for(let index=0;index<sent.length;index+=500)parts.push({capture_id:captureId,index:parts.length,messages:sent.slice(index,index+500)});
+    const parts=captureParts(captureId,sent);
     let artifacts=[];
     // Native stores normally live in harness homes outside the repository. The locator is
     // accepted only when it came from this configured core discovery, and raw bytes are safe
@@ -662,6 +676,16 @@ export class TeamsConnector {
     item.artifacts=[];item.commit.artifacts=[];delete item.commit.native_bundle;item.nativeDeferred=true;
     const digestCommit={...item.commit};delete digestCommit.digest;item.commit.digest=hash(canonicalJson({begin:item.begin,parts:item.parts,commit:digestCommit}));this.save();
   }
+  // A capture queued before parts were bounded by bytes can hold a part the server refuses as too large, and the
+  // outbox is flushed in order: it is split again, under a new capture id (the server drops the old reservation of
+  // the same revision) and its digest resealed, so it neither fails nor holds every later capture behind it.
+  refitParts(item){
+    if(item.parts.every(part=>part.messages.length<=PART_MESSAGES&&Buffer.byteLength(JSON.stringify(part))<=PART_BYTES+1024*1024))return;
+    const captureId=`cap_${randomBytes(16).toString('hex')}`;
+    item.begin={...item.begin,capture_id:captureId};item.parts=captureParts(captureId,item.parts.flatMap(part=>part.messages));
+    item.commit={...item.commit,capture_id:captureId,parts:item.parts.length};
+    const digestCommit={...item.commit};delete digestCommit.digest;item.commit.digest=hash(canonicalJson({begin:item.begin,parts:item.parts,commit:digestCommit}));this.save();
+  }
   async sendCapture(item){const begun=await this.request('POST',`/sources/${this.sourceId}/captures`,item.begin);if(begun.state==='committed')return begun.receipt;for(const part of item.parts)await this.request('PUT',`/sources/${this.sourceId}/captures/${item.begin.capture_id}/parts/${part.index}`,part);
     // A capture queued by an older connector holds its chunks inline; a current one reads them from its spill file.
     for(const artifact of item.artifacts??[])for(const chunk of artifact.spill?this.spilledChunks(artifact):artifact.chunks){if(!chunk)throw Object.assign(new Error('a spilled native artifact changed on disk while it was sent'),{code:'resync_required'});await this.request('PUT',`/sources/${this.sourceId}/uploads/${chunk.upload_id}/chunks/${chunk.index}`,chunk);}
@@ -677,6 +701,6 @@ export class TeamsConnector {
     if(!Array.isArray(configuration?.rules))return false;
     return !configuration.rules.some(rule=>rule.id===item.begin.rule_id&&rule.revision===item.begin.rule_revision&&rule.state==='active');
   }
-  async flushOutbox(){let latest=null;while(this.state.outbox.length){const item=this.state.outbox[0];let receipt;this.dropLostArtifacts(item);try{receipt=await this.sendCapture(item);}catch(error){if(error.status===409&&error.code==='carry_mismatch'){const key=`${item.begin.harness}:${item.begin.native_session_id}`;this.lastError=error;process.stderr.write(`supercode teams: the server does not hold what a capture of ${item.begin.harness} session ${item.begin.native_session_id} carried; it goes whole next time\n`);this.state.outbox.shift();if(this.state.origins[key])this.state.origins[key]={...this.state.origins[key],observed_keys:[],observed_digests:[],carry_refused:true};this.save();continue;}if(error.status===400&&error.code==='invalid_request'){this.lastError=error;process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id} the server refused as malformed: ${error.message}\n`);this.state.outbox.shift();this.save();continue;}if(!await this.ruleRejected(item,error))throw error;this.lastError=error;process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id}; its sync rule changed or was removed: ${error.message}\n`);this.state.outbox.shift();this.save();continue;}const key=`${item.begin.harness}:${item.begin.native_session_id}`,prior=this.state.origins[key]??{},next={...prior,generation:item.begin.generation,sequence:item.begin.sequence,digest:item.commit.digest,observed_keys:item.observed?.keys??prior.observed_keys,observed_digests:item.observed?.digests??prior.observed_digests,last_rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode};if(item.clearBaseline){delete next.baseline;delete next.baseline_digests;delete next.baseline_tool_tail;}if(item.nativeDeferred)next.native_deferred=true;else delete next.native_deferred;if(!item.begin.carry)delete next.carry_refused;if(item.stamp&&item.begin.rule_id)next.capture_stamps={...prior.capture_stamps,[item.begin.rule_id]:{stamp:item.stamp,rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode}};this.state.origins[key]=next;this.state.outbox.shift();this.save();if(item.artifacts?.length)this.pruneSpill();latest=receipt;}return latest;}
+  async flushOutbox(){let latest=null;while(this.state.outbox.length){const item=this.state.outbox[0];let receipt;this.dropLostArtifacts(item);this.refitParts(item);try{receipt=await this.sendCapture(item);}catch(error){if(error.status===409&&error.code==='carry_mismatch'){const key=`${item.begin.harness}:${item.begin.native_session_id}`;this.lastError=error;process.stderr.write(`supercode teams: the server does not hold what a capture of ${item.begin.harness} session ${item.begin.native_session_id} carried; it goes whole next time\n`);this.state.outbox.shift();if(this.state.origins[key])this.state.origins[key]={...this.state.origins[key],observed_keys:[],observed_digests:[],carry_refused:true};this.save();continue;}if(error.status===400&&error.code==='invalid_request'||error.status===413){this.lastError=error;this.noteSessionProblem({harness:item.begin.harness,nativeSessionId:item.begin.native_session_id,ruleId:item.begin.rule_id??null},error);process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id} the server refused${error.status===413?' (one message is larger than a request may be)':' as malformed'}: ${error.message}\n`);this.state.outbox.shift();this.save();continue;}if(!await this.ruleRejected(item,error))throw error;this.lastError=error;process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id}; its sync rule changed or was removed: ${error.message}\n`);this.state.outbox.shift();this.save();continue;}const key=`${item.begin.harness}:${item.begin.native_session_id}`,prior=this.state.origins[key]??{},next={...prior,generation:item.begin.generation,sequence:item.begin.sequence,digest:item.commit.digest,observed_keys:item.observed?.keys??prior.observed_keys,observed_digests:item.observed?.digests??prior.observed_digests,last_rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode};if(item.clearBaseline){delete next.baseline;delete next.baseline_digests;delete next.baseline_tool_tail;}if(item.nativeDeferred)next.native_deferred=true;else delete next.native_deferred;if(!item.begin.carry)delete next.carry_refused;if(item.stamp&&item.begin.rule_id)next.capture_stamps={...prior.capture_stamps,[item.begin.rule_id]:{stamp:item.stamp,rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode}};this.state.origins[key]=next;this.state.outbox.shift();this.save();if(item.artifacts?.length)this.pruneSpill();latest=receipt;}return latest;}
   stop(){if(this.timer)clearInterval(this.timer);if(this.heartbeatTimer)clearInterval(this.heartbeatTimer);this.abort.abort();for(const follower of this.followers?.values()??[])follower.controller.abort();return this.nativeEffects.close();}
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.2.78",
+  "version": "0.2.79",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",