@volter/supercode-teams 0.2.154 → 0.2.155

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.2.154",
+  "version": "0.2.155",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",

package/server/harness-routes.mjs

@@ -6,9 +6,10 @@
 // routes have always run: before authentication, once authenticated, before the
 // idempotency check, and last among a team's routes. Each hook returns true when it
 // answered the request.
+import { createHash } from 'node:crypto';
 import { bearer, hashToken, canRead, canManage, isAdmin } from '@volter/teams/server/auth';
 import { storageReport } from './storage-report.mjs';
-import { validateListQuery, validateSearchQuery, validateReadQuery, validateCaptureBegin, validateCapturePart, canonicalJson, carryDigest } from '../contracts/index.mjs';
+import { validateListQuery, validateSearchQuery, validateReadQuery, validateCaptureBegin, validateCapturePart, canonicalJson } from '../contracts/index.mjs';
 import { normalizeSearch, textFromContent, cursorHash, cursorToken } from './catalog.mjs';
 import { captureDigest, captureMessages, handleOtlpRoute, sha256, validateArtifactName } from './ingest.mjs';
 import { handleCatalogEvents, handleSessionEvents, sessionNames } from './event-routes.mjs';
@@ -325,20 +326,49 @@ async function artifactChunk(api,req,res,auth,sourceId,uploadId,index,requestId)
   send(res,200,response,requestId);
 }
 async function captureCommit(api,req,res,auth,sourceId,captureId,requestId){const source=await sourceAuth(api,auth,sourceId);const commit=await parseJson(req);exact(commit,['capture_id','parts','digest','title','repository','reported_user','recorded_activity_at','parent_native_id','coverage','artifacts','native_bundle']);if(commit.capture_id!==captureId||!Number.isSafeInteger(commit.parts)||commit.parts<1)fail(400,'invalid_request','invalid commit');const cap=await api.store.one('SELECT * FROM captures WHERE team_id=? AND id=? AND source_id=?',[auth.team_id,captureId,sourceId]);if(await expireStaging(api,auth.team_id,sourceId,captureId,'captures',cap))fail(409,'resync_required','capture reservation expired');if(!cap)fail(404,'not_found','capture not found');if(cap.state==='committed'){const receipt=JSON.parse(cap.receipt_json);if(receipt.digest!==commit.digest)fail(409,'capture_conflict','commit replay differs');receipt.duplicate=true;return send(res,200,result(api,auth,receipt),requestId);}const rows=await api.store.query('SELECT * FROM capture_parts WHERE team_id=? AND capture_id=? ORDER BY part_index',[auth.team_id,captureId]);if(rows.length!==commit.parts)fail(409,'batch_gap','capture parts incomplete');const begin=JSON.parse(cap.begin_json);if(source.kind==='connector'){const rule=await api.store.one('SELECT * FROM sync_rules WHERE team_id=? AND id=? AND source_id=? AND machine_id=?',[auth.team_id,begin.rule_id,sourceId,auth.machine_id]);if(!rule||rule.state!=='active'||rule.revision!==begin.rule_revision||rule.visibility!==begin.visibility)fail(rule?.state==='paused'?409:403,rule?.state==='paused'?'source_paused':'forbidden','sync rule changed before commit');}const parts=rows.map(x=>JSON.parse(x.payload_json));if(captureDigest(begin,parts,commit)!==commit.digest)fail(409,'capture_conflict','capture digest differs');let messages=captureMessages(parts);if(messages.length>100000)fail(413,'request_too_large','capture contains too many messages');
-  for(const manifest of commit.artifacts){validateArtifactName(manifest.relative_name);const upload=await api.store.one('SELECT source_id FROM uploads WHERE team_id=? AND id=?',[auth.team_id,manifest.upload_id]);if(!upload||upload.source_id!==sourceId)fail(409,'capture_conflict','artifact upload belongs to another source');const chunks=await api.store.query('SELECT chunk_index,bytes FROM artifact_chunks WHERE team_id=? AND upload_id=? UNION ALL SELECT r.chunk_index,b.bytes FROM artifact_chunk_refs r JOIN artifact_blobs b ON b.team_id=r.team_id AND b.sha256=r.sha256 WHERE r.team_id=? AND r.upload_id=? ORDER BY chunk_index',[auth.team_id,manifest.upload_id,auth.team_id,manifest.upload_id]);const bytes=Buffer.concat(chunks.map(x=>x.bytes));if(chunks.length!==manifest.chunks||bytes.length!==manifest.byte_length||sha256(bytes)!==manifest.sha256)fail(409,'capture_conflict','artifact manifest does not match staged chunks');}
+  // Native transcripts can be larger than the Worker's entire heap. Validate
+  // the retained bytes one chunk at a time; never materialize a whole artifact.
+  for (const manifest of commit.artifacts) {
+    validateArtifactName(manifest.relative_name);
+    const upload = await api.store.one('SELECT source_id FROM uploads WHERE team_id=? AND id=?', [auth.team_id, manifest.upload_id]);
+    if (!upload || upload.source_id !== sourceId) fail(409, 'capture_conflict', 'artifact upload belongs to another source');
+    const digest = createHash('sha256');
+    let count = 0, length = 0;
+    for (;;) {
+      const chunk = await api.store.one(`SELECT chunk_index,bytes FROM artifact_chunks WHERE team_id=? AND upload_id=? AND chunk_index>=?
+        UNION ALL SELECT r.chunk_index,b.bytes FROM artifact_chunk_refs r JOIN artifact_blobs b ON b.team_id=r.team_id AND b.sha256=r.sha256
+        WHERE r.team_id=? AND r.upload_id=? AND r.chunk_index>=? ORDER BY chunk_index LIMIT 1`,
+        [auth.team_id, manifest.upload_id, count, auth.team_id, manifest.upload_id, count]);
+      if (!chunk) break;
+      if (chunk.chunk_index !== count || count >= manifest.chunks) fail(409, 'capture_conflict', 'artifact chunks are not contiguous');
+      digest.update(chunk.bytes); length += chunk.bytes.length; count++;
+    }
+    if (count !== manifest.chunks || length !== manifest.byte_length || digest.digest('hex') !== manifest.sha256)
+      fail(409, 'capture_conflict', 'artifact manifest does not match staged chunks');
+  }
   if(commit.native_bundle!=null){const bundle=commit.native_bundle;exact(bundle,['id','harness','fidelity','residue','files'],'native_bundle');if(typeof bundle.id!=='string'||bundle.harness!==cap.harness||!['byte_lossless','value_lossless','semantic'].includes(bundle.fidelity)||!Array.isArray(bundle.residue)||!bundle.residue.every(x=>typeof x==='string')||!Array.isArray(bundle.files))fail(400,'invalid_request','invalid native bundle');const artifactIds=new Set(commit.artifacts.map(x=>x.upload_id)),seen=new Set();for(const file of bundle.files){exact(file,['upload_id','role'],'native_bundle file');if(!artifactIds.has(file.upload_id)||seen.has(file.upload_id)||!['primary','subagent','bundle','source_recovery'].includes(file.role))fail(400,'invalid_request','native bundle files must uniquely reference captured artifacts');seen.add(file.upload_id);}if(seen.size!==artifactIds.size||bundle.files.filter(x=>x.role==='primary').length!==1||new Set(commit.artifacts.map(x=>x.relative_name)).size!==commit.artifacts.length)fail(400,'invalid_request','native bundle must describe every unique artifact and one primary');}
   let origin=await api.store.one('SELECT * FROM origins WHERE team_id=? AND source_id=? AND harness=? AND native_id=?',[auth.team_id,sourceId,cap.harness,cap.native_id]);
-  // A capture that carries (schema 9's wire half) names its first carry.count messages by digest instead of sending
-  // them again: they are the ones this session's current version holds at those places, written by this source. The
-  // server rebuilds them from its rows and checks the digest; anything else is refused, and the source sends it whole.
-  if(begin.carry){
-    const {count,digest}=begin.carry,session=origin?await api.store.one('SELECT revision FROM sessions WHERE team_id=? AND id=?',[auth.team_id,origin.session_id]):null;
-    const rows=session?await api.store.query(`SELECT ordinal,source_id,native_key,recorded_at,value_json FROM messages m WHERE m.team_id=? AND m.session_id=? AND ${MESSAGE_AT()} AND m.ordinal<? ORDER BY m.ordinal`,[auth.team_id,origin.session_id,session.revision,session.revision,count]):[];
-    const carried=rows.map(row=>({native_key:row.native_key,recorded_at:row.recorded_at,value:JSON.parse(row.value_json)}));
-    if(rows.length!==count||rows.some((row,index)=>row.ordinal!==index||row.source_id!==sourceId)||carryDigest(carried)!==digest)fail(409,'carry_mismatch','the carried messages are not what this session holds; send the capture whole');
-    const sent=new Set(messages.map(m=>m.native_key));
-    if(carried.some(m=>sent.has(m.native_key)))fail(409,'capture_conflict','duplicate native message key');
-    messages=[...carried,...messages];
+  // Verify an unchanged prefix in bounded pages, then leave those rows open.
+  // Rebuilding and serializing the whole transcript for each appended turn can
+  // exhaust a hosted worker even though the client sends only the new tail.
+  let carriedCount = 0, carriedRevision = null;
+  if (begin.carry) {
+    const { count, digest } = begin.carry;
+    const session = origin ? await api.store.one('SELECT revision FROM sessions WHERE team_id=? AND id=?', [auth.team_id, origin.session_id]) : null;
+    if (!session || count + messages.length > 100000) fail(409, 'carry_mismatch', 'the carried prefix is unavailable or too large');
+    carriedRevision = session.revision;
+    const hash = createHash('sha256').update('['), sent = new Set(messages.map(m => m.native_key));
+    while (carriedCount < count) {
+      const row = await api.store.one(`SELECT ordinal,source_id,native_key,recorded_at,value_json FROM messages m
+        WHERE m.team_id=? AND m.session_id=? AND ${MESSAGE_AT()} AND m.ordinal=?`,
+        [auth.team_id, origin.session_id, carriedRevision, carriedRevision, carriedCount]);
+      if (!row || row.source_id !== sourceId) fail(409, 'carry_mismatch', 'the carried messages are not what this session holds; send the capture whole');
+      if (sent.has(row.native_key)) fail(409, 'capture_conflict', 'duplicate native message key');
+      if (carriedCount) hash.update(',');
+      hash.update(canonicalJson({native_key:row.native_key, recorded_at:row.recorded_at ?? null, value:JSON.parse(row.value_json)}));
+      carriedCount++;
+    }
+    if (hash.update(']').digest('hex') !== digest) fail(409, 'carry_mismatch', 'the carried messages are not what this session holds; send the capture whole');
   }
   let lineage=null,lineageValue=null,lineageMembers=[],lineageGuards=[];if(!origin&&source.kind==='connector'&&source.machine_id){const candidates=await api.store.query("SELECT * FROM continuations WHERE team_id=? AND state='running' AND json_extract(value_json,'$.new_session_id') IS NULL AND json_extract(value_json,'$.harness')=? AND json_extract(value_json,'$.destination.machine_id')=? AND json_extract(value_json,'$.remote_result.state')='succeeded' AND json_extract(value_json,'$.remote_result.receipt.native_session_id')=?",[auth.team_id,cap.harness,source.machine_id,cap.native_id]);if(candidates.length===1){const candidate=candidates[0],membership=await api.store.one('SELECT role,active FROM memberships WHERE team_id=? AND principal_id=?',[auth.team_id,candidate.requester_id]),reader={kind:'user',team_id:auth.team_id,principal_id:candidate.requester_id,role:membership?.role,active:membership?.active===1,now:api.clock.nowMs(),group_ids:(await api.store.query('SELECT group_id FROM group_members WHERE team_id=? AND principal_id=?',[auth.team_id,candidate.requester_id])).map(x=>x.group_id)};if(await canReadMerged(api,reader,await api.store.one('SELECT * FROM sessions WHERE team_id=? AND id=?',[auth.team_id,candidate.source_session_id]))){lineage=candidate;lineageValue=JSON.parse(candidate.value_json);lineageMembers=await constituents(api,auth.team_id,await api.store.one('SELECT * FROM sessions WHERE team_id=? AND id=?',[auth.team_id,candidate.source_session_id]));}}if(lineageValue){const lineageNow=api.clock.nowMs();lineageGuards=[{sql:'SELECT 1 ok FROM memberships WHERE team_id=? AND principal_id=? AND active=1',params:[auth.team_id,lineage.requester_id],code:'conflict',status:409,message:'continuation requester membership changed'},{sql:'SELECT 1 ok FROM credentials WHERE id=? AND principal_id=? AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at>?)',params:[lineageValue.issuer_credential_id,lineage.requester_id,lineageNow],code:'conflict',status:409,message:'continuation issuer credential changed'},...lineageMembers.map(member=>({sql:`SELECT 1 ok FROM sessions s WHERE s.team_id=? AND s.id=? AND s.deleted_at IS NULL AND (?=0 OR s.access_revision=?) AND (s.visibility='team' OR s.custodian_id=? OR EXISTS (SELECT 1 FROM session_grants g WHERE g.team_id=s.team_id AND g.session_id=s.id AND (g.expires_at IS NULL OR g.expires_at>?) AND (g.subject_kind='principal' AND g.subject_id=? OR g.subject_kind='group' AND EXISTS (SELECT 1 FROM group_members gm WHERE gm.team_id=s.team_id AND gm.group_id=g.subject_id AND gm.principal_id=?))))`,params:[auth.team_id,member.id,member.id===lineage.source_session_id?1:0,lineageValue.access_revision,lineage.requester_id,lineageNow,lineage.requester_id,lineage.requester_id],code:'conflict',status:409,message:'continuation source authorization changed'}))];}}
   const sid=origin?.session_id??api.ids.next('session'),parentOrigin=commit.parent_native_id?await api.store.one('SELECT session_id FROM origins WHERE team_id=? AND source_id=? AND harness=? AND native_id=? AND deleted_at IS NULL',[auth.team_id,sourceId,cap.harness,commit.parent_native_id]):null,revisionParam={$journalRevision:true},recorded=commit.recorded_activity_at,boundaryNow=api.clock.nowMs(),sortAt=recorded??boundaryNow,descriptor={id:sid,revision:'__JOURNAL_REVISION__',title:commit.title,harness:cap.harness,user:{principal_id:source.kind==='connector'?source.custodian_id:null,reported_user:commit.reported_user,status:source.kind==='connector'?'verified':commit.reported_user?'reported':'unknown',mapping_id:null},repository:commit.repository,parent_id:parentOrigin?.session_id??null,continued_from:lineage?.source_session_id??null,recorded_activity_at:recorded,received_at:boundaryNow,sort_at:sortAt,activity_basis:recorded==null?'received':'recorded',source_ids:[sourceId],machine_ids:source.machine_id?[source.machine_id]:[],coverage:commit.coverage,bindings:[],actions:{}},statements=[];
@@ -348,11 +378,12 @@ async function captureCommit(api,req,res,auth,sourceId,captureId,requestId){cons
   // stays open into this version; one that changed or went is closed at it; a new one is written. A capture of a long
   // session that gained one message writes one row, not the whole session again.
   const carryGuards=[],currentRevision=origin?(await api.store.one('SELECT revision FROM sessions WHERE team_id=? AND id=?',[auth.team_id,sid]))?.revision??null:null;
-  const held=new Map();if(currentRevision!=null)for(const row of await api.store.query(`SELECT revision,ordinal,id,source_id,native_key,recorded_at,value_json,until_revision FROM messages m WHERE m.team_id=? AND m.session_id=? AND ${MESSAGE_AT()}`,[auth.team_id,sid,currentRevision,currentRevision]))held.set(row.ordinal,row);
+  if (carriedRevision != null && currentRevision !== carriedRevision) fail(409, 'revision_mismatch', 'session changed during prefix verification; retry');
+  const held=new Map();if(currentRevision!=null)for(const row of await api.store.query(`SELECT revision,ordinal,id,source_id,native_key,recorded_at,value_json,until_revision FROM messages m WHERE m.team_id=? AND m.session_id=? AND ${MESSAGE_AT()} AND ordinal>=?`,[auth.team_id,sid,currentRevision,currentRevision,carriedCount]))held.set(row.ordinal,row);
   // Another commit to this session between this read and the write would make the carried rows wrong: refused, retried.
   if(currentRevision!=null)carryGuards.push({sql:'SELECT 1 ok FROM sessions WHERE team_id=? AND id=? AND revision=?',params:[auth.team_id,sid,currentRevision],code:'revision_mismatch',status:409,message:'session changed during capture; retry'});
   const close=row=>{if(row.until_revision===OPEN_REVISION)statements.push({sql:'UPDATE messages SET until_revision=? WHERE team_id=? AND session_id=? AND revision=? AND ordinal=?',params:[revisionParam,auth.team_id,sid,row.revision,row.ordinal]});};
-  let ordinal=0;for(const m of messages){const value=JSON.stringify(m.value),old=held.get(ordinal);held.delete(ordinal);
+  let ordinal=carriedCount;for(const m of messages){const value=JSON.stringify(m.value),old=held.get(ordinal);held.delete(ordinal);
     if(old&&old.source_id===sourceId&&old.native_key===m.native_key&&old.recorded_at===m.recorded_at&&old.value_json===value){if(old.until_revision!==OPEN_REVISION)statements.push({sql:'UPDATE messages SET until_revision=? WHERE team_id=? AND session_id=? AND revision=? AND ordinal=?',params:[OPEN_REVISION,auth.team_id,sid,old.revision,old.ordinal]});ordinal++;continue;}
     if(old)close(old);
     let mid=(await api.store.one('SELECT id FROM message_ids WHERE team_id=? AND source_id=? AND harness=? AND native_id=? AND native_key=?',[auth.team_id,sourceId,cap.harness,cap.native_id,m.native_key]))?.id;if(!mid){mid=api.ids.next('message');statements.push({sql:'INSERT INTO message_ids(team_id,source_id,harness,native_id,native_key,id) VALUES (?,?,?,?,?,?)',params:[auth.team_id,sourceId,cap.harness,cap.native_id,m.native_key,mid]});}statements.push({sql:'INSERT INTO messages(team_id,session_id,revision,ordinal,id,source_id,native_key,recorded_at,value_json,search_text,until_revision) VALUES (?,?,?,?,?,?,?,?,?,?,?)',params:[auth.team_id,sid,revisionParam,ordinal++,mid,sourceId,m.native_key,m.recorded_at,value,normalizeSearch(textFromContent(m.value.content)),OPEN_REVISION]});}

package/worker/fetch-adapter.mjs

@@ -20,10 +20,17 @@ export function serveFetch(handler, request, { remoteAddress = null } = {}) {
   let status = 200, responseHeaders = {}, started = false, ended = false, pending = 0;
   const start = () => { if (started) return; started = true; settle(new Response(readable, { status, headers: responseHeaders })); };
   const closed = () => { if (!ended) { ended = true; res.emit('close'); req.emit('close'); } };
-  request.signal?.addEventListener('abort', () => { closed(); writer.abort().catch(() => {}); });
+  const abort = () => { closed(); writer.abort().catch(() => {}); };
+  request.signal?.addEventListener('abort', abort, { once: true });
+  // A disconnected reader must release its timers/subscriptions even when no
+  // further data is written. Keep accessors live: Object.assign evaluates them
+  // once, which previously made every slow reader appear to have zero backlog.
+  writer.closed.then(closed, closed).finally(() => request.signal?.removeEventListener('abort', abort));
+  Object.defineProperties(res, {
+    writableLength: { get: () => pending },
+    headersSent: { get: () => started },
+  });
   Object.assign(res, {
-    get writableLength() { return pending; },
-    get headersSent() { return started; },
     writeHead(code, values = {}) { status = code; responseHeaders = { ...responseHeaders, ...values }; if (code === 204 || code === 304) { started = true; settle(new Response(null, { status, headers: responseHeaders })); } else start(); return res; },
     setHeader(name, value) { responseHeaders[name] = value; },
     write(chunk) {
@@ -31,7 +38,7 @@ export function serveFetch(handler, request, { remoteAddress = null } = {}) {
       if (ended) return false;
       const bytes = typeof chunk === 'string' ? encoder.encode(chunk) : chunk;
       pending += bytes.byteLength;
-      writer.write(bytes).then(() => { pending -= bytes.byteLength; if (pending === 0) res.emit('drain'); }, () => closed());
+      writer.write(bytes).then(() => { pending -= bytes.byteLength; if (pending === 0) res.emit('drain'); }, () => { pending -= bytes.byteLength; closed(); });
       return pending < 65536;
     },
     end(chunk) {