@volter/supercode-teams 0.2.145 → 0.2.147

package/connector/index.mjs

@@ -765,8 +765,17 @@ export class TeamsConnector {
       this.state.native_transfer??={sent_bytes:0,reused_bytes:0,chunks:0};
       this.state.native_transfer[reused?'reused_bytes':'sent_bytes']+=Buffer.byteLength(chunk.bytes_base64,'base64');
       this.state.native_transfer.chunks++;
+      const key=`${item.begin.harness}:${item.begin.native_session_id}`;
+      const transfers=this.state.native_transfers??={};
+      const transfer=transfers[key]??={since:this.clock.nowMs(),sent_bytes:0,reused_bytes:0,chunks:0,captures:0};
+      transfer[reused?'reused_bytes':'sent_bytes']+=Buffer.byteLength(chunk.bytes_base64,'base64');
+      transfer.chunks++;transfer.at=this.clock.nowMs();
+      for(const expired of Object.keys(transfers).sort((a,b)=>transfers[b].at-transfers[a].at).slice(128))delete transfers[expired];
     }
-    return this.request('POST',`/sources/${this.sourceId}/captures/${item.begin.capture_id}/commit`,item.commit);}
+    const receipt=await this.request('POST',`/sources/${this.sourceId}/captures/${item.begin.capture_id}/commit`,item.commit);
+    const transfer=this.state.native_transfers?.[`${item.begin.harness}:${item.begin.native_session_id}`];
+    if(transfer&&item.artifacts?.length){transfer.captures++;transfer.last_bundle_bytes=item.commit.artifacts.reduce((sum,a)=>sum+a.byte_length,0);}
+    return receipt;}
   // True when the server refused a queued capture because its sync rule was removed, paused or revised since it
   // was queued: the capture can never be accepted, so it must not hold every later capture behind it. The
   // refusal is confirmed against a fresh configuration; any other refusal (a revoked credential, a paused

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.2.145",
+  "version": "0.2.147",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",
@@ -60,9 +60,9 @@
     "@homebridge/node-pty-prebuilt-multiarch": "^0.14.1",
     "@volter/supercode-frontend": "^0.2.2",
     "@volter/supercode-harness-sdk": "^0.3.43",
-    "@volter/supercode-terminal": "^0.2.53",
+    "@volter/supercode-terminal": "^0.2.54",
     "@volter/identity": "^0.4.1",
-    "@volter/teams": "^0.1.54",
+    "@volter/teams": "^0.1.58",
     "@xterm/addon-serialize": "^0.14.0",
     "@xterm/headless": "^6.0.0",
     "protobufjs": "8.8.0",

package/server/continuations.mjs

@@ -53,4 +53,4 @@ async function authorizedRemote(api,auth,machineId,effectId,incarnation,routeFai
 
 export async function remoteContinuationRoute({api,req,res,auth,machineId,effectId,requestId,fail,send,result}){const {value}=await authorizedRemote(api,auth,machineId,effectId,req.headers['x-teams-incarnation'],fail);const ticket={kind:'remote_launch',continuation_id:value.id,operation_id:value.operation_id,harness:value.harness,fidelity:value.fidelity,artifact:value.artifact,destination:value.destination,expires_at:Math.min(value.expires_at,value.destination_authorization.expires_at)};return send(res,200,result(api,auth,ticket),requestId);}
 
-export async function remoteContinuationArtifactRoute({api,req,res,auth,machineId,effectId,uploadId,requestId,fail}){const {value}=await authorizedRemote(api,auth,machineId,effectId,req.headers['x-teams-incarnation'],fail);const file=value.artifact.files.find(item=>item.upload_id===uploadId);if(!file)fail(404,'not_found','artifact is not in the continuation bundle');const rows=await api.store.query('SELECT bytes FROM artifact_chunk_data WHERE team_id=? AND upload_id=? ORDER BY chunk_index',[auth.team_id,uploadId]),bytes=Buffer.concat(rows.map(row=>row.bytes));if(bytes.length!==file.byte_length||createHash('sha256').update(bytes).digest('hex')!==file.sha256)fail(409,'conflict','artifact bytes changed');let start=0,end=bytes.length-1,status=200;const range=req.headers.range;if(range!==undefined){const match=/^bytes=(\d*)-(\d*)$/.exec(range);if(!match||(!match[1]&&!match[2]))fail(416,'invalid_request','invalid artifact range');if(match[1]){start=Number(match[1]);end=match[2]?Number(match[2]):end;}else{const suffix=Number(match[2]);start=Math.max(0,bytes.length-suffix);}if(!Number.isSafeInteger(start)||!Number.isSafeInteger(end)||start<0||end<start||start>=bytes.length)fail(416,'invalid_request','artifact range is unsatisfiable');end=Math.min(end,bytes.length-1);status=206;}const body=bytes.subarray(start,end+1);res.writeHead(status,{'Content-Type':file.media_type,'Content-Length':String(body.length),'Accept-Ranges':'bytes','Cache-Control':'no-store','X-Request-ID':requestId,...(status===206?{'Content-Range':`bytes ${start}-${end}/${bytes.length}`}:{})});res.end(body);}
+export async function remoteContinuationArtifactRoute({api,req,res,auth,machineId,effectId,uploadId,requestId,fail}){const {value}=await authorizedRemote(api,auth,machineId,effectId,req.headers['x-teams-incarnation'],fail);const file=value.artifact.files.find(item=>item.upload_id===uploadId);if(!file)fail(404,'not_found','artifact is not in the continuation bundle');const rows=await api.store.query('SELECT chunk_index,bytes FROM artifact_chunks WHERE team_id=? AND upload_id=? UNION ALL SELECT r.chunk_index,b.bytes FROM artifact_chunk_refs r JOIN artifact_blobs b ON b.team_id=r.team_id AND b.sha256=r.sha256 WHERE r.team_id=? AND r.upload_id=? ORDER BY chunk_index',[auth.team_id,uploadId,auth.team_id,uploadId]),bytes=Buffer.concat(rows.map(row=>row.bytes));if(bytes.length!==file.byte_length||createHash('sha256').update(bytes).digest('hex')!==file.sha256)fail(409,'conflict','artifact bytes changed');let start=0,end=bytes.length-1,status=200;const range=req.headers.range;if(range!==undefined){const match=/^bytes=(\d*)-(\d*)$/.exec(range);if(!match||(!match[1]&&!match[2]))fail(416,'invalid_request','invalid artifact range');if(match[1]){start=Number(match[1]);end=match[2]?Number(match[2]):end;}else{const suffix=Number(match[2]);start=Math.max(0,bytes.length-suffix);}if(!Number.isSafeInteger(start)||!Number.isSafeInteger(end)||start<0||end<start||start>=bytes.length)fail(416,'invalid_request','artifact range is unsatisfiable');end=Math.min(end,bytes.length-1);status=206;}const body=bytes.subarray(start,end+1);res.writeHead(status,{'Content-Type':file.media_type,'Content-Length':String(body.length),'Accept-Ranges':'bytes','Cache-Control':'no-store','X-Request-ID':requestId,...(status===206?{'Content-Range':`bytes ${start}-${end}/${bytes.length}`}:{})});res.end(body);}

package/server/harness-routes.mjs

@@ -285,15 +285,18 @@ async function artifactChunk(api,req,res,auth,sourceId,uploadId,index,requestId)
   const upload=await api.store.one('SELECT * FROM uploads WHERE team_id=? AND id=?',[auth.team_id,uploadId]);
   if(await expireStaging(api,auth.team_id,sourceId,uploadId,'uploads',upload))fail(409,'resync_required','upload reservation expired');
   if(upload&&upload.source_id!==sourceId)fail(409,'capture_conflict','upload belongs to another source');
-  const old=await api.store.one('SELECT sha256 FROM artifact_chunk_data WHERE team_id=? AND upload_id=? AND chunk_index=?',[auth.team_id,uploadId,index]);
+  const old=await api.store.one('SELECT sha256 FROM artifact_chunks WHERE team_id=? AND upload_id=? AND chunk_index=? UNION ALL SELECT sha256 FROM artifact_chunk_refs WHERE team_id=? AND upload_id=? AND chunk_index=?',[auth.team_id,uploadId,index,auth.team_id,uploadId,index]);
   if(old){if(old.sha256!==b.sha256)fail(409,'capture_conflict','artifact chunk replay differs');return send(res,200,result(api,auth,{index,sha256:b.sha256}),requestId);}
   let bytes;
   if (b.reuse === true) {
     // A source may reuse only bytes it already uploaded. Knowing another
     // source's content hash never grants read access to its native history.
-    const held = await api.store.one(`SELECT c.bytes FROM artifact_chunk_data c
-      JOIN uploads u ON u.team_id=c.team_id AND u.id=c.upload_id
-      WHERE c.team_id=? AND c.sha256=? AND u.source_id=? LIMIT 1`, [auth.team_id,b.sha256,sourceId]);
+    const held = await api.store.one(`SELECT b.bytes FROM artifact_blobs b WHERE b.team_id=? AND b.sha256=?
+      AND EXISTS (SELECT 1 FROM artifact_chunk_refs r JOIN uploads u ON u.team_id=r.team_id AND u.id=r.upload_id
+        WHERE r.team_id=b.team_id AND r.sha256=b.sha256 AND u.source_id=?)
+      UNION ALL SELECT c.bytes FROM artifact_chunks c WHERE c.team_id=? AND c.sha256=?
+      AND EXISTS (SELECT 1 FROM uploads u WHERE u.team_id=c.team_id AND u.id=c.upload_id AND u.source_id=?) LIMIT 1`,
+      [auth.team_id,b.sha256,sourceId,auth.team_id,b.sha256,sourceId]);
     if (!held) fail(404,'chunk_missing','this source has no retained chunk with that digest');
     bytes = Buffer.from(held.bytes);
   } else {
@@ -311,8 +314,8 @@ async function artifactChunk(api,req,res,auth,sourceId,uploadId,index,requestId)
   }
   const now=api.clock.nowMs(),response=result(api,auth,{index,sha256:b.sha256,reused:b.reuse===true,stored_bytes:blob?0:bytes.length});
   const guards=[
-    {sql:"SELECT 1 ok WHERE EXISTS (SELECT 1 FROM credentials c JOIN sources s ON s.team_id=c.team_id AND s.id=c.source_id WHERE c.id=? AND c.source_id=? AND c.revoked_at IS NULL AND (c.expires_at IS NULL OR c.expires_at>?) AND s.state='active') AND NOT EXISTS (SELECT 1 FROM capture_expiries WHERE team_id=? AND source_id=? AND id=?) AND NOT EXISTS (SELECT 1 FROM uploads WHERE team_id=? AND id=? AND source_id<>?) AND COALESCE((SELECT committed FROM uploads WHERE team_id=? AND id=?),0)=0 AND COALESCE((SELECT created_at+86400000>? FROM uploads WHERE team_id=? AND id=?),1) AND (SELECT count(*) FROM artifact_chunk_data WHERE team_id=? AND upload_id=?)=?",params:[auth.credential_id,sourceId,now,auth.team_id,sourceId,uploadId,auth.team_id,uploadId,sourceId,auth.team_id,uploadId,now,auth.team_id,uploadId,auth.team_id,uploadId,index],code:'batch_gap',status:409,message:'upload authorization, expiry, or sequence changed'},
-    {sql:'SELECT 1 ok WHERE COALESCE((SELECT sum(length(bytes)) FROM artifact_chunk_data WHERE team_id=? AND upload_id=?),0)+?<=?',params:[auth.team_id,uploadId,bytes.length,api.uploadLimitBytes],code:'request_too_large',status:413,message:'artifact exceeds configured upload cap'}
+    {sql:"SELECT 1 ok WHERE EXISTS (SELECT 1 FROM credentials c JOIN sources s ON s.team_id=c.team_id AND s.id=c.source_id WHERE c.id=? AND c.source_id=? AND c.revoked_at IS NULL AND (c.expires_at IS NULL OR c.expires_at>?) AND s.state='active') AND NOT EXISTS (SELECT 1 FROM capture_expiries WHERE team_id=? AND source_id=? AND id=?) AND NOT EXISTS (SELECT 1 FROM uploads WHERE team_id=? AND id=? AND source_id<>?) AND COALESCE((SELECT committed FROM uploads WHERE team_id=? AND id=?),0)=0 AND COALESCE((SELECT created_at+86400000>? FROM uploads WHERE team_id=? AND id=?),1) AND ((SELECT count(*) FROM artifact_chunks WHERE team_id=? AND upload_id=?)+(SELECT count(*) FROM artifact_chunk_refs WHERE team_id=? AND upload_id=?))=?",params:[auth.credential_id,sourceId,now,auth.team_id,sourceId,uploadId,auth.team_id,uploadId,sourceId,auth.team_id,uploadId,now,auth.team_id,uploadId,auth.team_id,uploadId,auth.team_id,uploadId,index],code:'batch_gap',status:409,message:'upload authorization, expiry, or sequence changed'},
+    {sql:'SELECT 1 ok WHERE COALESCE((SELECT sum(length(bytes)) FROM artifact_chunks WHERE team_id=? AND upload_id=?),0)+COALESCE((SELECT sum(length(b.bytes)) FROM artifact_chunk_refs r JOIN artifact_blobs b ON b.team_id=r.team_id AND b.sha256=r.sha256 WHERE r.team_id=? AND r.upload_id=?),0)+?<=?',params:[auth.team_id,uploadId,auth.team_id,uploadId,bytes.length,api.uploadLimitBytes],code:'request_too_large',status:413,message:'artifact exceeds configured upload cap'}
   ];
   await guardedMutation(api,res,auth,guards,[
     {sql:'INSERT OR IGNORE INTO uploads(team_id,id,source_id,created_at) VALUES (?,?,?,?)',params:[auth.team_id,uploadId,sourceId,now]},
@@ -322,7 +325,7 @@ async function artifactChunk(api,req,res,auth,sourceId,uploadId,index,requestId)
   send(res,200,response,requestId);
 }
 async function captureCommit(api,req,res,auth,sourceId,captureId,requestId){const source=await sourceAuth(api,auth,sourceId);const commit=await parseJson(req);exact(commit,['capture_id','parts','digest','title','repository','reported_user','recorded_activity_at','parent_native_id','coverage','artifacts','native_bundle']);if(commit.capture_id!==captureId||!Number.isSafeInteger(commit.parts)||commit.parts<1)fail(400,'invalid_request','invalid commit');const cap=await api.store.one('SELECT * FROM captures WHERE team_id=? AND id=? AND source_id=?',[auth.team_id,captureId,sourceId]);if(await expireStaging(api,auth.team_id,sourceId,captureId,'captures',cap))fail(409,'resync_required','capture reservation expired');if(!cap)fail(404,'not_found','capture not found');if(cap.state==='committed'){const receipt=JSON.parse(cap.receipt_json);if(receipt.digest!==commit.digest)fail(409,'capture_conflict','commit replay differs');receipt.duplicate=true;return send(res,200,result(api,auth,receipt),requestId);}const rows=await api.store.query('SELECT * FROM capture_parts WHERE team_id=? AND capture_id=? ORDER BY part_index',[auth.team_id,captureId]);if(rows.length!==commit.parts)fail(409,'batch_gap','capture parts incomplete');const begin=JSON.parse(cap.begin_json);if(source.kind==='connector'){const rule=await api.store.one('SELECT * FROM sync_rules WHERE team_id=? AND id=? AND source_id=? AND machine_id=?',[auth.team_id,begin.rule_id,sourceId,auth.machine_id]);if(!rule||rule.state!=='active'||rule.revision!==begin.rule_revision||rule.visibility!==begin.visibility)fail(rule?.state==='paused'?409:403,rule?.state==='paused'?'source_paused':'forbidden','sync rule changed before commit');}const parts=rows.map(x=>JSON.parse(x.payload_json));if(captureDigest(begin,parts,commit)!==commit.digest)fail(409,'capture_conflict','capture digest differs');let messages=captureMessages(parts);if(messages.length>100000)fail(413,'request_too_large','capture contains too many messages');
-  for(const manifest of commit.artifacts){validateArtifactName(manifest.relative_name);const upload=await api.store.one('SELECT source_id FROM uploads WHERE team_id=? AND id=?',[auth.team_id,manifest.upload_id]);if(!upload||upload.source_id!==sourceId)fail(409,'capture_conflict','artifact upload belongs to another source');const chunks=await api.store.query('SELECT c.* FROM artifact_chunk_data c JOIN uploads u ON u.team_id=c.team_id AND u.id=c.upload_id WHERE c.team_id=? AND c.upload_id=? AND u.source_id=? ORDER BY c.chunk_index',[auth.team_id,manifest.upload_id,sourceId]);const bytes=Buffer.concat(chunks.map(x=>x.bytes));if(chunks.length!==manifest.chunks||bytes.length!==manifest.byte_length||sha256(bytes)!==manifest.sha256)fail(409,'capture_conflict','artifact manifest does not match staged chunks');}
+  for(const manifest of commit.artifacts){validateArtifactName(manifest.relative_name);const upload=await api.store.one('SELECT source_id FROM uploads WHERE team_id=? AND id=?',[auth.team_id,manifest.upload_id]);if(!upload||upload.source_id!==sourceId)fail(409,'capture_conflict','artifact upload belongs to another source');const chunks=await api.store.query('SELECT chunk_index,bytes FROM artifact_chunks WHERE team_id=? AND upload_id=? UNION ALL SELECT r.chunk_index,b.bytes FROM artifact_chunk_refs r JOIN artifact_blobs b ON b.team_id=r.team_id AND b.sha256=r.sha256 WHERE r.team_id=? AND r.upload_id=? ORDER BY chunk_index',[auth.team_id,manifest.upload_id,auth.team_id,manifest.upload_id]);const bytes=Buffer.concat(chunks.map(x=>x.bytes));if(chunks.length!==manifest.chunks||bytes.length!==manifest.byte_length||sha256(bytes)!==manifest.sha256)fail(409,'capture_conflict','artifact manifest does not match staged chunks');}
   if(commit.native_bundle!=null){const bundle=commit.native_bundle;exact(bundle,['id','harness','fidelity','residue','files'],'native_bundle');if(typeof bundle.id!=='string'||bundle.harness!==cap.harness||!['byte_lossless','value_lossless','semantic'].includes(bundle.fidelity)||!Array.isArray(bundle.residue)||!bundle.residue.every(x=>typeof x==='string')||!Array.isArray(bundle.files))fail(400,'invalid_request','invalid native bundle');const artifactIds=new Set(commit.artifacts.map(x=>x.upload_id)),seen=new Set();for(const file of bundle.files){exact(file,['upload_id','role'],'native_bundle file');if(!artifactIds.has(file.upload_id)||seen.has(file.upload_id)||!['primary','subagent','bundle','source_recovery'].includes(file.role))fail(400,'invalid_request','native bundle files must uniquely reference captured artifacts');seen.add(file.upload_id);}if(seen.size!==artifactIds.size||bundle.files.filter(x=>x.role==='primary').length!==1||new Set(commit.artifacts.map(x=>x.relative_name)).size!==commit.artifacts.length)fail(400,'invalid_request','native bundle must describe every unique artifact and one primary');}
   let origin=await api.store.one('SELECT * FROM origins WHERE team_id=? AND source_id=? AND harness=? AND native_id=?',[auth.team_id,sourceId,cap.harness,cap.native_id]);
   // A capture that carries (schema 9's wire half) names its first carry.count messages by digest instead of sending

package/server/storage-report.mjs

@@ -20,7 +20,7 @@ export async function storageReport({ api, res, auth, url, requestId }) {
       p.native_id,p.harness,p.captured_at,
       json_extract(u.manifest_json,'$.role') role,json_extract(u.manifest_json,'$.media_type') media_type,
       json_extract(u.manifest_json,'$.sha256') sha256,
-      (SELECT COALESCE(SUM(length(bytes)),0) FROM artifact_chunk_data c WHERE c.team_id=u.team_id AND c.upload_id=u.id) bytes,
+      ((SELECT COALESCE(SUM(length(bytes)),0) FROM artifact_chunks c WHERE c.team_id=u.team_id AND c.upload_id=u.id) + (SELECT COALESCE(SUM(length(b.bytes)),0) FROM artifact_chunk_refs r JOIN artifact_blobs b ON b.team_id=r.team_id AND b.sha256=r.sha256 WHERE r.team_id=u.team_id AND r.upload_id=u.id)) bytes,
       EXISTS (SELECT 1 FROM session_artifacts a WHERE a.team_id=u.team_id AND a.upload_id=u.id) version_referenced,
       EXISTS (SELECT 1 FROM continuations c WHERE c.team_id=u.team_id AND instr(c.value_json,u.id)>0) continuation_referenced,
       EXISTS (SELECT 1 FROM captures c WHERE c.team_id=u.team_id AND c.state='staging' AND instr(c.begin_json,u.id)>0) staging_referenced

package/server/store-core.mjs

@@ -111,7 +111,7 @@ export function createStoreHandlers({ all, get, run, tx }) {
     const upload=get(`SELECT u.manifest_json FROM session_artifacts a JOIN uploads u ON u.team_id=a.team_id AND u.id=a.upload_id
       WHERE a.team_id=? AND a.session_id=? AND a.upload_id=? AND u.committed=1`,[args.teamId,session.id,args.uploadId]);
     if(!upload)domainError(404,'not_found','artifact not found');
-    const chunks=all('SELECT bytes FROM artifact_chunk_data WHERE team_id=? AND upload_id=? ORDER BY chunk_index',[args.teamId,args.uploadId]);
+    const chunks=all('SELECT chunk_index,bytes FROM artifact_chunks WHERE team_id=? AND upload_id=? UNION ALL SELECT r.chunk_index,b.bytes FROM artifact_chunk_refs r JOIN artifact_blobs b ON b.team_id=r.team_id AND b.sha256=r.sha256 WHERE r.team_id=? AND r.upload_id=? ORDER BY chunk_index',[args.teamId,args.uploadId,args.teamId,args.uploadId]);
     return {manifest:JSON.parse(upload.manifest_json),bytes:Buffer.concat(chunks.map(row=>row.bytes))};
   });
   return {

package/worker/index.mjs

@@ -84,9 +84,9 @@ export class TeamsServer extends DurableObject {
       this.api = createTeamsApi({ publicOrigin: normalizePublicOrigin(env.PUBLIC_ORIGIN), identityIssuer: env.VOLTER_ISSUER || null, identityOrganizations: identityOrganizationsFromEnv(env), delegatedIssuer: env.DELEGATED_ISSUER || null });
       this.api.storageUsage = () => {
         const sql = ctx.storage.sql;
-        let free = 0, page = 0;
-        try { free = sql.exec('PRAGMA freelist_count').one().freelist_count; page = sql.exec('PRAGMA page_size').one().page_size; } catch { /* conservative allocation measure if host omits the pragmas */ }
-        return { allocated: sql.databaseSize, used: Math.max(0, sql.databaseSize - free * page), limit: 10 * 1024 ** 3 };
+        let freeBytes = null;
+        try { freeBytes = sql.exec('PRAGMA freelist_count').one().freelist_count * sql.exec('PRAGMA page_size').one().page_size; } catch { /* preserve an explicit unknown if the host omits these pragmas */ }
+        return { allocated: sql.databaseSize, free_bytes: freeBytes, used: Math.max(0, sql.databaseSize - (freeBytes ?? 0)), used_is_upper_bound: freeBytes === null, limit: 10 * 1024 ** 3 };
       };
       this.api.attach({ store: new DirectStore(createStoreHandlers(db)), harnessRoutes });
       const existing = await this.api.store.one("SELECT value FROM server_meta WHERE key='server_id'");