@volter/supercode-teams 0.2.143 → 0.2.144

package/connector/index.mjs

@@ -751,7 +751,21 @@ export class TeamsConnector {
   }
   async sendCapture(item){const begun=await this.request('POST',`/sources/${this.sourceId}/captures`,item.begin);if(begun.state==='committed')return begun.receipt;for(const part of item.parts)await this.request('PUT',`/sources/${this.sourceId}/captures/${item.begin.capture_id}/parts/${part.index}`,part);
     // A capture queued by an older connector holds its chunks inline; a current one reads them from its spill file.
-    for(const artifact of item.artifacts??[])for(const chunk of artifact.spill?this.spilledChunks(artifact):artifact.chunks){if(!chunk)throw Object.assign(new Error('a spilled native artifact changed on disk while it was sent'),{code:'resync_required'});await this.request('PUT',`/sources/${this.sourceId}/uploads/${chunk.upload_id}/chunks/${chunk.index}`,chunk);}
+    for(const artifact of item.artifacts??[])for(const chunk of artifact.spill?this.spilledChunks(artifact):artifact.chunks){
+      if(!chunk)throw Object.assign(new Error('a spilled native artifact changed on disk while it was sent'),{code:'resync_required'});
+      const path=`/sources/${this.sourceId}/uploads/${chunk.upload_id}/chunks/${chunk.index}`;
+      let reused=false;
+      if(this.chunkReuse!==false)try{
+        await this.request('PUT',path,{upload_id:chunk.upload_id,index:chunk.index,sha256:chunk.sha256,reuse:true});reused=true;this.chunkReuse=true;
+      }catch(error){
+        if(error.code==='invalid_request'&&error.status===400)this.chunkReuse=false; // older server
+        else if(error.code!=='chunk_missing')throw error;
+      }
+      if(!reused)await this.request('PUT',path,chunk);
+      this.state.native_transfer??={sent_bytes:0,reused_bytes:0,chunks:0};
+      this.state.native_transfer[reused?'reused_bytes':'sent_bytes']+=Buffer.byteLength(chunk.bytes_base64,'base64');
+      this.state.native_transfer.chunks++;
+    }
     return this.request('POST',`/sources/${this.sourceId}/captures/${item.begin.capture_id}/commit`,item.commit);}
   // True when the server refused a queued capture because its sync rule was removed, paused or revised since it
   // was queued: the capture can never be accepted, so it must not hold every later capture behind it. The

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.2.143",
+  "version": "0.2.144",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",

package/server/continuations.mjs

@@ -53,4 +53,4 @@ async function authorizedRemote(api,auth,machineId,effectId,incarnation,routeFai
 
 export async function remoteContinuationRoute({api,req,res,auth,machineId,effectId,requestId,fail,send,result}){const {value}=await authorizedRemote(api,auth,machineId,effectId,req.headers['x-teams-incarnation'],fail);const ticket={kind:'remote_launch',continuation_id:value.id,operation_id:value.operation_id,harness:value.harness,fidelity:value.fidelity,artifact:value.artifact,destination:value.destination,expires_at:Math.min(value.expires_at,value.destination_authorization.expires_at)};return send(res,200,result(api,auth,ticket),requestId);}
 
-export async function remoteContinuationArtifactRoute({api,req,res,auth,machineId,effectId,uploadId,requestId,fail}){const {value}=await authorizedRemote(api,auth,machineId,effectId,req.headers['x-teams-incarnation'],fail);const file=value.artifact.files.find(item=>item.upload_id===uploadId);if(!file)fail(404,'not_found','artifact is not in the continuation bundle');const rows=await api.store.query('SELECT bytes FROM artifact_chunks WHERE team_id=? AND upload_id=? ORDER BY chunk_index',[auth.team_id,uploadId]),bytes=Buffer.concat(rows.map(row=>row.bytes));if(bytes.length!==file.byte_length||createHash('sha256').update(bytes).digest('hex')!==file.sha256)fail(409,'conflict','artifact bytes changed');let start=0,end=bytes.length-1,status=200;const range=req.headers.range;if(range!==undefined){const match=/^bytes=(\d*)-(\d*)$/.exec(range);if(!match||(!match[1]&&!match[2]))fail(416,'invalid_request','invalid artifact range');if(match[1]){start=Number(match[1]);end=match[2]?Number(match[2]):end;}else{const suffix=Number(match[2]);start=Math.max(0,bytes.length-suffix);}if(!Number.isSafeInteger(start)||!Number.isSafeInteger(end)||start<0||end<start||start>=bytes.length)fail(416,'invalid_request','artifact range is unsatisfiable');end=Math.min(end,bytes.length-1);status=206;}const body=bytes.subarray(start,end+1);res.writeHead(status,{'Content-Type':file.media_type,'Content-Length':String(body.length),'Accept-Ranges':'bytes','Cache-Control':'no-store','X-Request-ID':requestId,...(status===206?{'Content-Range':`bytes ${start}-${end}/${bytes.length}`}:{})});res.end(body);}
+export async function remoteContinuationArtifactRoute({api,req,res,auth,machineId,effectId,uploadId,requestId,fail}){const {value}=await authorizedRemote(api,auth,machineId,effectId,req.headers['x-teams-incarnation'],fail);const file=value.artifact.files.find(item=>item.upload_id===uploadId);if(!file)fail(404,'not_found','artifact is not in the continuation bundle');const rows=await api.store.query('SELECT bytes FROM artifact_chunk_data WHERE team_id=? AND upload_id=? ORDER BY chunk_index',[auth.team_id,uploadId]),bytes=Buffer.concat(rows.map(row=>row.bytes));if(bytes.length!==file.byte_length||createHash('sha256').update(bytes).digest('hex')!==file.sha256)fail(409,'conflict','artifact bytes changed');let start=0,end=bytes.length-1,status=200;const range=req.headers.range;if(range!==undefined){const match=/^bytes=(\d*)-(\d*)$/.exec(range);if(!match||(!match[1]&&!match[2]))fail(416,'invalid_request','invalid artifact range');if(match[1]){start=Number(match[1]);end=match[2]?Number(match[2]):end;}else{const suffix=Number(match[2]);start=Math.max(0,bytes.length-suffix);}if(!Number.isSafeInteger(start)||!Number.isSafeInteger(end)||start<0||end<start||start>=bytes.length)fail(416,'invalid_request','artifact range is unsatisfiable');end=Math.min(end,bytes.length-1);status=206;}const body=bytes.subarray(start,end+1);res.writeHead(status,{'Content-Type':file.media_type,'Content-Length':String(body.length),'Accept-Ranges':'bytes','Cache-Control':'no-store','X-Request-ID':requestId,...(status===206?{'Content-Range':`bytes ${start}-${end}/${bytes.length}`}:{})});res.end(body);}

package/server/harness-routes.mjs

@@ -277,39 +277,52 @@ async function capturePart(api,req,res,auth,sourceId,captureId,index,requestId){
 async function artifactChunk(api,req,res,auth,sourceId,uploadId,index,requestId){
   await sourceAuth(api,auth,sourceId);
   const b=await parseJson(req);
-  exact(b,['upload_id','index','sha256','bytes_base64']);
+  exact(b,['upload_id','index','sha256','bytes_base64','reuse']);
+  if (!/^[a-f0-9]{64}$/.test(b.sha256 ?? '') || (b.reuse !== undefined && b.reuse !== true) || (b.reuse === true && b.bytes_base64 !== undefined)) fail(400,'invalid_request','invalid chunk digest or reuse request');
   if(b.upload_id!==uploadId||b.index!==index)fail(400,'invalid_request','upload path/body mismatch');
-  let bytes;
-  try{bytes=Buffer.from(b.bytes_base64,'base64');}catch{fail(400,'invalid_request','invalid base64');}
-  if(bytes.length>1048576)fail(413,'request_too_large','artifact chunk exceeds 1 MiB');
-  if(sha256(bytes)!==b.sha256)fail(409,'capture_conflict','artifact chunk digest differs');
   const expired=await api.store.one('SELECT id FROM capture_expiries WHERE team_id=? AND source_id=? AND id=?',[auth.team_id,sourceId,uploadId]);
   if(expired)fail(409,'resync_required','upload reservation expired');
   const upload=await api.store.one('SELECT * FROM uploads WHERE team_id=? AND id=?',[auth.team_id,uploadId]);
   if(await expireStaging(api,auth.team_id,sourceId,uploadId,'uploads',upload))fail(409,'resync_required','upload reservation expired');
   if(upload&&upload.source_id!==sourceId)fail(409,'capture_conflict','upload belongs to another source');
-  const old=await api.store.one('SELECT sha256 FROM artifact_chunks WHERE team_id=? AND upload_id=? AND chunk_index=?',[auth.team_id,uploadId,index]);
+  const old=await api.store.one('SELECT sha256 FROM artifact_chunk_data WHERE team_id=? AND upload_id=? AND chunk_index=?',[auth.team_id,uploadId,index]);
   if(old){if(old.sha256!==b.sha256)fail(409,'capture_conflict','artifact chunk replay differs');return send(res,200,result(api,auth,{index,sha256:b.sha256}),requestId);}
+  let bytes;
+  if (b.reuse === true) {
+    // A source may reuse only bytes it already uploaded. Knowing another
+    // source's content hash never grants read access to its native history.
+    const held = await api.store.one(`SELECT c.bytes FROM artifact_chunk_data c
+      JOIN uploads u ON u.team_id=c.team_id AND u.id=c.upload_id
+      WHERE c.team_id=? AND c.sha256=? AND u.source_id=? LIMIT 1`, [auth.team_id,b.sha256,sourceId]);
+    if (!held) fail(404,'chunk_missing','this source has no retained chunk with that digest');
+    bytes = Buffer.from(held.bytes);
+  } else {
+    try { bytes=Buffer.from(b.bytes_base64,'base64'); } catch { fail(400,'invalid_request','invalid base64'); }
+  }
+  if(bytes.length>1048576)fail(413,'request_too_large','artifact chunk exceeds 1 MiB');
+  if(sha256(bytes)!==b.sha256)fail(409,'capture_conflict','artifact chunk digest differs');
+  const blob = await api.store.one('SELECT 1 ok FROM artifact_blobs WHERE team_id=? AND sha256=?',[auth.team_id,b.sha256]);
   // Keep allocation headroom for heartbeats, messages and reclamation. Replays
   // above remain readable; a refused chunk stays in the connector's outbox.
   const usage = api.storageUsage?.();
-  if (usage && usage.used + bytes.length >= usage.limit * 0.9) {
+  if (usage && usage.used + (blob ? 0 : bytes.length) >= usage.limit * 0.9) {
     res.setHeader('Retry-After', '60');
     fail(503, 'storage_pressure', 'native upload deferred while Teams storage is above its headroom threshold');
   }
-  const now=api.clock.nowMs(),response=result(api,auth,{index,sha256:b.sha256});
+  const now=api.clock.nowMs(),response=result(api,auth,{index,sha256:b.sha256,reused:b.reuse===true,stored_bytes:blob?0:bytes.length});
   const guards=[
-    {sql:"SELECT 1 ok WHERE EXISTS (SELECT 1 FROM credentials c JOIN sources s ON s.team_id=c.team_id AND s.id=c.source_id WHERE c.id=? AND c.source_id=? AND c.revoked_at IS NULL AND (c.expires_at IS NULL OR c.expires_at>?) AND s.state='active') AND NOT EXISTS (SELECT 1 FROM capture_expiries WHERE team_id=? AND source_id=? AND id=?) AND NOT EXISTS (SELECT 1 FROM uploads WHERE team_id=? AND id=? AND source_id<>?) AND COALESCE((SELECT committed FROM uploads WHERE team_id=? AND id=?),0)=0 AND COALESCE((SELECT created_at+86400000>? FROM uploads WHERE team_id=? AND id=?),1) AND (SELECT count(*) FROM artifact_chunks WHERE team_id=? AND upload_id=?)=?",params:[auth.credential_id,sourceId,now,auth.team_id,sourceId,uploadId,auth.team_id,uploadId,sourceId,auth.team_id,uploadId,now,auth.team_id,uploadId,auth.team_id,uploadId,index],code:'batch_gap',status:409,message:'upload authorization, expiry, or sequence changed'},
-    {sql:'SELECT 1 ok WHERE COALESCE((SELECT sum(length(bytes)) FROM artifact_chunks WHERE team_id=? AND upload_id=?),0)+?<=?',params:[auth.team_id,uploadId,bytes.length,api.uploadLimitBytes],code:'request_too_large',status:413,message:'artifact exceeds configured upload cap'}
+    {sql:"SELECT 1 ok WHERE EXISTS (SELECT 1 FROM credentials c JOIN sources s ON s.team_id=c.team_id AND s.id=c.source_id WHERE c.id=? AND c.source_id=? AND c.revoked_at IS NULL AND (c.expires_at IS NULL OR c.expires_at>?) AND s.state='active') AND NOT EXISTS (SELECT 1 FROM capture_expiries WHERE team_id=? AND source_id=? AND id=?) AND NOT EXISTS (SELECT 1 FROM uploads WHERE team_id=? AND id=? AND source_id<>?) AND COALESCE((SELECT committed FROM uploads WHERE team_id=? AND id=?),0)=0 AND COALESCE((SELECT created_at+86400000>? FROM uploads WHERE team_id=? AND id=?),1) AND (SELECT count(*) FROM artifact_chunk_data WHERE team_id=? AND upload_id=?)=?",params:[auth.credential_id,sourceId,now,auth.team_id,sourceId,uploadId,auth.team_id,uploadId,sourceId,auth.team_id,uploadId,now,auth.team_id,uploadId,auth.team_id,uploadId,index],code:'batch_gap',status:409,message:'upload authorization, expiry, or sequence changed'},
+    {sql:'SELECT 1 ok WHERE COALESCE((SELECT sum(length(bytes)) FROM artifact_chunk_data WHERE team_id=? AND upload_id=?),0)+?<=?',params:[auth.team_id,uploadId,bytes.length,api.uploadLimitBytes],code:'request_too_large',status:413,message:'artifact exceeds configured upload cap'}
   ];
   await guardedMutation(api,res,auth,guards,[
     {sql:'INSERT OR IGNORE INTO uploads(team_id,id,source_id,created_at) VALUES (?,?,?,?)',params:[auth.team_id,uploadId,sourceId,now]},
-    {sql:'INSERT INTO artifact_chunks(team_id,upload_id,chunk_index,sha256,bytes) VALUES (?,?,?,?,?)',params:[auth.team_id,uploadId,index,b.sha256,bytes]},
+    {sql:'INSERT OR IGNORE INTO artifact_blobs(team_id,sha256,bytes) VALUES (?,?,?)',params:[auth.team_id,b.sha256,bytes]},
+    {sql:'INSERT INTO artifact_chunk_refs(team_id,upload_id,chunk_index,sha256) VALUES (?,?,?,?)',params:[auth.team_id,uploadId,index,b.sha256]},
   ],{status:200,body:response,targetId:uploadId});
   send(res,200,response,requestId);
 }
 async function captureCommit(api,req,res,auth,sourceId,captureId,requestId){const source=await sourceAuth(api,auth,sourceId);const commit=await parseJson(req);exact(commit,['capture_id','parts','digest','title','repository','reported_user','recorded_activity_at','parent_native_id','coverage','artifacts','native_bundle']);if(commit.capture_id!==captureId||!Number.isSafeInteger(commit.parts)||commit.parts<1)fail(400,'invalid_request','invalid commit');const cap=await api.store.one('SELECT * FROM captures WHERE team_id=? AND id=? AND source_id=?',[auth.team_id,captureId,sourceId]);if(await expireStaging(api,auth.team_id,sourceId,captureId,'captures',cap))fail(409,'resync_required','capture reservation expired');if(!cap)fail(404,'not_found','capture not found');if(cap.state==='committed'){const receipt=JSON.parse(cap.receipt_json);if(receipt.digest!==commit.digest)fail(409,'capture_conflict','commit replay differs');receipt.duplicate=true;return send(res,200,result(api,auth,receipt),requestId);}const rows=await api.store.query('SELECT * FROM capture_parts WHERE team_id=? AND capture_id=? ORDER BY part_index',[auth.team_id,captureId]);if(rows.length!==commit.parts)fail(409,'batch_gap','capture parts incomplete');const begin=JSON.parse(cap.begin_json);if(source.kind==='connector'){const rule=await api.store.one('SELECT * FROM sync_rules WHERE team_id=? AND id=? AND source_id=? AND machine_id=?',[auth.team_id,begin.rule_id,sourceId,auth.machine_id]);if(!rule||rule.state!=='active'||rule.revision!==begin.rule_revision||rule.visibility!==begin.visibility)fail(rule?.state==='paused'?409:403,rule?.state==='paused'?'source_paused':'forbidden','sync rule changed before commit');}const parts=rows.map(x=>JSON.parse(x.payload_json));if(captureDigest(begin,parts,commit)!==commit.digest)fail(409,'capture_conflict','capture digest differs');let messages=captureMessages(parts);if(messages.length>100000)fail(413,'request_too_large','capture contains too many messages');
-  for(const manifest of commit.artifacts){validateArtifactName(manifest.relative_name);const upload=await api.store.one('SELECT source_id FROM uploads WHERE team_id=? AND id=?',[auth.team_id,manifest.upload_id]);if(!upload||upload.source_id!==sourceId)fail(409,'capture_conflict','artifact upload belongs to another source');const chunks=await api.store.query('SELECT c.* FROM artifact_chunks c JOIN uploads u ON u.team_id=c.team_id AND u.id=c.upload_id WHERE c.team_id=? AND c.upload_id=? AND u.source_id=? ORDER BY c.chunk_index',[auth.team_id,manifest.upload_id,sourceId]);const bytes=Buffer.concat(chunks.map(x=>x.bytes));if(chunks.length!==manifest.chunks||bytes.length!==manifest.byte_length||sha256(bytes)!==manifest.sha256)fail(409,'capture_conflict','artifact manifest does not match staged chunks');}
+  for(const manifest of commit.artifacts){validateArtifactName(manifest.relative_name);const upload=await api.store.one('SELECT source_id FROM uploads WHERE team_id=? AND id=?',[auth.team_id,manifest.upload_id]);if(!upload||upload.source_id!==sourceId)fail(409,'capture_conflict','artifact upload belongs to another source');const chunks=await api.store.query('SELECT c.* FROM artifact_chunk_data c JOIN uploads u ON u.team_id=c.team_id AND u.id=c.upload_id WHERE c.team_id=? AND c.upload_id=? AND u.source_id=? ORDER BY c.chunk_index',[auth.team_id,manifest.upload_id,sourceId]);const bytes=Buffer.concat(chunks.map(x=>x.bytes));if(chunks.length!==manifest.chunks||bytes.length!==manifest.byte_length||sha256(bytes)!==manifest.sha256)fail(409,'capture_conflict','artifact manifest does not match staged chunks');}
   if(commit.native_bundle!=null){const bundle=commit.native_bundle;exact(bundle,['id','harness','fidelity','residue','files'],'native_bundle');if(typeof bundle.id!=='string'||bundle.harness!==cap.harness||!['byte_lossless','value_lossless','semantic'].includes(bundle.fidelity)||!Array.isArray(bundle.residue)||!bundle.residue.every(x=>typeof x==='string')||!Array.isArray(bundle.files))fail(400,'invalid_request','invalid native bundle');const artifactIds=new Set(commit.artifacts.map(x=>x.upload_id)),seen=new Set();for(const file of bundle.files){exact(file,['upload_id','role'],'native_bundle file');if(!artifactIds.has(file.upload_id)||seen.has(file.upload_id)||!['primary','subagent','bundle','source_recovery'].includes(file.role))fail(400,'invalid_request','native bundle files must uniquely reference captured artifacts');seen.add(file.upload_id);}if(seen.size!==artifactIds.size||bundle.files.filter(x=>x.role==='primary').length!==1||new Set(commit.artifacts.map(x=>x.relative_name)).size!==commit.artifacts.length)fail(400,'invalid_request','native bundle must describe every unique artifact and one primary');}
   let origin=await api.store.one('SELECT * FROM origins WHERE team_id=? AND source_id=? AND harness=? AND native_id=?',[auth.team_id,sourceId,cap.harness,cap.native_id]);
   // A capture that carries (schema 9's wire half) names its first carry.count messages by digest instead of sending

package/server/storage-report.mjs

@@ -20,12 +20,17 @@ export async function storageReport({ api, res, auth, url, requestId }) {
       p.native_id,p.harness,p.captured_at,
       json_extract(u.manifest_json,'$.role') role,json_extract(u.manifest_json,'$.media_type') media_type,
       json_extract(u.manifest_json,'$.sha256') sha256,
-      (SELECT COALESCE(SUM(length(bytes)),0) FROM artifact_chunks c WHERE c.team_id=u.team_id AND c.upload_id=u.id) bytes,
+      (SELECT COALESCE(SUM(length(bytes)),0) FROM artifact_chunk_data c WHERE c.team_id=u.team_id AND c.upload_id=u.id) bytes,
       EXISTS (SELECT 1 FROM session_artifacts a WHERE a.team_id=u.team_id AND a.upload_id=u.id) version_referenced,
       EXISTS (SELECT 1 FROM continuations c WHERE c.team_id=u.team_id AND instr(c.value_json,u.id)>0) continuation_referenced,
       EXISTS (SELECT 1 FROM captures c WHERE c.team_id=u.team_id AND c.state='staging' AND instr(c.begin_json,u.id)>0) staging_referenced
     FROM page u JOIN sources s ON s.team_id=u.team_id AND s.id=u.source_id
     LEFT JOIN machines m ON m.team_id=s.team_id AND m.id=s.machine_id
     LEFT JOIN provenance p ON p.team_id=u.team_id AND p.upload_id=u.id ORDER BY u.id`, [auth.team_id, after, auth.team_id]);
-  send(res, 200, result(api, auth, { items: rows, next: rows.length === 100 ? rows.at(-1).id : null, usage: api.storageUsage?.() ?? null }), requestId);
+  const chunks = await api.store.one(`SELECT
+    (SELECT COALESCE(SUM(length(bytes)),0) FROM artifact_blobs WHERE team_id=?) shared_bytes,
+    (SELECT COUNT(*) FROM artifact_blobs WHERE team_id=?) shared_chunks,
+    (SELECT COUNT(*) FROM artifact_chunk_refs WHERE team_id=?) chunk_references,
+    (SELECT COALESCE(SUM(length(bytes)),0) FROM artifact_chunks WHERE team_id=?) legacy_bytes`, Array(4).fill(auth.team_id));
+  send(res, 200, result(api, auth, { items: rows, next: rows.length === 100 ? rows.at(-1).id : null, usage: api.storageUsage?.() ?? null, chunks }), requestId);
 }

package/server/store-core.mjs

@@ -111,7 +111,7 @@ export function createStoreHandlers({ all, get, run, tx }) {
     const upload=get(`SELECT u.manifest_json FROM session_artifacts a JOIN uploads u ON u.team_id=a.team_id AND u.id=a.upload_id
       WHERE a.team_id=? AND a.session_id=? AND a.upload_id=? AND u.committed=1`,[args.teamId,session.id,args.uploadId]);
     if(!upload)domainError(404,'not_found','artifact not found');
-    const chunks=all('SELECT bytes FROM artifact_chunks WHERE team_id=? AND upload_id=? ORDER BY chunk_index',[args.teamId,args.uploadId]);
+    const chunks=all('SELECT bytes FROM artifact_chunk_data WHERE team_id=? AND upload_id=? ORDER BY chunk_index',[args.teamId,args.uploadId]);
     return {manifest:JSON.parse(upload.manifest_json),bytes:Buffer.concat(chunks.map(row=>row.bytes))};
   });
   return {