@volter/supercode-teams 0.2.148 → 0.2.150

package/connector/index.mjs

@@ -408,6 +408,7 @@ export class TeamsConnector {
   // Whether a capture may attach the native bundle of a file in this state: its translation is cached (and did
   // not fail), or it has been idle for nativeIdleMs.
   nativeReady(key,stat){
+    if ((this.state.native_upload_deferred_until ?? 0) > this.clock.nowMs()) return false;
     const cached=this.state.native_cache[key];
     if(cached?.stamp===fileStamp(stat))return !cached.failed;
     return this.clock.nowMs()-stat.mtimeMs>=this.nativeIdleMs;
@@ -739,6 +740,17 @@ export class TeamsConnector {
     item.artifacts=[];item.commit.artifacts=[];delete item.commit.native_bundle;item.nativeDeferred=true;
     const digestCommit={...item.commit};delete digestCommit.digest;item.commit.digest=hash(canonicalJson({begin:item.begin,parts:item.parts,commit:digestCommit}));this.save();
   }
+  deferNativeUnderPressure(item) {
+    if (!item.artifacts?.length) return false;
+    // A native bundle waiting for disk headroom must not hold every message
+    // behind it in the shared outbox. Commit the messages and retry the native
+    // bundle after pressure clears, using the same deferred-capture mechanism.
+    item.artifacts=[];item.commit.artifacts=[];delete item.commit.native_bundle;item.nativeDeferred=true;
+    const commit={...item.commit};delete commit.digest;
+    item.commit.digest=hash(canonicalJson({begin:item.begin,parts:item.parts,commit}));
+    this.state.native_upload_deferred_until=this.clock.nowMs()+60000;
+    this.save();return true;
+  }
   // A capture queued before parts were bounded by bytes can hold a part the server refuses as too large, and the
   // outbox is flushed in order: it is split again, under a new capture id (the server drops the old reservation of
   // the same revision) and its digest resealed, so it neither fails nor holds every later capture behind it.
@@ -787,7 +799,7 @@ export class TeamsConnector {
     if(!Array.isArray(configuration?.rules))return false;
     return !configuration.rules.some(rule=>rule.id===item.begin.rule_id&&rule.revision===item.begin.rule_revision&&rule.state==='active');
   }
-  async flushOutbox(){let latest=null;while(this.state.outbox.length){const item=this.state.outbox[0];let receipt;this.dropLostArtifacts(item);this.refitParts(item);try{receipt=await this.sendCapture(item);}catch(error){if(error.status===409&&error.code==='carry_mismatch'){const key=`${item.begin.harness}:${item.begin.native_session_id}`;this.lastError=error;process.stderr.write(`supercode teams: the server does not hold what a capture of ${item.begin.harness} session ${item.begin.native_session_id} carried; it goes whole next time\n`);this.state.outbox.shift();if(this.state.origins[key])this.state.origins[key]={...this.state.origins[key],observed_keys:[],observed_digests:[],carry_refused:true};this.save();continue;}if(error.status===400&&error.code==='invalid_request'||error.status===413){this.lastError=error;this.noteSessionProblem({harness:item.begin.harness,nativeSessionId:item.begin.native_session_id,ruleId:item.begin.rule_id??null},error);(this.droppedCaptures??=new Set()).add(`${item.begin.harness}:${item.begin.native_session_id}`);process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id} the server refused${error.status===413?' (one message is larger than a request may be)':' as malformed'}: ${error.message}\n`);this.state.outbox.shift();this.save();continue;}if(!await this.ruleRejected(item,error))throw error;this.lastError=error;process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id}; its sync rule changed or was removed: ${error.message}\n`);this.state.outbox.shift();this.save();continue;}const key=`${item.begin.harness}:${item.begin.native_session_id}`,prior=this.state.origins[key]??{},next={...prior,generation:item.begin.generation,sequence:item.begin.sequence,digest:item.commit.digest,observed_keys:item.observed?.keys??prior.observed_keys,observed_digests:item.observed?.digests??prior.observed_digests,last_rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode};if(item.clearBaseline){delete next.baseline;delete next.baseline_digests;delete next.baseline_tool_tail;}if(item.nativeDeferred)next.native_deferred=true;else delete next.native_deferred;if(!item.begin.carry)delete next.carry_refused;if(item.stamp&&item.begin.rule_id)next.capture_stamps={...prior.capture_stamps,[item.begin.rule_id]:{stamp:item.stamp,rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode}};this.state.origins[key]=next;this.state.outbox.shift();this.save();// A queued capture is often flushed by another session's capture: its own session's problem ends with it.
+  async flushOutbox(){let latest=null;while(this.state.outbox.length){const item=this.state.outbox[0];let receipt;this.dropLostArtifacts(item);this.refitParts(item);try{receipt=await this.sendCapture(item);}catch(error){if(error.code==='storage_pressure'&&this.deferNativeUnderPressure(item))continue;if(error.status===409&&error.code==='carry_mismatch'){const key=`${item.begin.harness}:${item.begin.native_session_id}`;this.lastError=error;process.stderr.write(`supercode teams: the server does not hold what a capture of ${item.begin.harness} session ${item.begin.native_session_id} carried; it goes whole next time\n`);this.state.outbox.shift();if(this.state.origins[key])this.state.origins[key]={...this.state.origins[key],observed_keys:[],observed_digests:[],carry_refused:true};this.save();continue;}if(error.status===400&&error.code==='invalid_request'||error.status===413){this.lastError=error;this.noteSessionProblem({harness:item.begin.harness,nativeSessionId:item.begin.native_session_id,ruleId:item.begin.rule_id??null},error);(this.droppedCaptures??=new Set()).add(`${item.begin.harness}:${item.begin.native_session_id}`);process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id} the server refused${error.status===413?' (one message is larger than a request may be)':' as malformed'}: ${error.message}\n`);this.state.outbox.shift();this.save();continue;}if(!await this.ruleRejected(item,error))throw error;this.lastError=error;process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id}; its sync rule changed or was removed: ${error.message}\n`);this.state.outbox.shift();this.save();continue;}const key=`${item.begin.harness}:${item.begin.native_session_id}`,prior=this.state.origins[key]??{},next={...prior,generation:item.begin.generation,sequence:item.begin.sequence,digest:item.commit.digest,observed_keys:item.observed?.keys??prior.observed_keys,observed_digests:item.observed?.digests??prior.observed_digests,last_rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode};if(item.clearBaseline){delete next.baseline;delete next.baseline_digests;delete next.baseline_tool_tail;}if(item.nativeDeferred)next.native_deferred=true;else delete next.native_deferred;if(!item.begin.carry)delete next.carry_refused;if(item.stamp&&item.begin.rule_id)next.capture_stamps={...prior.capture_stamps,[item.begin.rule_id]:{stamp:item.stamp,rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode}};this.state.origins[key]=next;this.state.outbox.shift();this.save();// A queued capture is often flushed by another session's capture: its own session's problem ends with it.
 this.clearSessionProblem(item.begin.harness,item.begin.native_session_id);if(item.artifacts?.length)this.pruneSpill();latest=receipt;}return latest;}
   stop(){try{this.flushState();}catch{}if(this.timer)clearInterval(this.timer);if(this.liveCaptureTimer)clearInterval(this.liveCaptureTimer);if(this.heartbeatTimer)clearInterval(this.heartbeatTimer);this.abort.abort();for(const follower of this.followers?.values()??[])follower.controller.abort();return this.nativeEffects.close();}
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.2.148",
+  "version": "0.2.150",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",

package/server/harness-routes.mjs

@@ -307,8 +307,8 @@ async function artifactChunk(api,req,res,auth,sourceId,uploadId,index,requestId)
   const blob = await api.store.one('SELECT 1 ok FROM artifact_blobs WHERE team_id=? AND sha256=?',[auth.team_id,b.sha256]);
   // Keep allocation headroom for heartbeats, messages and reclamation. Replays
   // above remain readable; a refused chunk stays in the connector's outbox.
-  const usage = api.storageUsage?.();
-  if (usage && usage.used + (blob ? 0 : bytes.length) >= usage.limit * 0.9) {
+  const usage = await api.storageUsage?.();
+  if (usage && (usage.used + (blob ? 0 : bytes.length) >= usage.limit * 0.9 || (!blob && usage.available_bytes != null && usage.available_bytes - bytes.length < (usage.min_free_bytes ?? 0)))) {
     res.setHeader('Retry-After', '60');
     fail(503, 'storage_pressure', 'native upload deferred while Teams storage is above its headroom threshold');
   }

package/server/index.mjs

@@ -25,11 +25,11 @@ export function createTeamsServer({dataDir,listen={host:'127.0.0.1',port:7331},p
   // The harness's own routes on this server (harness-routes.mjs); the handler owns the neutral ones.
   api.attach({harnessRoutes});
   const events=api.events;
-  let store,http,upgrade,lockFd,maintenanceTimer,started=false;
+  let store,http,upgrade,lockFd,maintenanceTimer,maintenanceWork,started=false;
   api.start=async()=>{
     if(started)throw new Error('Teams server already started'); mkdirSync(dataDir,{recursive:true,mode:0o700});chmodSync(dataDir,0o700);
     const lock=join(dataDir,'server.lock'); const acquire=()=>{lockFd=openSync(lock,'wx',0o600);writeFileSync(lockFd,JSON.stringify({pid:process.pid,started_at:clock.nowMs()}));};try{acquire();}catch(e){let recover=false;try{const owner=JSON.parse(readFileSync(lock,'utf8'));if(Number.isInteger(owner.pid)&&owner.pid>1){try{process.kill(owner.pid,0);}catch(error){recover=error.code==='ESRCH';}}}catch{}if(!recover)throw new Error(`Teams data directory is already owned: ${dataDir}`,{cause:e});unlinkSync(lock);acquire();}
-    try { const dbPath=join(dataDir,'teams.sqlite'); store=new TeamsStore(dbPath); api.attach({store});
+    try { const dbPath=join(dataDir,'teams.sqlite'); store=new TeamsStore(dbPath,{storageLimitBytes:Number(process.env.TEAMS_STORAGE_LIMIT_BYTES)||10*1024**3,storageMinFreeBytes:Number(process.env.TEAMS_STORAGE_MIN_FREE_BYTES)||0}); api.attach({store}); api.storageUsage=()=>store.call('storageUsage');
       const existing=await store.one("SELECT value FROM server_meta WHERE key='server_id'");
       const credentialPath=join(dataDir,'bootstrap-credential.json'),pendingPath=join(dataDir,'bootstrap-credential.pending.json');
       let serverId;
@@ -43,10 +43,23 @@ export function createTeamsServer({dataDir,listen={host:'127.0.0.1',port:7331},p
         if(!existsSync(credentialPath)&&existsSync(pendingPath)){const pending=JSON.parse(readFileSync(pendingPath,'utf8'));if(pending.server_id!==serverId)throw new Error('bootstrap credential recovery does not match server identity');renameSync(pendingPath,credentialPath);chmodSync(credentialPath,0o600);}
       }
       api.attach({serverId});
-      await maintain(api);maintenanceTimer=setInterval(()=>{void maintain(api).catch(()=>{});},60000);maintenanceTimer.unref?.();
+      const maintenance=async()=>{
+        let next=60000;
+        try{
+          const result=await maintain(api);next=result?.backlog?3000:60000;
+          if(!result?.backlog){const reclaimed=await store.call('reclaimStorage');if(reclaimed.after.allocated<reclaimed.before.allocated)console.log(`Teams storage reclaimed: ${JSON.stringify(reclaimed)}`);}
+          const usage=await api.storageUsage();
+          if(usage.used>=usage.limit*0.8||usage.available_bytes<usage.min_free_bytes+usage.limit*0.2)console.warn(`Teams storage pressure: ${JSON.stringify(usage)}; native admission stops at90% or the configured disk reserve`);
+        }catch(error){console.error(`Teams maintenance failed: ${error.message}`);next=3000;}
+        if(started){maintenanceTimer=setTimeout(()=>{maintenanceWork=maintenance();},next);maintenanceTimer.unref?.();}
+        return next;
+      };
+      const delay=await maintenance();
+      // The initial pass precedes listen; subsequent passes never overlap.
+      maintenanceTimer=setTimeout(()=>{maintenanceWork=maintenance();},delay);maintenanceTimer.unref?.();
       chmodSync(dbPath,0o600); http=createHttpServer(createHttpHandler(api));upgrade=createMachineUpgradeHandler(api,{authenticate});http.on('upgrade',upgrade); await new Promise((resolve,reject)=>{http.once('error',reject);http.listen(listen.port??0,listen.host??'127.0.0.1',resolve);});started=true;const addr=http.address();api.attach({localOrigin:`http://${addr.address.includes(':')?`[${addr.address}]`:addr.address}:${addr.port}`});return {address:`${addr.address}:${addr.port}`,server_id:serverId};
     } catch(e){if(maintenanceTimer){clearInterval(maintenanceTimer);maintenanceTimer=undefined;}await store?.close().catch(()=>{});store=null;api.attach({store:null});if(lockFd!==undefined){closeSync(lockFd);lockFd=undefined;}try{unlinkSync(lock);}catch{}throw e;}
   };
-  api.close=async()=>{if(!started&&lockFd===undefined)return;started=false;if(maintenanceTimer){clearInterval(maintenanceTimer);maintenanceTimer=undefined;}events.close();closeAttachmentRelays(api,'access_revoked');upgrade?.close();await new Promise(resolve=>http?.close(()=>resolve()));await store?.close();store=null;if(lockFd!==undefined){closeSync(lockFd);lockFd=undefined;}try{unlinkSync(join(dataDir,'server.lock'));}catch{}api.attach({store:null});};
+  api.close=async()=>{if(!started&&lockFd===undefined)return;started=false;if(maintenanceTimer){clearInterval(maintenanceTimer);maintenanceTimer=undefined;}events.close();closeAttachmentRelays(api,'access_revoked');upgrade?.close();await new Promise(resolve=>http?.close(()=>resolve()));await maintenanceWork;await store?.close();store=null;if(lockFd!==undefined){closeSync(lockFd);lockFd=undefined;}try{unlinkSync(join(dataDir,'server.lock'));}catch{}api.attach({store:null});};
   return api;
 }

package/server/storage-report.mjs

@@ -32,5 +32,5 @@ export async function storageReport({ api, res, auth, url, requestId }) {
     (SELECT COUNT(*) FROM artifact_blobs WHERE team_id=?) shared_chunks,
     (SELECT COUNT(*) FROM artifact_chunk_refs WHERE team_id=?) chunk_references,
     (SELECT COALESCE(SUM(length(bytes)),0) FROM artifact_chunks WHERE team_id=?) legacy_bytes`, Array(4).fill(auth.team_id));
-  send(res, 200, result(api, auth, { items: rows, next: rows.length === 100 ? rows.at(-1).id : null, usage: api.storageUsage?.() ?? null, chunks }), requestId);
+  send(res, 200, result(api, auth, { items: rows, next: rows.length === 100 ? rows.at(-1).id : null, usage: await api.storageUsage?.() ?? null, chunks }), requestId);
 }

package/server/store-worker.mjs

@@ -1,5 +1,6 @@
 import { parentPort, workerData, isMainThread, Worker } from 'node:worker_threads';
 import { DatabaseSync } from 'node:sqlite';
+import { statfsSync,statSync } from 'node:fs';
 import { createStoreHandlers, migrate, StoreClient } from './store-core.mjs';
 
 // The Node host's store: node:sqlite in a worker thread, so a long statement never
@@ -12,12 +13,27 @@ if (!isMainThread) {
   const run = (sql, p=[]) => db.prepare(sql).run(...p);
   const tx = fn => { db.exec('BEGIN IMMEDIATE'); try { const out=fn(); db.exec('COMMIT'); return out; } catch(e) { db.exec('ROLLBACK'); throw e; } };
   migrate({ exec: (sql) => db.exec(sql), tx, version: () => db.prepare('PRAGMA user_version').get().user_version, setVersion: (n) => db.exec(`PRAGMA user_version=${n}`) });
-  const handlers = { ...createStoreHandlers({ all, get, run, tx }), close: () => { db.close(); return true; } };
+  const usage=()=>{
+    const size=get('PRAGMA page_size').page_size,pages=get('PRAGMA page_count').page_count,free=get('PRAGMA freelist_count').freelist_count;
+    const disk=statfsSync(workerData.dbPath);let wal=0;try{wal=statSync(`${workerData.dbPath}-wal`).size;}catch{}
+    return {allocated:pages*size,free_bytes:free*size,used:(pages-free)*size,used_is_upper_bound:false,wal_bytes:wal,available_bytes:disk.bavail*disk.bsize,limit:workerData.storageLimitBytes,min_free_bytes:workerData.storageMinFreeBytes};
+  };
+  const reclaim=()=>{
+    db.exec('PRAGMA wal_checkpoint(PASSIVE)');
+    const before=usage();
+    // Legacy databases have auto_vacuum=NONE. Compact through the owning store
+    // after retention drains, with room for SQLite's temporary live-page copy.
+    if(before.free_bytes>=512*1024*1024&&before.free_bytes>before.allocated/2&&before.available_bytes>before.used+64*1024*1024){
+      db.exec('VACUUM');db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
+    }
+    return {before,after:usage()};
+  };
+  const handlers = { ...createStoreHandlers({ all, get, run, tx }), storageUsage:usage, reclaimStorage:reclaim, close: () => { db.close(); return true; } };
   parentPort.on('message', ({id,op,args}) => { try { parentPort.postMessage({id,result:handlers[op](args)}); } catch(error) { parentPort.postMessage({id,error:{message:error.message,code:error.code,status:error.status,stack:error.stack}}); } });
 }
 
 export class TeamsStore extends StoreClient {
-  constructor(dbPath) { super(); this.worker=new Worker(new URL(import.meta.url),{workerData:{dbPath}}); this.next=1; this.pending=new Map(); this.worker.on('message',m=>{const p=this.pending.get(m.id); if(!p)return; this.pending.delete(m.id); m.error?p.reject(Object.assign(new Error(m.error.message),m.error)):p.resolve(m.result);}); this.worker.on('error',e=>{for(const p of this.pending.values())p.reject(e);this.pending.clear();}); }
+  constructor(dbPath,{storageLimitBytes=10*1024**3,storageMinFreeBytes=0}={}) { super(); this.worker=new Worker(new URL(import.meta.url),{workerData:{dbPath,storageLimitBytes,storageMinFreeBytes}}); this.next=1; this.pending=new Map(); this.worker.on('message',m=>{const p=this.pending.get(m.id); if(!p)return; this.pending.delete(m.id); m.error?p.reject(Object.assign(new Error(m.error.message),m.error)):p.resolve(m.result);}); this.worker.on('error',e=>{for(const p of this.pending.values())p.reject(e);this.pending.clear();}); }
   call(op,args={}) { return new Promise((resolve,reject)=>{const id=this.next++;this.pending.set(id,{resolve,reject});this.worker.postMessage({id,op,args});}); }
   async close() { await this.call('close'); await this.worker.terminate(); }
 }