@volter/supercode-teams 0.2.140 → 0.2.142

package/connector/index.d.ts

@@ -38,9 +38,9 @@ export interface TeamsConnectorOptions {
   fetch?: typeof globalThis.fetch;
   /** Milliseconds between heartbeats, sent on their own timer independent of the capture pass (default 15000). */
   heartbeatMs?: number;
-  /** A native file at least nativeIdleBytes long gets its native bundle only after this long without a write (default 60000). */
+  /** A native file gets its native bundle only after this long without a write (default 60000). */
   nativeIdleMs?: number;
-  /** Size from which a native file's bundle waits for it to be idle (default 8 MiB). */
+  /** Size from which translated native bundles are cached locally (default 8 MiB). */
   nativeIdleBytes?: number;
 }
 

package/connector/index.mjs

@@ -72,7 +72,8 @@ const CHUNK_BYTES=1024*1024;
 // harness server's only request reader while it runs; a live Claude Code transcript is appended to on every
 // message and tool result, so translating it mid-turn is wasted (the file has moved by the time it finishes).
 // A minute is four beats with no write: the turn is over, or waiting on the user or one long tool call.
-// Smaller files translate in well under a second and keep getting a bundle with every capture.
+// Native bundles wait for a quiet file at every size; uploading a whole small
+// transcript on every live capture also grows quadratically with the session.
 const NATIVE_IDLE_MS=60000,NATIVE_IDLE_BYTES=8*1024*1024;
 // Loading a transcript reads all of it, and a live one changes on every message: a 135 MB Claude transcript took
 // over 30 s a load on a Windows PC, on every beat, and translating it held the harness server past 272 s while every
@@ -405,11 +406,11 @@ export class TeamsConnector {
     return false;
   }
   // Whether a capture may attach the native bundle of a file in this state: its translation is cached (and did
-  // not fail), or it is small, or it has been idle for nativeIdleMs.
+  // not fail), or it has been idle for nativeIdleMs.
   nativeReady(key,stat){
     const cached=this.state.native_cache[key];
     if(cached?.stamp===fileStamp(stat))return !cached.failed;
-    return stat.size<this.nativeIdleBytes||this.clock.nowMs()-stat.mtimeMs>=this.nativeIdleMs;
+    return this.clock.nowMs()-stat.mtimeMs>=this.nativeIdleMs;
   }
   // Native artifact bytes are spilled to content-addressed files under the state directory, never into
   // connector.json: a 129 MB transcript was ~172 MB of base64 held in the outbox and rewritten on every save().

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.2.140",
+  "version": "0.2.142",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",

package/server/harness-routes.mjs

@@ -7,6 +7,7 @@
 // idempotency check, and last among a team's routes. Each hook returns true when it
 // answered the request.
 import { bearer, hashToken, canRead, canManage, isAdmin } from '@volter/teams/server/auth';
+import { storageReport } from './storage-report.mjs';
 import { validateListQuery, validateSearchQuery, validateReadQuery, validateCaptureBegin, validateCapturePart, canonicalJson, carryDigest } from '../contracts/index.mjs';
 import { normalizeSearch, textFromContent, cursorHash, cursorToken } from './catalog.mjs';
 import { captureDigest, captureMessages, handleOtlpRoute, sha256, validateArtifactName } from './ingest.mjs';
@@ -77,6 +78,7 @@ async function beforeIdempotency({ api, req, res, auth, url, path, sub, requestI
 
 async function team({ api, req, res, auth, url, path, sub, requestId }) {
   let m;
+  if(req.method==='GET'&&sub==='/storage/uploads'){await storageReport({api,res,auth,url,requestId});return true;}
   if(req.method==='GET'&&sub==='/sessions'){await listSessions(api,res,auth,url,requestId,false);return true;}
   if(req.method==='GET'&&sub==='/sessions/search'){await listSessions(api,res,auth,url,requestId,true);return true;}
   if(req.method==='GET'&&sub==='/sessions/events'){await handleCatalogEvents({api,req,res,auth,url,requestId,qobj,canReadMerged,readerProjection,fail});return true;}
@@ -288,6 +290,13 @@ async function artifactChunk(api,req,res,auth,sourceId,uploadId,index,requestId)
   if(upload&&upload.source_id!==sourceId)fail(409,'capture_conflict','upload belongs to another source');
   const old=await api.store.one('SELECT sha256 FROM artifact_chunks WHERE team_id=? AND upload_id=? AND chunk_index=?',[auth.team_id,uploadId,index]);
   if(old){if(old.sha256!==b.sha256)fail(409,'capture_conflict','artifact chunk replay differs');return send(res,200,result(api,auth,{index,sha256:b.sha256}),requestId);}
+  // Keep allocation headroom for heartbeats, messages and reclamation. Replays
+  // above remain readable; a refused chunk stays in the connector's outbox.
+  const usage = api.storageUsage?.();
+  if (usage && usage.used + bytes.length >= usage.limit * 0.9) {
+    res.setHeader('Retry-After', '60');
+    fail(503, 'storage_pressure', 'native upload deferred while Teams storage is above its headroom threshold');
+  }
   const now=api.clock.nowMs(),response=result(api,auth,{index,sha256:b.sha256});
   const guards=[
     {sql:"SELECT 1 ok WHERE EXISTS (SELECT 1 FROM credentials c JOIN sources s ON s.team_id=c.team_id AND s.id=c.source_id WHERE c.id=? AND c.source_id=? AND c.revoked_at IS NULL AND (c.expires_at IS NULL OR c.expires_at>?) AND s.state='active') AND NOT EXISTS (SELECT 1 FROM capture_expiries WHERE team_id=? AND source_id=? AND id=?) AND NOT EXISTS (SELECT 1 FROM uploads WHERE team_id=? AND id=? AND source_id<>?) AND COALESCE((SELECT committed FROM uploads WHERE team_id=? AND id=?),0)=0 AND COALESCE((SELECT created_at+86400000>? FROM uploads WHERE team_id=? AND id=?),1) AND (SELECT count(*) FROM artifact_chunks WHERE team_id=? AND upload_id=?)=?",params:[auth.credential_id,sourceId,now,auth.team_id,sourceId,uploadId,auth.team_id,uploadId,sourceId,auth.team_id,uploadId,now,auth.team_id,uploadId,auth.team_id,uploadId,index],code:'batch_gap',status:409,message:'upload authorization, expiry, or sequence changed'},

package/server/storage-report.mjs

@@ -0,0 +1,31 @@
+// Administrative, read-only retention inventory. No artifact bytes, native
+// paths, prompts or credentials leave this door; cursors never write storage.
+import { isAdmin } from '@volter/teams/server/auth';
+import { fail, result, send } from '@volter/teams/server/http';
+
+export async function storageReport({ api, res, auth, url, requestId }) {
+  if (!isAdmin(auth)) fail(403, 'forbidden', 'storage inventory requires a team administrator');
+  for (const key of url.searchParams.keys()) if (key !== 'after') fail(400, 'invalid_request', `unknown query field ${key}`);
+  const after = url.searchParams.get('after') ?? '';
+  if (after && !/^upload_[a-zA-Z0-9_-]{1,100}$/.test(after)) fail(400, 'invalid_request', 'invalid upload cursor');
+  const rows = await api.store.query(`WITH page AS MATERIALIZED (
+      SELECT * FROM uploads WHERE team_id=? AND id>? ORDER BY id LIMIT 100
+    ), provenance AS MATERIALIZED (
+      SELECT c.team_id,json_extract(a.value,'$.upload_id') upload_id,
+        c.native_id,c.harness,MIN(c.created_at) captured_at
+      FROM captures c,json_each(c.commit_json,'$.artifacts') a
+      WHERE c.team_id=? AND json_extract(a.value,'$.upload_id') IN (SELECT id FROM page)
+      GROUP BY c.team_id,json_extract(a.value,'$.upload_id')
+    ) SELECT u.id,u.source_id,s.machine_id,m.name machine,u.created_at,u.committed,
+      p.native_id,p.harness,p.captured_at,
+      json_extract(u.manifest_json,'$.role') role,json_extract(u.manifest_json,'$.media_type') media_type,
+      json_extract(u.manifest_json,'$.sha256') sha256,
+      (SELECT COALESCE(SUM(length(bytes)),0) FROM artifact_chunks c WHERE c.team_id=u.team_id AND c.upload_id=u.id) bytes,
+      EXISTS (SELECT 1 FROM session_artifacts a WHERE a.team_id=u.team_id AND a.upload_id=u.id) version_referenced,
+      EXISTS (SELECT 1 FROM continuations c WHERE c.team_id=u.team_id AND instr(c.value_json,u.id)>0) continuation_referenced,
+      EXISTS (SELECT 1 FROM captures c WHERE c.team_id=u.team_id AND c.state='staging' AND instr(c.begin_json,u.id)>0) staging_referenced
+    FROM page u JOIN sources s ON s.team_id=u.team_id AND s.id=u.source_id
+    LEFT JOIN machines m ON m.team_id=s.team_id AND m.id=s.machine_id
+    LEFT JOIN provenance p ON p.team_id=u.team_id AND p.upload_id=u.id ORDER BY u.id`, [auth.team_id, after, auth.team_id]);
+  send(res, 200, result(api, auth, { items: rows, next: rows.length === 100 ? rows.at(-1).id : null, usage: api.storageUsage?.() ?? null }), requestId);
+}

package/worker/index.mjs

@@ -72,52 +72,22 @@ function durableSql(storage) {
   return { all, get, run, tx, exec: (query) => sql.exec(query) };
 }
 
-/** Aggregate storage diagnostics only: no record contents or credentials. */
-function logStorageBreakdown(sql) {
-  console.log(`storage: database_bytes=${sql.databaseSize}`);
-  try {
-    for (const row of sql.exec('SELECT name,SUM(pgsize) bytes,COUNT(*) pages FROM dbstat GROUP BY name ORDER BY SUM(pgsize) DESC')) console.log(`storage table: ${JSON.stringify(row)}`);
-    return;
-  } catch (error) { console.log(`storage: physical table allocation unavailable: ${error.message}`); }
-  // Some hosts omit dbstat. Logical payload lengths exclude indexes/page
-  // overhead; TEXT lengths count characters and BLOB lengths count bytes.
-  const quote = name => `"${name.replaceAll('"', '""')}"`;
-  for (const { name } of sql.exec("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")) {
-    try {
-      const columns = [...sql.exec(`PRAGMA table_info(${quote(name)})`)];
-      const payload = columns.filter(column => /TEXT|BLOB/i.test(column.type)).map(column => `COALESCE(length(${quote(column.name)}),0)`);
-      const [row] = sql.exec(`SELECT COUNT(*) rows,COALESCE(SUM(${payload.join('+') || '0'}),0) payload_units FROM ${quote(name)}`);
-      console.log(`storage table logical: ${JSON.stringify({ name, ...row })}`);
-    } catch (error) { console.log(`storage table ${name}: ${error.message}`); }
-  }
-  try {
-    const now = Date.now();
-    const rows = sql.exec(`WITH chunk_sizes AS MATERIALIZED (SELECT team_id,upload_id,COUNT(*) chunks,SUM(length(bytes)) bytes FROM artifact_chunks GROUP BY team_id,upload_id),
-      staging AS MATERIALIZED (SELECT team_id,begin_json FROM captures WHERE state='staging'),
-      classified AS MATERIALIZED (SELECT u.team_id,u.id,u.created_at,
-        CASE WHEN EXISTS (SELECT 1 FROM session_artifacts a WHERE a.team_id=u.team_id AND a.upload_id=u.id) THEN 'version-referenced'
-        WHEN EXISTS (SELECT 1 FROM continuations c WHERE c.team_id=u.team_id AND instr(c.value_json,u.id)>0) THEN 'continuation-referenced'
-        WHEN EXISTS (SELECT 1 FROM staging c WHERE c.team_id=u.team_id AND instr(c.begin_json,u.id)>0) THEN 'staging-referenced'
-        WHEN u.created_at+86400000>? THEN 'unreferenced-under-one-day'
-        WHEN u.committed=1 THEN 'eligible-committed' ELSE 'expired-staging' END category
-        FROM uploads u)
-      SELECT u.category,COUNT(*) uploads,SUM(c.chunks) chunks,SUM(c.bytes) blob_bytes,
-        MIN(u.created_at) oldest_at,MAX(u.created_at) newest_at
-      FROM chunk_sizes c LEFT JOIN classified u ON u.team_id=c.team_id AND u.id=c.upload_id GROUP BY u.category`, now);
-    for (const row of rows) console.log(`storage uploads: ${JSON.stringify(row)}`);
-  } catch (error) { console.log(`storage upload classification: ${error.message}`); }
-}
-
 export class TeamsServer extends DurableObject {
   constructor(ctx, env) {
     super(ctx, env);
     this.ready = ctx.blockConcurrencyWhile(async () => {
       // Read before migrations/maintenance: allocation failures must not hide
       // the table responsible, including hosts with a lower storage limit.
-      logStorageBreakdown(ctx.storage.sql);
+      console.log(`storage: database_bytes=${ctx.storage.sql.databaseSize}`);
       const db = durableSql(ctx.storage);
       migrate({ exec: db.exec, tx: db.tx, version: () => ctx.storage.kv.get('schema_version') ?? 0, setVersion: (n) => ctx.storage.kv.put('schema_version', n) });
       this.api = createTeamsApi({ publicOrigin: normalizePublicOrigin(env.PUBLIC_ORIGIN), identityIssuer: env.VOLTER_ISSUER || null, identityOrganizations: identityOrganizationsFromEnv(env), delegatedIssuer: env.DELEGATED_ISSUER || null });
+      this.api.storageUsage = () => {
+        const sql = ctx.storage.sql;
+        let free = 0, page = 0;
+        try { free = sql.exec('PRAGMA freelist_count').one().freelist_count; page = sql.exec('PRAGMA page_size').one().page_size; } catch { /* conservative allocation measure if host omits the pragmas */ }
+        return { allocated: sql.databaseSize, used: Math.max(0, sql.databaseSize - free * page), limit: 10 * 1024 ** 3 };
+      };
       this.api.attach({ store: new DirectStore(createStoreHandlers(db)), harnessRoutes });
       const existing = await this.api.store.one("SELECT value FROM server_meta WHERE key='server_id'");
       if (existing) this.api.attach({ serverId: existing.value });
@@ -131,7 +101,7 @@ export class TeamsServer extends DurableObject {
       this.handler = createHttpHandler(this.api);
       // Retention must also run after a restart: a full object may have failed to re-arm its alarm.
       if (this.api.serverId) await maintain(this.api).catch(error => console.error(`maintain startup: ${error.message}`));
-      if (!(await ctx.storage.getAlarm())) await ctx.storage.setAlarm(Date.now() + MAINTAIN_MS);
+      if (!(await ctx.storage.getAlarm())) await ctx.storage.setAlarm(Date.now() + MAINTAIN_MS).catch(error => console.error(`maintain alarm scheduling failed: ${error.message}`));
     });
   }
 
@@ -160,6 +130,8 @@ export class TeamsServer extends DurableObject {
   async alarm() {
     await this.ready;
     const done = this.api.serverId ? await maintain(this.api).catch(error => { console.error(`maintain failed: ${error.message}`); return null; }) : null;
+    const usage = this.api.storageUsage();
+    if (usage.used >= usage.limit * 0.8) console.warn(`Teams storage pressure: ${JSON.stringify(usage)}; native upload admission stops at 90%`);
     // The database's size on every full pass, where `wrangler tail` reads it.
     if (!done?.backlog) {
       // What grows: free pages (space deletes left for reuse), and each busy table's newest rowid (its insert rate).