@volter/supercode-teams 0.3.142 → 0.3.143

package/README.md

@@ -188,3 +188,7 @@ Only the session custodian can grant this binding, in person or through an app t
 no session-backed channel is created. The agent DM uses the exact resource `${ROOM}:dm:${PRINCIPAL}`.
 `unbind` accepts the same flags. `--conversation` remains available for existing grants; it is mutually exclusive
 with `--agent`. Opening a DM reads the existing session; sending a turn is a separate action.
+
+## Read server work diagnostics
+
+An administrator can run `supercode teams storage work --json` to read recent handler records and retained slow work. Records name normalized HTTP routes, catalog refreshes, transitions and maintenance steps, with SQL call and cursor scan/write counts. The Worker retains up to 32 slow records across a restart. Elapsed milliseconds are not CPU time; hosted per-query timing is unavailable. [Recorded private Worker proof](../../docs/evidence/teams-handler-work-2026-10-07.md).

package/bin/workspace.mjs

@@ -95,6 +95,11 @@ export async function workspaceMain(argv = process.argv.slice(2), env = process.
   if (verb === 'sessions') return sessionsCommand(sub, command);
   if (verb === 'calendar') return calendarCommand(sub, { ...command, teamPath });
   if (verb === 'settings') return settingsCommand(sub, command);
+  if (verb === 'storage') {
+    only(flags, ['json', 'help', 'context']);requireArgs(rest, 0, 0, 'storage work');
+    if(sub!=='work')usage('storage work');
+    return output(await client.request('GET',teamPath(context,'/storage/work')));
+  }
   if (verb === 'members') return membersCommand(sub, command);
   if (verb === 'groups') return groupsCommand(sub, command);
   if (verb === 'sources') return sourcesCommand(sub, command);
@@ -1434,6 +1439,7 @@ function printHelp() {
   identity link
   identities observations|sightings|resolve|observe|declare-person|link|infer|revoke|revoke-inference --body-file FILE
   server start [--data-dir DIR] [--listen HOST:PORT] [--public-origin ORIGIN] [--init-team NAME] | server token
+  storage work [--context NAME] [--json]     administrators: recent handler work and retained slow records
   publish pane|root TARGET --room ROOM --workplace ORIGIN --organization ORG [--scope view|control] [--name] [--key] [--machine] [--token-stdin] [--record-only]
   unpublish SURFACE --room ROOM --workplace ORIGIN --organization ORG [--machine] [--token-stdin] [--record-only|--workplace-only]
   connect|disconnect|status|fold   (fold: the connector's state back into connector.json for an older daemon, with it stopped)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.3.142",
+  "version": "0.3.143",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",
@@ -70,7 +70,7 @@
     "@volter/supercode-health": "^0.1.0",
     "@volter/supercode-terminal": "^0.2.54",
     "@volter/supercode-trace": "^0.1.0",
-    "@volter/teams": "0.2.37",
+    "@volter/teams": "0.2.38",
     "@xterm/addon-serialize": "^0.14.0",
     "@xterm/headless": "^6.0.0",
     "protobufjs": "8.8.0",

package/server/event-routes.mjs

@@ -221,7 +221,7 @@ export async function handleCatalogEvents({ api, req, res, auth, url, requestId,
   const stream = await openStream(api, req, res, auth, requestId);
   let lastCursor = null;
   const enqueue = streamQueue(stream, async () => {
-    const page = await catalogWindow(api, auth, query);
+    const page = await (api.work ? api.work('catalogWindow', () => catalogWindow(api, auth, query)) : catalogWindow(api, auth, query));
     // Authorization and expiry can change a window without creating a session revision.
     // A binding's observed_at moves with every heartbeat; only a change of what the window says is a reset.
     const digest = createHash('sha256').update(JSON.stringify(page.data, (key, value) => key === 'bindings' && Array.isArray(value) ? value.map(({ observed_at: _seen, ...binding }) => binding) : value)).digest('hex');

package/server/harness-routes.mjs

@@ -88,6 +88,11 @@ async function beforeIdempotency({ api, req, res, auth, url, path, sub, requestI
 
 async function team({ api, req, res, auth, url, path, sub, requestId }) {
   let m;
+  if(req.method==='GET'&&sub==='/storage/work'){
+    if(!isAdmin(auth))fail(403,'forbidden','Teams work diagnostics require an administrator');
+    if(!api.workReport)fail(501,'unsupported','this host does not record Teams work diagnostics');
+    send(res,200,result(api,auth,api.workReport()),requestId);return true;
+  }
   if(req.method==='GET'&&sub==='/storage/uploads'){await storageReport({api,res,auth,url,requestId});return true;}
   if(req.method==='GET'&&sub==='/sessions'){await listSessions(api,res,auth,url,requestId,false);return true;}
   if(req.method==='GET'&&sub==='/sessions/search'){await listSessions(api,res,auth,url,requestId,true);return true;}

package/server/work-meter.mjs

@@ -0,0 +1,84 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { createHash } from 'node:crypto';
+
+// Only contract path words survive. Never record a caller's query, identity, body or SQL parameters.
+const WORDS = new Set('api v1 me health teams sessions messages search events catalogue machines heartbeat configuration roots sources captures parts commit uploads chunks members groups credentials permissions bindings agents publications attachments operations continuations sync-rules view-bindings exports audit storage work'.split(' '));
+export function workRoute(input) {
+  try { return new URL(input, 'http://local').pathname.split('/').map(part => !part || WORDS.has(part) ? part : ':id').join('/'); }
+  catch { return '/:invalid'; }
+}
+const rounded = value => Math.round(value * 1000) / 1000;
+
+/** Attribute synchronous SQLite work to its handler, rather than the invocation a timer happened to inherit. */
+export class WorkMeter {
+  constructor({ emit = record => console.log(JSON.stringify(record)), now = () => performance.now(), retainSlow = null, advanceClock = null, sqlTiming = true } = {}) {
+    this.now = now; this.advanceClock = advanceClock; this.sqlTiming = sqlTiming;
+    this.context = new AsyncLocalStorage(); this.recent = []; this.slow = [];
+    this.emit = record => {
+      emit(record);
+      if (!['teams_work', 'teams_sql_work'].includes(record.event)) return;
+      this.recent.push(record); if (this.recent.length > 64) this.recent.shift();
+      if ((record.elapsed_ms ?? 0) < 1000 && (record.rows_read ?? 0) + (record.rows_written ?? 0) < 5000) return;
+      this.slow.push(record); if (this.slow.length > 32) this.slow.shift();
+      try { retainSlow?.(this.slow); }
+      catch { emit({ event: 'teams_work_retention_failed', work_id: record.work_id ?? null }); }
+    };
+  }
+  restoreSlow(records) { if (Array.isArray(records)) this.slow = records.slice(-32); }
+  report() { return { clock: this.advanceClock ? 'elapsed milliseconds through an end-of-work timer boundary; SQL timing unavailable on the hosted Worker' : 'elapsed milliseconds; sql_ms measures synchronous SQL, not CPU', recent: this.recent.slice(), slow: this.slow.slice() }; }
+  async run(work, fn, detail = {}) {
+    const parent = this.context.getStore();
+    const record = { work, ...detail, work_id: crypto.randomUUID(), parent_work_id: parent?.work_id ?? null, started_at: new Date().toISOString(), started: this.now(), sql_ms: 0, sql_calls: 0, rows_read: 0, rows_written: 0, rows_known: true, returned_rows: 0, slowest_sql: null, most_rows_sql: null, finished: false };
+    this.emit({ event: 'teams_work_start', work, ...detail, work_id: record.work_id, parent_work_id: record.parent_work_id, started_at: record.started_at });
+    return this.context.run(record, async () => {
+      let outcome = 'ok';
+      try { return await fn(); }
+      catch (error) { outcome = 'error'; throw error; }
+      finally {
+        record.finished = true;
+        // Hosted Workers freeze clocks during CPU work. Yield once after the work, not once per SQL call.
+        // This reports elapsed time, including the timer's wait; it does not manufacture a CPU reading.
+        if (this.advanceClock) await this.advanceClock();
+        const { started, finished: _finished, rows_known, ...data } = record;
+        this.emit({ event: 'teams_work', ...data, slowest_sql: this.sqlTiming ? record.slowest_sql : null, rows_read: rows_known ? record.rows_read : null, rows_written: rows_known ? record.rows_written : null, elapsed_ms: rounded(this.now() - started), sql_ms: this.sqlTiming ? rounded(record.sql_ms) : null, outcome });
+        // Nested steps are separately named, and their SQL remains counted in the enclosing handler.
+        if (parent && !parent.finished) {
+          for (const field of ['sql_ms', 'sql_calls', 'rows_read', 'rows_written', 'returned_rows']) parent[field] += record[field];
+          parent.rows_known &&= record.rows_known;
+          if (record.slowest_sql && (!parent.slowest_sql || record.slowest_sql.elapsed_ms > parent.slowest_sql.elapsed_ms)) parent.slowest_sql = record.slowest_sql;
+          if (record.most_rows_sql && (!parent.most_rows_sql || record.most_rows_sql.rows_read + record.most_rows_sql.rows_written > parent.most_rows_sql.rows_read + parent.most_rows_sql.rows_written)) parent.most_rows_sql = record.most_rows_sql;
+        }
+      }
+    });
+  }
+  detail(values) {
+    const record = this.context.getStore();
+    if (record && !record.finished) Object.assign(record, values);
+  }
+  sql(query, fn) {
+    const record = this.context.getStore();
+    if (!record || record.finished) {
+      const start = this.now(); let result;
+      try { result = fn(); return result; }
+      finally {
+        const elapsed = this.now() - start;
+        if (elapsed >= 100 || (result?.rowsRead ?? 0) >= 5000) this.emit({ event: 'teams_sql_work', work: 'unscoped', origin_work: record?.work ?? null, fingerprint: createHash('sha256').update(query).digest('hex').slice(0, 16), elapsed_ms: this.sqlTiming ? rounded(elapsed) : null, rows_read: result?.rowsRead ?? null, rows_written: result?.rowsWritten ?? null, returned_rows: result?.returnedRows ?? null });
+      }
+    }
+    const start = this.now(); let result;
+    try { result = fn(); return result; }
+    finally {
+      const elapsed = this.now() - start;
+      record.sql_calls++; record.sql_ms += elapsed;
+      record.rows_known &&= Number.isFinite(result?.rowsRead) && Number.isFinite(result?.rowsWritten);
+      record.rows_read += result?.rowsRead ?? 0; record.rows_written += result?.rowsWritten ?? 0;
+      record.returned_rows += result?.returnedRows ?? 0;
+      if (!record.slowest_sql || elapsed > record.slowest_sql.elapsed_ms) {
+        record.slowest_sql = { fingerprint: createHash('sha256').update(query).digest('hex').slice(0, 16), operation: /^\s*(\w+)/.exec(query)?.[1]?.toUpperCase() ?? 'UNKNOWN', elapsed_ms: rounded(elapsed), rows_read: result?.rowsRead ?? null, rows_written: result?.rowsWritten ?? null, returned_rows: result?.returnedRows ?? null };
+      }
+      if (Number.isFinite(result?.rowsRead) && Number.isFinite(result?.rowsWritten) && (!record.most_rows_sql || result.rowsRead + result.rowsWritten > record.most_rows_sql.rows_read + record.most_rows_sql.rows_written)) {
+        record.most_rows_sql = { fingerprint: createHash('sha256').update(query).digest('hex').slice(0, 16), operation: /^\s*(\w+)/.exec(query)?.[1]?.toUpperCase() ?? 'UNKNOWN', rows_read: result.rowsRead, rows_written: result.rowsWritten, returned_rows: result?.returnedRows ?? null };
+      }
+    }
+  }
+}

package/worker/index.mjs

@@ -18,6 +18,7 @@ import { setOtlpSchemaSources } from '../server/otlp/transport.mjs';
 import { setViewerSources } from '@volter/teams/server/viewer';
 import { serveFetch } from './fetch-adapter.mjs';
 import { webSocketDuplex } from './websocket-duplex.mjs';
+import { WorkMeter, workRoute } from '../server/work-meter.mjs';
 // The hosted viewer's files, bundled as text (wrangler.toml rules) and served at /view.
 import viewerPage from '../../volter-teams/viewer/index.html';
 import viewerScript from '../../volter-teams/viewer/viewer.mjs';
@@ -56,31 +57,47 @@ const MAINTAIN_MS = 60_000;
 const BOOTSTRAP_TOKEN = /^sct_(cred_[0-9a-f]{32})\.[A-Za-z0-9_-]{16,}$/;
 
 /** SQLite primitives over the object's storage, in store-core's shape. */
-function durableSql(storage) {
+function durableSql(storage, meter) {
   const sql = storage.sql;
   const param = (value) => (value instanceof Uint8Array ? value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength) : value);
   const row = (value) => { for (const key of Object.keys(value)) if (value[key] instanceof ArrayBuffer) value[key] = Buffer.from(value[key]); return value; };
-  const all = (query, params = []) => sql.exec(query, ...params.map(param)).toArray().map(row);
+  const measured = (query, fn) => meter.sql(query, fn);
+  const all = (query, params = []) => measured(query, () => {
+    const cursor = sql.exec(query, ...params.map(param));
+    const values = cursor.toArray().map(row);
+    return { values, rowsRead: cursor.rowsRead, rowsWritten: cursor.rowsWritten, returnedRows: values.length };
+  }).values;
   const get = (query, params = []) => all(query, params)[0] ?? null;
-  const run = (query, params = []) => { sql.exec(query, ...params.map(param)); return { changes: sql.exec('SELECT changes() AS c').one().c }; };
+  const run = (query, params = []) => {
+    const counts = measured(query, () => {
+      const cursor = sql.exec(query, ...params.map(param));
+      return { rowsRead: cursor.rowsRead, rowsWritten: cursor.rowsWritten, returnedRows: 0 };
+    });
+    return { ...counts, changes: get('SELECT changes() AS c').c };
+  };
   const tx = (fn) => storage.transactionSync(fn);
-  return { all, get, run, tx, exec: (query) => sql.exec(query) };
+  return { all, get, run, tx, exec: (query) => { all(query); } };
 }
 
 export class TeamsServer extends DurableObject {
   constructor(ctx, env) {
     super(ctx, env);
-    this.ready = ctx.blockConcurrencyWhile(async () => {
+    this.work = new WorkMeter({ retainSlow: records => ctx.storage.kv.put('teams_work_slow_v1', JSON.stringify(records)), advanceClock: () => scheduler.wait(0), sqlTiming: false });
+    this.ready = ctx.blockConcurrencyWhile(() => this.work.run('activation', async () => {
       // Read before migrations/maintenance: allocation failures must not hide
       // the table responsible, including hosts with a lower storage limit.
       console.log(`storage: database_bytes=${ctx.storage.sql.databaseSize}`);
-      const db = durableSql(ctx.storage);
+      const db = durableSql(ctx.storage, this.work);
       migrate({ exec: db.exec, tx: db.tx, version: () => ctx.storage.kv.get('schema_version') ?? 0, setVersion: (n) => ctx.storage.kv.put('schema_version', n) });
+      const retained = ctx.storage.kv.get('teams_work_slow_v1');
+      if (retained) this.work.restoreSlow(JSON.parse(retained));
       this.api = createTeamsApi({ publicOrigin: normalizePublicOrigin(env.PUBLIC_ORIGIN), appRootSecret: env.SUPERCODE_TEAMS_BOOTSTRAP_TOKEN || null });
+      this.api.work = (name, fn, detail) => this.work.run(name, fn, detail);
+      this.api.workReport = () => this.work.report();
       this.api.storageUsage = () => {
         const sql = ctx.storage.sql;
         let freeBytes = null;
-        try { freeBytes = sql.exec('PRAGMA freelist_count').one().freelist_count * sql.exec('PRAGMA page_size').one().page_size; } catch { /* preserve an explicit unknown if the host omits these pragmas */ }
+        try { freeBytes = db.get('PRAGMA freelist_count').freelist_count * db.get('PRAGMA page_size').page_size; } catch { /* preserve an explicit unknown if the host omits these pragmas */ }
         return { allocated: sql.databaseSize, free_bytes: freeBytes, used: Math.max(0, sql.databaseSize - (freeBytes ?? 0)), used_is_upper_bound: freeBytes === null, limit: 10 * 1024 ** 3 };
       };
       this.api.attach({ store: new DirectStore(createStoreHandlers(db)), harnessRoutes });
@@ -93,7 +110,16 @@ export class TeamsServer extends DurableObject {
           this.api.attach({ serverId: await bootstrapServer(this.api, { initTeam: env.INIT_TEAM, bootstrap: { server_id: ids.next('server'), team_id: ids.next('team'), principal_id: ids.next('user'), credential_id: match[1], token: env.SUPERCODE_TEAMS_BOOTSTRAP_TOKEN } }) });
         }
       }
-      this.handler = createHttpHandler(this.api);
+      const handler = createHttpHandler(this.api);
+      this.handler = (req, res) => {
+        const pending = this.work.run('http', async () => {
+        const writeHead = res.writeHead;
+        res.writeHead = (...args) => { this.work.detail({ status: args[0], request_id: args[1]?.['X-Request-ID'] ?? null }); return writeHead.apply(res, args); };
+        return handler(req, res);
+        }, { method: req.method, route: workRoute(req.url) });
+        ctx.waitUntil(pending);
+        return pending;
+      };
       // Retention never runs here: an activation serves its first request as soon as the store is open, and a reset is
       // never followed by a pass that holds every request back (t_6c01c966). A capacity failure can exhaust an old
       // alarm's retries, and an exhausted alarm still reads its old time, already past, while it never fires again: so
@@ -101,13 +127,13 @@ export class TeamsServer extends DurableObject {
       const armed = await ctx.storage.getAlarm().catch(() => null);
       const now = Date.now(), soon = now + 5_000;
       if (armed == null || armed > soon || armed < now) await ctx.storage.setAlarm(soon).catch(error => console.error(`maintain alarm scheduling failed: ${error.message}`));
-    });
+    }));
   }
 
   async fetch(request) {
     await this.ready;
     if (!this.api.serverId) return Response.json({ version: 1, error: { code: 'unavailable', message: 'set the SUPERCODE_TEAMS_BOOTSTRAP_TOKEN secret (supercode teams server token) before first use' } }, { status: 503 });
-    if (request.headers.get('upgrade')?.toLowerCase() === 'websocket') return this.upgrade(request);
+    if (request.headers.get('upgrade')?.toLowerCase() === 'websocket') return this.work.run('websocket.upgrade', () => this.upgrade(request), { method: request.method, route: workRoute(request.url) });
     return serveFetch(this.handler, request, { remoteAddress: request.headers.get('cf-connecting-ip') });
   }
 
@@ -128,7 +154,7 @@ export class TeamsServer extends DurableObject {
 
   async alarm() {
     await this.ready;
-    const done = this.api.serverId ? await maintain(this.api).catch(error => { console.error(`maintain failed: ${error.message}`); return null; }) : null;
+    const done = this.api.serverId ? await this.work.run('alarm', () => maintain(this.api)).catch(error => { console.error(`maintain failed: ${error.message}`); return null; }) : null;
     const usage = this.api.storageUsage();
     if (usage.used >= usage.limit * 0.8) console.warn(`Teams storage pressure: ${JSON.stringify(usage)}; native upload admission stops at 90%`);
     // The database's size on every full pass, where `wrangler tail` reads it.
@@ -156,6 +182,7 @@ const objectFailure = (error) => ({
 export default {
   async fetch(request, env) {
     const repeatable = request.method === 'GET' || request.method === 'HEAD';
+    const route = { method: request.method, route: workRoute(request.url) };
     const call = () => env.TEAMS_SERVER.get(env.TEAMS_SERVER.idFromName('server')).fetch(repeatable ? request.clone() : request);
     // A request safe to repeat (a read, or a link's WebSocket upgrade: every machine's connection) is tried once more
     // when the runtime says the failure is retryable and the object is not overloaded: a reset object (its code
@@ -166,10 +193,10 @@ export default {
     try { return await call(); }
     catch (error) {
       first = objectFailure(error);
-      if (!(repeatable && first.retryable && !first.overloaded)) return unavailable(first, null);
+      if (!(repeatable && first.retryable && !first.overloaded)) return unavailable(first, null, route);
     }
     try { return await call(); }
-    catch (error) { return unavailable(objectFailure(error), first); }
+    catch (error) { return unavailable(objectFailure(error), first, route); }
   },
 };
 
@@ -180,9 +207,9 @@ const PUBLIC_NAMES = new Set(['Error', 'TypeError', 'RangeError', 'ReferenceErro
 /** Failures before the HTTP handler (object initialization, migrations, storage, a reset) as our API contract, not a
  *  platform HTML page. The answer goes to callers not yet authenticated, so it names only the error's kind and the
  *  runtime's flags; its message (storage or SQL text, internal paths) is in the log line its request id leads to. */
-function unavailable(failure, retried) {
+function unavailable(failure, retried, route) {
   const requestId = crypto.randomUUID();
-  console.error(JSON.stringify({ event: 'teams_unavailable', request_id: requestId, failure, ...(retried ? { first: retried } : {}) }));
+  console.error(JSON.stringify({ event: 'teams_unavailable', request_id: requestId, ...route, failure, ...(retried ? { first: retried } : {}) }));
   const cause = `${PUBLIC_NAMES.has(failure.name) ? failure.name : 'Error'}${failure.reset ? ', object reset' : ''}${failure.overloaded ? ', overloaded' : ''}${retried ? ', after one retry' : ''}`;
-  return Response.json({ version: 1, error: { code: 'unavailable', message: `Teams server is temporarily unavailable (${cause})`, request_id: requestId, retryable: true, cause: { name: PUBLIC_NAMES.has(failure.name) ? failure.name : 'Error', retryable: failure.retryable, overloaded: failure.overloaded, remote: failure.remote, reset: failure.reset } } }, { status: 503, headers: { 'Retry-After': failure.overloaded ? '15' : '5', 'X-Request-ID': requestId } });
+  return Response.json({ version: 1, error: { code: 'unavailable', message: `Teams server is temporarily unavailable (${cause})`, request_id: requestId, ...route, retryable: true, cause: { name: PUBLIC_NAMES.has(failure.name) ? failure.name : 'Error', retryable: failure.retryable, overloaded: failure.overloaded, remote: failure.remote, reset: failure.reset } } }, { status: 503, headers: { 'Retry-After': failure.overloaded ? '15' : '5', 'X-Request-ID': requestId } });
 }