@volter/teams 0.2.8 → 0.2.10

package/http-client.d.ts

@@ -83,8 +83,10 @@ export interface TeamClient {
     catalogue(options?: RequestOptions): Promise<Result<{kinds: string[]}>>;
     createSource(body: {name: string}, options?: RequestOptions): Promise<Result<{id: string; name: string}>>;
     publish(source: string, body: unknown, options?: RequestOptions): Promise<Result<OntologyEvent>>;
+    purge(ids: string[], options?: RequestOptions): Promise<Result<{purged: string[]; unknown: string[]}>>;
     follow(query?: Query, options?: StreamOptions): AsyncIterable<PublishedEventFrame>;
   };
+  purgeSessionText(sessions: string[], values: string[], options?: RequestOptions): Promise<Result<{sessions: Record<string, {messages: number; natives: number; artifacts_removed: number; artifacts_left: number; chunks_removed: number; shared_chunks_kept: number; shared_chunks_holding_value: number}>}>>;
   sessions: {
     list(query?: Query, options?: RequestOptions): Promise<Page>;
     search(query: Query, options?: RequestOptions): Promise<unknown>;

package/http-client.mjs

@@ -84,7 +84,9 @@ class TeamClient {
       follow: (q = {},o = {}) => events(root,base+'/events'+query(q),o),
       createSource: (body,o) => post('/event-sources',body,o),
       publish: (source,body,o) => post(resource('/event-sources',source)+'/events',body,o),
+      purge: (ids,o) => post('/events/purge',{events:ids},o),
     };
+    this.purgeSessionText = (sessions, values, o) => post('/sessions/purge', { sessions, values }, o);
     this.apps = {
       catalogue: o => get('/apps/catalogue', o),
       list: o => get('/apps', o), get: (id, o) => get(resource('/apps', id), o),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/teams",
-  "version": "0.2.8",
+  "version": "0.2.10",
   "type": "module",
   "description": "Volter Teams, the neutral team plane: machine identity and daemon core, the local door and server link, admission from machine grants, product doors, and the Teams server's neutral routes (company RFC 0008).",
   "license": "MIT",

package/server/events.mjs

@@ -1,3 +1,17 @@
 import { EventEmitter } from 'node:events';
 export class TeamEvents extends EventEmitter { publish(teamId,event){this.emit(`team:${teamId}`,event); if(event.session_id)this.emit(`session:${teamId}:${event.session_id}`,event);} subscribe(key,fn){this.on(key,fn);return()=>this.off(key,fn);} close(){this.emit('shutdown');this.removeAllListeners();} }
 export function writeSse(res,event){const cursor=event.cursor??'';res.write(`id: ${cursor}\nevent: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);}
+// A stream's backlog cap is met by waiting for the response to flush, never by measuring the server's own unflushed bytes:
+// a Durable Object's SQLite answers as microtasks, so a snapshot written in one turn never lets the response drain, and a
+// 1 MiB backlog checked there cut every reader of a large team as slow (217 Task heads, 1,022 KB, with a reader that only
+// counted). Only a reader that takes nothing for the patience window is slow. Resolves 'ready', 'closed' or 'slow'.
+export const SSE_BACKLOG_BYTES=1048576, SSE_PATIENCE_MS=30000;
+export function sseRoom(res,bytes,{cap=SSE_BACKLOG_BYTES,patienceMs=SSE_PATIENCE_MS}={}){
+  // An event larger than the cap still goes out whole once nothing is waiting: a snapshot of any size completes.
+  if(res.writableLength===0||res.writableLength+bytes<=cap)return Promise.resolve('ready');
+  return new Promise(resolve=>{
+    const done=outcome=>{clearTimeout(timer);res.off('drain',drained);res.off('close',ended);resolve(outcome);};
+    const drained=()=>done('ready'),ended=()=>done('closed'),timer=setTimeout(()=>done('slow'),patienceMs);
+    res.on('drain',drained);res.on('close',ended);
+  });
+}

package/server/published-events.mjs

@@ -3,11 +3,14 @@ import { appAuthentication } from './apps.mjs';
 import { isAdmin } from './auth.mjs';
 import { exact } from './request-json.mjs';
 import { canonicalJson } from './wire.mjs';
-import { writeSse } from './events.mjs';
+import { sseRoom, writeSse } from './events.mjs';
 import { APP_EVENTS, PUBLISHABLE_EVENTS, eventReadAction } from './event-catalogue.mjs';
 import { EVENT_DATA_LIMIT } from '../event-limits.mjs';
 
 const stmt = (sql, ...params) => ({sql, params});
+// A Durable Object binds at most 100 parameters to a statement, so id lists go in batches.
+const batches = (list, size = 90) => Array.from({length: Math.ceil(list.length / size)}, (_, at) => list.slice(at * size, (at + 1) * size));
+const holes = list => list.map(() => '?').join(',');
 const fail = (status,code,message) => {throw Object.assign(new Error(message),{status,code});};
 const text = (value,label) => {if(typeof value!=='string'||!value||value.length>256||/[\x00-\x1f]/.test(value))fail(400,'invalid_request',`${label} must be a nonempty string of at most 256 characters`);return value;};
 const eventOf = row => ({id:row.id,org:row.team_id,at:row.created_at,actor:JSON.parse(row.actor_json),kind:row.kind,resource:{kind:row.resource_kind,id:row.resource_id},data:JSON.parse(row.data_json)});
@@ -84,6 +87,68 @@ export async function publishedEventRoute(ctx) {
     await guardedMutation(api,res,auth,guards,[stmt('INSERT INTO org_events(id,team_id,source_id,source_revision,kind,resource_kind,resource_id,actor_json,data_json,created_at) VALUES (?,?,?,?,?,?,?,?,?,?)',body.id,auth.team_id,publication[1],body.revision,body.kind,body.resource.kind,body.resource.id,JSON.stringify(actor),JSON.stringify(body.data),now)],{status:201,body:response,targetId:body.resource.id});
     send(res,201,response,requestId);return true;
   }
+  // An admin purges published events by id: each one's data becomes a tombstone (and so does the resource snapshot it
+  // left), its row and sequence kept so followers' cursors hold. The request names ids only, never the text purged, and
+  // the log line names the ids. A secret pasted into a card reaches here with the card; this is how it leaves.
+  if(sub==='/events/purge'&&req.method==='POST'){
+    if(!isAdmin(auth))fail(403,'forbidden','an admin purges events');
+    const body=await parseJson(req);exact(body,['events']);
+    if(!Array.isArray(body.events)||!body.events.length||body.events.length>500||body.events.some(id=>typeof id!=='string'||!/^event_[0-9a-f]{64}$/.test(id)))fail(400,'invalid_request','events must name 1 to 500 event ids');
+    const ids=[...new Set(body.events)],found=[];
+    for(const part of batches(ids))found.push(...(await api.store.query(`SELECT id FROM org_events WHERE team_id=? AND id IN (${holes(part)})`,[auth.team_id,...part])).map(row=>row.id));
+    const tombstone='{"purged":true}',response=result(api,auth,{purged:found,unknown:ids.filter(id=>!found.includes(id))});
+    await guardedMutation(api,res,auth,authorityGuards(auth,api.clock.nowMs()),batches(found).flatMap(part=>[
+      stmt(`UPDATE org_events SET data_json=? WHERE team_id=? AND id IN (${holes(part)})`,tombstone,auth.team_id,...part),
+      stmt(`UPDATE event_resources SET event_json=json_set(event_json,'$.data',json(?)) WHERE team_id=? AND json_extract(event_json,'$.id') IN (${holes(part)})`,tombstone,auth.team_id,...part),
+      // Each event was also copied whole into the webhook body of every subscribed installation (migration 015): pending
+      // rows would still deliver it, and finished ones keep it for a week.
+      stmt(`UPDATE app_webhook_deliveries SET body=json_set(body,'$.data',json(?)) WHERE installation_id IN (SELECT id FROM app_installations WHERE team_id=?) AND event_id IN (${holes(part)})`,tombstone,auth.team_id,...part),
+    ]),{status:200,body:response,targetId:null});
+    console.log(`events purged by ${auth.principal_id}: ${found.join(',') || 'none'}`);
+    send(res,200,response,requestId);return true;
+  }
+  // An admin redacts text from named sessions' recorded messages and the captures they came from. The text is in the
+  // request body (never a URL, never logged); the answer and the log line are counts per session. Ids alone cannot find
+  // which messages hold a pasted secret, and a search would carry it in a URL.
+  if(sub==='/sessions/purge'&&req.method==='POST'){
+    if(!isAdmin(auth))fail(403,'forbidden','an admin purges session text');
+    const body=await parseJson(req);exact(body,['sessions','values']);
+    if(!Array.isArray(body.sessions)||!body.sessions.length||body.sessions.length>20||body.sessions.some(id=>typeof id!=='string'||!/^ses_[0-9a-f]{32}$/.test(id)))fail(400,'invalid_request','sessions must name 1 to 20 session ids');
+    if(!Array.isArray(body.values)||!body.values.length||body.values.length>20||body.values.some(v=>typeof v!=='string'||v.length<16||v.length>512||/["\\\u0000-\u001f]/.test(v)))fail(400,'invalid_request','values must be 1 to 20 strings of 16 to 512 characters with no quotes, backslashes or control characters');
+    const marker='[redacted]',counts={};
+    for(const session of new Set(body.sessions)){
+      const natives=(await api.store.query("SELECT DISTINCT json_extract(descriptor_json,'$.native_session_id') AS native FROM bindings WHERE team_id=? AND session_id=?",[auth.team_id,session])).map(row=>row.native).filter(Boolean);
+      const statements=[];
+      for(const value of body.values){
+        statements.push(stmt('UPDATE messages SET value_json=replace(value_json,?,?),search_text=replace(search_text,?,?) WHERE team_id=? AND session_id=? AND (instr(value_json,?)>0 OR instr(search_text,?)>0)',value,marker,value,marker,auth.team_id,session,value,value));
+        if(natives.length)statements.push(stmt(`UPDATE capture_parts SET payload_json=replace(payload_json,?,?) WHERE team_id=? AND instr(payload_json,?)>0 AND capture_id IN (SELECT id FROM captures WHERE team_id=? AND native_id IN (${natives.map(()=>'?').join(',')}))`,value,marker,auth.team_id,value,auth.team_id,...natives));
+      }
+      // The native transcript each capture uploaded (and any export made from it) holds the text byte for byte, behind
+      // a manifest's sha256: those files go whole, so a later native export or continuation answers fidelity_unavailable
+      // rather than the secret. An upload or content-addressed chunk another session still names is kept, and the answer
+      // counts how many kept chunks still hold a value.
+      const mine="SELECT upload_id FROM session_artifacts WHERE team_id=? AND session_id=?",others="SELECT 1 FROM session_artifacts o WHERE o.team_id=? AND o.session_id<>? AND o.upload_id=",both=[auth.team_id,session,auth.team_id,session];
+      const uploads=(await api.store.query(`SELECT DISTINCT upload_id FROM session_artifacts a WHERE team_id=? AND session_id=? AND NOT EXISTS (${others}a.upload_id)`,both)).map(row=>row.upload_id);
+      const blobs=(await api.store.query(`SELECT DISTINCT sha256 FROM artifact_chunk_refs r WHERE team_id=? AND upload_id IN (${mine}) AND NOT EXISTS (${others}r.upload_id)`,[auth.team_id,...both])).map(row=>row.sha256);
+      statements.push(
+        stmt(`DELETE FROM artifact_chunks WHERE team_id=? AND upload_id IN (${mine}) AND NOT EXISTS (${others}artifact_chunks.upload_id)`,auth.team_id,...both),
+        stmt(`DELETE FROM artifact_chunk_refs WHERE team_id=? AND upload_id IN (${mine}) AND NOT EXISTS (${others}artifact_chunk_refs.upload_id)`,auth.team_id,...both),
+        stmt('DELETE FROM session_artifacts WHERE team_id=? AND session_id=?',auth.team_id,session),
+        ...batches(uploads).map(part=>stmt(`DELETE FROM uploads WHERE team_id=? AND id IN (${holes(part)}) AND NOT EXISTS (SELECT 1 FROM session_artifacts a WHERE a.team_id=uploads.team_id AND a.upload_id=uploads.id)`,auth.team_id,...part)),
+        ...batches(blobs).map(part=>stmt(`DELETE FROM artifact_blobs WHERE team_id=? AND sha256 IN (${holes(part)}) AND NOT EXISTS (SELECT 1 FROM artifact_chunk_refs r WHERE r.team_id=artifact_blobs.team_id AND r.sha256=artifact_blobs.sha256)`,auth.team_id,...part)));
+      const before=await api.store.one(`SELECT (SELECT count(*) FROM messages WHERE team_id=? AND session_id=? AND (${body.values.map(()=>'instr(value_json,?)>0').join(' OR ')})) messages`,[auth.team_id,session,...body.values]);
+      await api.store.guardedTransaction(authorityGuards(auth,api.clock.nowMs()),statements);
+      let kept=0,holding=0;
+      for(const part of batches(blobs,70)){
+        const row=await api.store.one(`SELECT count(*) kept,coalesce(sum(${body.values.map(()=>'instr(CAST(bytes AS TEXT),?)>0').join(' OR ')}),0) holding FROM artifact_blobs WHERE team_id=? AND sha256 IN (${holes(part)})`,[...body.values,auth.team_id,...part]);
+        kept+=row?.kept??0;holding+=row?.holding??0;
+      }
+      const left=await api.store.one('SELECT count(*) n FROM session_artifacts WHERE team_id=? AND session_id=?',[auth.team_id,session]);
+      counts[session]={messages:before?.messages??0,natives:natives.length,artifacts_removed:uploads.length,artifacts_left:left?.n??0,chunks_removed:blobs.length-kept,shared_chunks_kept:kept,shared_chunks_holding_value:holding};
+    }
+    console.log(`session text purged by ${auth.principal_id}: ${JSON.stringify(counts)}`);
+    send(res,200,result(api,auth,{sessions:counts}),requestId);return true;
+  }
   if(sub==='/events/catalogue'&&req.method==='GET'){send(res,200,result(api,auth,{kinds:APP_EVENTS}),requestId);return true;}
   if(sub==='/events'&&req.method==='GET'){await eventStream(ctx);return true;}
   return false;
@@ -99,7 +164,7 @@ async function eventStream({api,req,res,auth,url,requestId}){
   let closed=false,running=null,dirty=false;
   const close=()=>{if(closed)return;closed=true;clearInterval(timer);unsubscribe();req.off('close',close);api.events.off('shutdown',close);api.openStreams.set(auth.credential_id,Math.max(0,(api.openStreams.get(auth.credential_id)??1)-1));res.end();};
   const terminal=(type,reason)=>{if(!closed)writeSse(res,{type,reason,terminal:true});close();};
-  const write=event=>{if(closed)return false;if(res.writableLength+Buffer.byteLength(JSON.stringify(event))>1048576){terminal('resync_required','slow_reader');return false;}writeSse(res,event);return true;};
+  const write=async event=>{if(closed)return false;const room=await sseRoom(res,Buffer.byteLength(JSON.stringify(event)));if(closed||room==='closed')return false;if(room==='slow'){terminal('resync_required','slow_reader');return false;}writeSse(res,event);return true;};
   const readable=async event=>{
     const current=await liveAuth(api,auth);
     if(current.epoch!==auth.epoch)fail(409,'resync_required','authorization changed during event delivery');
@@ -120,21 +185,21 @@ async function eventStream({api,req,res,auth,url,requestId}){
         {kind:'one',sql:'SELECT COALESCE((SELECT max(sequence) FROM org_events WHERE team_id=?),(SELECT pruned_through FROM event_retention WHERE team_id=?),0) last',params:[auth.team_id,auth.team_id]},
         {kind:'query',sql:'SELECT r.sequence FROM event_resources r JOIN org_events e ON e.sequence=r.sequence WHERE r.team_id=? AND (? IS NULL OR r.source_id=?) AND (? IS NULL OR r.resource_kind IN (SELECT value FROM json_each(?)) OR e.kind IN (SELECT value FROM json_each(?))) ORDER BY r.source_id,r.resource_kind,r.resource_id',params:[auth.team_id,query.source,query.source,filterResources,filterResources,filterKinds]},
       ]);
-      cursor=head.last;write({type:'snapshot_begin'});
+      cursor=head.last;if(!await write({type:'snapshot_begin'}))return;
       for(let at=0;at<resources.length;at+=32){
         const sequences=resources.slice(at,at+32).map(row=>row.sequence);
         const rows=await api.store.query('SELECT * FROM org_events WHERE team_id=? AND sequence IN (SELECT value FROM json_each(?)) ORDER BY sequence',[auth.team_id,JSON.stringify(sequences)]);
         if(rows.length!==sequences.length){terminal('resync_required','snapshot_retention');return;}
-        for(const row of rows){const event=eventOf(row);if(await readable(event))if(!write({type:'snapshot_item',sequence:row.sequence,data:event}))return;}
+        for(const row of rows){const event=eventOf(row);if(await readable(event))if(!await write({type:'snapshot_item',sequence:row.sequence,data:event}))return;}
       }
-      write({type:'snapshot_end',cursor:codec.encode(cursor)});
+      if(!await write({type:'snapshot_end',cursor:codec.encode(cursor)}))return;
     }
     for(;;){
       const rows=await api.store.query('SELECT * FROM org_events WHERE team_id=? AND sequence>? AND (? IS NULL OR source_id=?) AND (? IS NULL OR kind IN (SELECT value FROM json_each(?))) ORDER BY sequence LIMIT 128',[auth.team_id,cursor,query.source,query.source,filterKinds,filterKinds]);
-      if(!rows.length){cursor=Math.max(cursor,bounds.last);write({type:'checkpoint',cursor:codec.encode(cursor)});break;}
-      for(const row of rows){if(closed)return;cursor=row.sequence;const event=eventOf(row);if(await readable(event))write({type:'event',sequence:row.sequence,cursor:codec.encode(cursor),data:event});}
+      if(!rows.length){cursor=Math.max(cursor,bounds.last);await write({type:'checkpoint',cursor:codec.encode(cursor)});break;}
+      for(const row of rows){if(closed)return;cursor=row.sequence;const event=eventOf(row);if(await readable(event)&&!await write({type:'event',sequence:row.sequence,cursor:codec.encode(cursor),data:event}))return;}
       // Advance even when the reader was not authorized for a resource; never leak its data.
-      write({type:'checkpoint',cursor:codec.encode(cursor)});if(rows.length<128)break;
+      if(!await write({type:'checkpoint',cursor:codec.encode(cursor)}))return;if(rows.length<128)break;
     }
   };
   const enqueue=()=>{dirty=true;if(running)return running;running=(async()=>{while(dirty&&!closed){dirty=false;await drain();}})().catch(error=>terminal([401,403].includes(error.status)?'access_revoked':'resync_required',error.code??'unavailable')).finally(()=>{running=null;});return running;};
@@ -142,5 +207,5 @@ async function eventStream({api,req,res,auth,url,requestId}){
   const timer=setInterval(()=>void enqueue().then(()=>{if(!closed)res.write(': heartbeat\n\n');}),15000);timer.unref?.();
   req.once('close',close);api.events.once('shutdown',close);
   res.writeHead(200,{'Content-Type':'text/event-stream','Cache-Control':'no-store','X-Request-ID':requestId});await enqueue();
-  if(!closed)write({type:'checkpoint',cursor:codec.encode(cursor)});
+  if(!closed)await write({type:'checkpoint',cursor:codec.encode(cursor)});
 }