@volter/teams 0.2.39 → 0.2.41

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/teams",
-  "version": "0.2.39",
+  "version": "0.2.41",
   "type": "module",
   "description": "Volter Teams, the neutral team plane: machine identity and daemon core, the local door and server link, admission from machine grants, product doors, and the Teams server's neutral routes (company RFC 0008).",
   "license": "MIT",

package/server/http.mjs

@@ -23,7 +23,21 @@ const parseJson=req=>parseRequestJson(req,fail);
 const result=(api,auth,data)=>({version:1,context:{server_id:api.serverId,team_id:auth.team_id,principal_id:auth.principal_id},data});
 function sideEffectStatements(api,res,status,body,now,targetId=null,journal=false){const statements=[];if(status<400&&res._idempotencyRecord){const record=res._idempotencyRecord,safe=redactSecretReplay(record.path,body);statements.push({sql:'INSERT INTO idempotency(team_id,principal_id,route,key,request_digest,response_status,response_json,expires_at) VALUES (?,?,?,?,?,?,?,?)',params:[record.teamId,record.principalId,record.route,record.key,record.requestDigest,status,journal?{$journalJsonValue:safe}:JSON.stringify(safe),now+86400000]});}if(res._auditRecord){const record=res._auditRecord;statements.push({sql:'INSERT INTO audit(team_id,id,actor_id,operation,target_id,outcome,revision,created_at,actor_json) VALUES (?,?,?,?,?,?,?,?,?)',params:[record.teamId,api.ids.next('audit'),record.actorId,record.operation,targetId,status<400?'allowed':'denied',journal?{$journalRevision:true}:null,now,JSON.stringify(record.act??[])]});}return statements;}
 async function persistSideEffects(api,res,status,body){const statements=sideEffectStatements(api,res,status,body,api.clock.nowMs());if(statements.length)await api.store.transaction(statements);}
-function send(res,status,body,requestId,headers={}){const deliver=()=>{res.writeHead(status,{'Content-Type':'application/json; charset=utf-8','Cache-Control':'no-store','X-Request-ID':requestId,...headers});res.end(body===null?'':JSON.stringify(body));};if(!res._sideEffectsCommitted&&res._api&&(res._idempotencyRecord||res._auditRecord))return persistSideEffects(res._api,res,status,body).then(deliver,error=>{if(/UNIQUE constraint failed: idempotency\./.test(error?.message??''))return deliver();if(res.headersSent)return;res.writeHead(500,{'Content-Type':'application/json; charset=utf-8','Cache-Control':'no-store','X-Request-ID':requestId});res.end(JSON.stringify({error:{code:'internal',message:'The request was not recorded.'}}));});deliver();}
+function send(res,status,body,requestId,headers={}){
+  const deliver=(actualStatus=status,actualBody=body,actualHeaders=headers)=>{
+    // The host observes the response actually issued, including a refusal
+    // whose audit write failed. It decides which diagnostic fields are safe.
+    res._workResponse?.(actualStatus,actualBody);
+    res.writeHead(actualStatus,{'Content-Type':'application/json; charset=utf-8','Cache-Control':'no-store','X-Request-ID':requestId,...actualHeaders});
+    res.end(actualBody===null?'':JSON.stringify(actualBody));
+  };
+  if(!res._sideEffectsCommitted&&res._api&&(res._idempotencyRecord||res._auditRecord))return persistSideEffects(res._api,res,status,body).then(()=>deliver(),error=>{
+    if(/UNIQUE constraint failed: idempotency\./.test(error?.message??''))return deliver();
+    if(res.headersSent)return;
+    deliver(500,{error:{code:'internal',message:'The request was not recorded.'}},{});
+  });
+  deliver();
+}
 async function guardedMutation(api,res,auth,guards,statements,{status,body,targetId=null,journal=false}){const now=api.clock.nowMs(),all=[...statements,...sideEffectStatements(api,res,status,body,now,targetId,journal)],result=journal?await api.store.journalTransaction(auth.team_id,[...guards,...appAuthGuards(auth,now)],all):await api.store.guardedTransaction([...guards,...appAuthGuards(auth,now)],all);res._sideEffectsCommitted=true;api.events.publish(auth.team_id,{type:'events_available'});return result;}
 const requireRevision=(req,current,label='resource')=>{const expected=Number(String(req.headers['if-match']??'').replaceAll('"',''));if(!Number.isSafeInteger(expected))fail(400,'invalid_request','If-Match is required');if(expected!==current)fail(409,'revision_mismatch',`${label} revision changed`);return expected;};
 const CORS={'Access-Control-Allow-Origin':'*','Access-Control-Expose-Headers':'ETag, X-Request-ID, Content-Range'};
@@ -398,12 +412,54 @@ async function revokeMachine(api,req,res,auth,machineId,requestId){
 }
 /** This machine's sessions someone is watching live (an open follow stream), as {harness, native_id}. */
 async function watchedFor(api,teamId,machineId){const ids=[...(api.watchers?.keys()??[])].filter(key=>key.startsWith(`${teamId}:`)).map(key=>key.slice(teamId.length+1)).slice(0,90);if(!ids.length)return [];return (await api.store.query(`SELECT o.harness,o.native_id FROM origins o JOIN sources s ON s.team_id=o.team_id AND s.id=o.source_id WHERE o.team_id=? AND s.machine_id=? AND o.session_id IN (${ids.map(()=>'?').join(',')})`,[teamId,machineId,...ids])).map(({harness,native_id})=>({harness,native_id}));}
-async function heartbeat(api,req,res,auth,machineId,requestId){if(auth.kind!=='machine'||auth.machine_id!==machineId)fail(403,'forbidden','exact machine credential required');const machine=await api.store.one('SELECT m.*,(SELECT linked FROM machine_presence p WHERE p.team_id=m.team_id AND p.machine_id=m.id) presence_linked FROM machines m WHERE m.team_id=? AND m.id=?',[auth.team_id,machineId]);if(!machine||machine.revoked_at!=null)fail(403,'forbidden','machine enrollment is not active');const b=await parseJson(req);exact(b,['incarnation_id','sequence','bindings','roots','host_key_id','activity']);if(b.activity!==undefined&&(!Array.isArray(b.activity)||b.activity.length>4096))fail(400,'invalid_request','invalid activity');if(b.host_key_id!==undefined&&(typeof b.host_key_id!=='string'||!/^[A-Za-z0-9_-]{16,128}$/.test(b.host_key_id)))fail(400,'invalid_request','invalid host_key_id');if(typeof b.incarnation_id!=='string'||!Number.isSafeInteger(b.sequence)||b.sequence<1||!Array.isArray(b.bindings)||!Array.isArray(b.roots))fail(400,'invalid_request','invalid heartbeat');if(machine.incarnation_id===b.incarnation_id&&b.sequence<=machine.heartbeat_sequence)return send(res,200,result(api,auth,{...machineJson(machine,api.clock.nowMs(),api),watched:await watchedFor(api,auth.team_id,machineId)}),requestId);const now=api.clock.nowMs(),newIncarnation=machine.incarnation_id!==null&&machine.incarnation_id!==b.incarnation_id,statements=[{sql:'UPDATE machines SET incarnation_id=?,heartbeat_sequence=?,last_seen_at=?,host_key_id=COALESCE(?,host_key_id),revision=revision+1 WHERE team_id=? AND id=?',params:[b.incarnation_id,b.sequence,now,b.host_key_id??null,auth.team_id,machineId]},{sql:'DELETE FROM machine_roots WHERE team_id=? AND machine_id=?',params:[auth.team_id,machineId]}];if(newIncarnation)statements.push({sql:"UPDATE attachments SET state='closed',closed_at=? WHERE team_id=? AND machine_id=? AND state IN ('pending','active')",params:[now,auth.team_id,machineId]},{sql:"UPDATE operations SET state='unknown' WHERE team_id=? AND id IN (SELECT operation_id FROM native_effects WHERE team_id=? AND machine_id=? AND state IN ('pending','running'))",params:[auth.team_id,auth.team_id,machineId]},{sql:"UPDATE native_effects SET state='unknown',result_json=? WHERE team_id=? AND machine_id=? AND state IN ('pending','running')",params:[JSON.stringify({state:'unknown',error:null}),auth.team_id,machineId]});statements.push({sql:'DELETE FROM bindings WHERE team_id=? AND machine_id=? AND id NOT IN (SELECT value FROM json_each(?)) AND id NOT IN (SELECT binding_id FROM attachments WHERE team_id=? AND machine_id=?) AND id NOT IN (SELECT binding_id FROM native_effects WHERE team_id=? AND machine_id=? AND binding_id IS NOT NULL)',params:[auth.team_id,machineId,JSON.stringify(b.bindings.map(binding=>binding.id)),auth.team_id,machineId,auth.team_id,machineId]});// What each synced session's harness is doing now: this heartbeat's report replaces the source's last one.
-  if(Array.isArray(b.activity)){const source=await api.store.one("SELECT id FROM sources WHERE team_id=? AND machine_id=? AND kind='connector'",[auth.team_id,machineId]);if(source){statements.push({sql:'DELETE FROM session_activity WHERE team_id=? AND source_id=?',params:[auth.team_id,source.id]});for(const item of b.activity){exact(item,['harness','native_id','turn','since'],'activity');if(typeof item.harness!=='string'||!item.harness||item.harness.length>64||typeof item.native_id!=='string'||!item.native_id||item.native_id.length>256||typeof item.turn!=='string'||!item.turn||item.turn.length>32||(item.since!=null&&!Number.isSafeInteger(item.since)))fail(400,'invalid_request','invalid activity');statements.push({sql:'INSERT OR REPLACE INTO session_activity(team_id,source_id,harness,native_id,turn,since,observed_at) VALUES (?,?,?,?,?,?,?)',params:[auth.team_id,source.id,item.harness,item.native_id,item.turn,item.since??null,now]});}}}
-  for(const root of b.roots){exact(root,['id','machine_id','name','harnesses'],'root');if(root.machine_id!==machineId)fail(400,'invalid_request','root machine mismatch');statements.push({sql:'INSERT INTO machine_roots(team_id,machine_id,id,value_json) VALUES (?,?,?,?)',params:[auth.team_id,machineId,root.id,JSON.stringify(root)]});}// A binding is stored against the session its native id names; one naming no session this source published is
-  // said back (unbound_bindings), never dropped in silence.
-  const unbound=[];for(const binding of b.bindings){const origin=await api.store.one('SELECT 1 ok FROM origins WHERE team_id=? AND source_id=? AND harness=? AND native_id=? AND deleted_at IS NULL',[auth.team_id,auth.source_id,binding.harness,binding.native_session_id]);if(!origin)unbound.push({id:binding.id,harness:binding.harness,native_session_id:binding.native_session_id,reason:'no_session'});}
-  for(const binding of b.bindings)statements.push({sql:'INSERT INTO bindings(team_id,id,machine_id,session_id,incarnation_id,descriptor_json,observed_at) SELECT ?,?,?,o.session_id,?,?,? FROM origins o WHERE o.team_id=? AND o.source_id=? AND o.harness=? AND o.native_id=? AND o.deleted_at IS NULL ON CONFLICT(team_id,id) DO UPDATE SET machine_id=excluded.machine_id,session_id=excluded.session_id,incarnation_id=excluded.incarnation_id,descriptor_json=excluded.descriptor_json,observed_at=excluded.observed_at',params:[auth.team_id,binding.id,machineId,b.incarnation_id,JSON.stringify(binding),now,auth.team_id,auth.source_id,binding.harness,binding.native_session_id]});const updated={...machine,incarnation_id:b.incarnation_id,heartbeat_sequence:b.sequence,last_seen_at:now,revision:machine.revision+1},response=result(api,auth,machineJson(updated,now,api)),guards=[{sql:"SELECT 1 ok FROM credentials c JOIN machines m ON m.team_id=c.team_id AND m.id=c.machine_id JOIN sources s ON s.team_id=c.team_id AND s.id=c.source_id JOIN memberships mem ON mem.team_id=m.team_id AND mem.principal_id=m.custodian_id WHERE c.id=? AND c.kind='machine' AND c.machine_id=? AND c.revoked_at IS NULL AND (c.expires_at IS NULL OR c.expires_at>?) AND m.revoked_at IS NULL AND mem.active=1 AND s.state!='revoked' AND (m.incarnation_id IS NULL OR m.incarnation_id!=? OR m.heartbeat_sequence<?)",params:[auth.credential_id,machineId,now,b.incarnation_id,b.sequence],code:'authentication_required',status:401,message:'machine authorization or heartbeat sequence changed'}];await guardedMutation(api,res,auth,guards,statements,{status:200,body:response,targetId:machineId});send(res,200,response?.data?{...response,data:{...response.data,watched:await watchedFor(api,auth.team_id,machineId),...(unbound.length?{unbound_bindings:unbound}:{})}}:response,requestId);}
+async function heartbeat(api,req,res,auth,machineId,requestId){
+  if(auth.kind!=='machine'||auth.machine_id!==machineId)fail(403,'forbidden','exact machine credential required');
+  const machine=await api.store.one('SELECT m.*,(SELECT linked FROM machine_presence p WHERE p.team_id=m.team_id AND p.machine_id=m.id) presence_linked FROM machines m WHERE m.team_id=? AND m.id=?',[auth.team_id,machineId]);
+  if(!machine||machine.revoked_at!=null)fail(403,'forbidden','machine enrollment is not active');
+  const b=await parseJson(req);
+  exact(b,['incarnation_id','sequence','bindings','roots','host_key_id','activity']);
+  if(b.activity!==undefined&&(!Array.isArray(b.activity)||b.activity.length>4096))fail(400,'invalid_request','invalid activity');
+  if(b.host_key_id!==undefined&&(typeof b.host_key_id!=='string'||!/^[A-Za-z0-9_-]{16,128}$/.test(b.host_key_id)))fail(400,'invalid_request','invalid host_key_id');
+  if(typeof b.incarnation_id!=='string'||!Number.isSafeInteger(b.sequence)||b.sequence<1||!Array.isArray(b.bindings)||!Array.isArray(b.roots))fail(400,'invalid_request','invalid heartbeat');
+  if(machine.incarnation_id===b.incarnation_id&&b.sequence<=machine.heartbeat_sequence)return send(res,200,result(api,auth,{...machineJson(machine,api.clock.nowMs(),api),watched:await watchedFor(api,auth.team_id,machineId)}),requestId);
+  const rootIds=new Set();
+  for(const root of b.roots){
+    exact(root,['id','machine_id','name','harnesses'],'root');
+    if(root.machine_id!==machineId)fail(400,'invalid_request','root machine mismatch');
+    if(rootIds.has(root.id))fail(400,'invalid_request','duplicate root');
+    rootIds.add(root.id);
+  }
+  for(const binding of b.bindings){
+    if(!binding||typeof binding!=='object'||typeof binding.id!=='string'||!binding.id||typeof binding.harness!=='string'||typeof binding.native_session_id!=='string')fail(400,'invalid_request','invalid binding');
+    if(binding.machine_id!==machineId||binding.incarnation_id!==b.incarnation_id)fail(400,'invalid_request','binding machine or incarnation mismatch');
+  }
+  let activity=null;
+  if(Array.isArray(b.activity)){
+    const byNative=new Map();
+    for(const item of b.activity){
+      exact(item,['harness','native_id','turn','since'],'activity');
+      if(typeof item.harness!=='string'||!item.harness||item.harness.length>64||typeof item.native_id!=='string'||!item.native_id||item.native_id.length>256||typeof item.turn!=='string'||!item.turn||item.turn.length>32||(item.since!=null&&!Number.isSafeInteger(item.since)))fail(400,'invalid_request','invalid activity');
+      byNative.set(JSON.stringify([item.harness,item.native_id]),{harness:item.harness,native_id:item.native_id,turn:item.turn,since:item.since??null});
+    }
+    const entries=[...byNative.entries()].sort(([a],[b])=>a<b?-1:a>b?1:0);
+    const json=canonicalJson(entries.map(([,item])=>item));
+    activity={json,digest:sha256(json)};
+  }
+  const now=api.clock.nowMs(),newIncarnation=machine.incarnation_id!==null&&machine.incarnation_id!==b.incarnation_id;
+  const statements=[{sql:'UPDATE machines SET incarnation_id=?,heartbeat_sequence=?,last_seen_at=?,host_key_id=COALESCE(?,host_key_id),revision=revision+1 WHERE team_id=? AND id=?',params:[b.incarnation_id,b.sequence,now,b.host_key_id??null,auth.team_id,machineId]}];
+  if(newIncarnation)statements.push(
+    {sql:"UPDATE attachments SET state='closed',closed_at=? WHERE team_id=? AND machine_id=? AND state IN ('pending','active')",params:[now,auth.team_id,machineId]},
+    {sql:"UPDATE operations SET state='unknown' WHERE team_id=? AND id IN (SELECT operation_id FROM native_effects WHERE team_id=? AND machine_id=? AND state IN ('pending','running'))",params:[auth.team_id,auth.team_id,machineId]},
+    {sql:"UPDATE native_effects SET state='unknown',result_json=? WHERE team_id=? AND machine_id=? AND state IN ('pending','running')",params:[JSON.stringify({state:'unknown',error:null}),auth.team_id,machineId]}
+  );
+  const snapshotIndex=statements.length;
+  statements.push({kind:'heartbeat_snapshot',snapshot:{teamId:auth.team_id,machineId,sourceId:auth.source_id,incarnationId:b.incarnation_id,now,rootsJson:canonicalJson(b.roots),bindingsJson:canonicalJson(b.bindings),activity}});
+  const updated={...machine,incarnation_id:b.incarnation_id,heartbeat_sequence:b.sequence,last_seen_at:now,revision:machine.revision+1},response=result(api,auth,machineJson(updated,now,api));
+  const guards=[{sql:"SELECT 1 ok FROM credentials c JOIN machines m ON m.team_id=c.team_id AND m.id=c.machine_id JOIN sources s ON s.team_id=c.team_id AND s.id=c.source_id JOIN memberships mem ON mem.team_id=m.team_id AND mem.principal_id=m.custodian_id WHERE c.id=? AND c.kind='machine' AND c.machine_id=? AND c.revoked_at IS NULL AND (c.expires_at IS NULL OR c.expires_at>?) AND m.revoked_at IS NULL AND mem.active=1 AND s.state!='revoked' AND (m.incarnation_id IS NULL OR m.incarnation_id!=? OR m.heartbeat_sequence<?)",params:[auth.credential_id,machineId,now,b.incarnation_id,b.sequence],code:'authentication_required',status:401,message:'machine authorization or heartbeat sequence changed'}];
+  const results=await guardedMutation(api,res,auth,guards,statements,{status:200,body:response,targetId:machineId});
+  const {unbound}=results[snapshotIndex];
+  send(res,200,{...response,data:{...response.data,watched:await watchedFor(api,auth.team_id,machineId),...(unbound.length?{unbound_bindings:unbound}:{})}},requestId);
+}
 async function machineConfiguration(api,res,auth,machineId,requestId){
   if(auth.kind!=='machine'||auth.machine_id!==machineId)fail(auth.kind==='machine'?404:403,auth.kind==='machine'?'not_found':'forbidden','not found');
   const [machine,rules]=await api.store.snapshot([

package/server/migrations/027.mjs

@@ -0,0 +1,13 @@
+// A full heartbeat refreshes one report clock, not every activity/binding row.
+export default `
+CREATE TABLE machine_heartbeat_state (
+  team_id TEXT NOT NULL, machine_id TEXT NOT NULL, activity_source_id TEXT NOT NULL,
+  activity_digest TEXT NOT NULL, activity_observed_at INTEGER NOT NULL,
+  PRIMARY KEY(team_id,machine_id),
+  FOREIGN KEY(team_id,machine_id) REFERENCES machines(team_id,id)
+) STRICT;
+CREATE INDEX bindings_machine ON bindings(team_id,machine_id,id);
+CREATE INDEX machine_roots_machine ON machine_roots(team_id,machine_id,id);
+CREATE INDEX native_effects_binding ON native_effects(team_id,machine_id,binding_id);
+CREATE INDEX sources_machine_kind ON sources(team_id,machine_id,kind);
+`;

package/server/migrations/index.mjs

@@ -25,7 +25,8 @@ import m023 from './023.mjs';
 import m024 from './024.mjs';
 import m025 from './025.mjs';
 import m026 from './026.mjs';
-export const MIGRATIONS = [m001, m002, m003, m004, m005, m006, m007, m008, m009, m010, m011, m012, m013, m014, m015, m016, m017, m018, m019, m020, m021, m022, m023, m024, m025, m026];
+import m027 from './027.mjs';
+export const MIGRATIONS = [m001, m002, m003, m004, m005, m006, m007, m008, m009, m010, m011, m012, m013, m014, m015, m016, m017, m018, m019, m020, m021, m022, m023, m024, m025, m026, m027];
 /** A message row's `until_revision` while its session's current version still holds it (schema 9). */
 export const OPEN_REVISION = Number.MAX_SAFE_INTEGER;
 /** SQL: message row `m` belongs to the version at revision ?1 (bind the revision twice). */

package/server/published-events.mjs

@@ -11,6 +11,7 @@ const stmt = (sql, ...params) => ({sql, params});
 // A Durable Object binds at most 100 parameters to a statement, so id lists go in batches.
 const batches = (list, size = 90) => Array.from({length: Math.ceil(list.length / size)}, (_, at) => list.slice(at * size, (at + 1) * size));
 const holes = list => list.map(() => '?').join(',');
+const EVENT_REPLAY_BATCH = 32;
 const fail = (status,code,message) => {throw Object.assign(new Error(message),{status,code});};
 const text = (value,label) => {if(typeof value!=='string'||!value||value.length>256||/[\x00-\x1f]/.test(value))fail(400,'invalid_request',`${label} must be a nonempty string of at most 256 characters`);return value;};
 const eventOf = row => ({id:row.id,org:row.team_id,at:row.created_at,actor:JSON.parse(row.actor_json),kind:row.kind,resource:{kind:row.resource_kind,id:row.resource_id},data:JSON.parse(row.data_json)});
@@ -179,8 +180,17 @@ async function eventStream({api,req,res,auth,url,requestId}){
   let closed=false,running=null,dirty=false;
   const close=()=>{if(closed)return;closed=true;clearInterval(timer);unsubscribe();req.off('close',close);api.events.off('shutdown',close);release();res.end();};
   const terminal=(type,reason)=>{if(!closed)writeSse(res,{type,reason,terminal:true});close();};
-  const write=async event=>{if(closed)return false;const room=await sseRoom(res,Buffer.byteLength(JSON.stringify(event)));if(closed||room==='closed')return false;if(room==='slow'){terminal('resync_required','slow_reader');return false;}writeSse(res,event);return true;};
+  const detail=values=>api.workDetail?.(values);
+  // SQLite on the object resolves as microtasks. An await per event does not
+  // let another request or the response transport run. The hosted meter yields
+  // at the end of each page; the Node host needs the same timer boundary.
+  const page=async(name,work,values={})=>{
+    if(api.work)return api.work(`publishedEvents.${name}`,work,values);
+    try{return await work();}finally{await new Promise(resolve=>setTimeout(resolve,0));}
+  };
+  const write=async event=>{if(closed)return false;detail({stage:'write',queued_bytes:res.writableLength??0});const room=await sseRoom(res,Buffer.byteLength(JSON.stringify(event)));if(closed||room==='closed')return false;if(room==='slow'){terminal('resync_required','slow_reader');return false;}writeSse(res,event);return true;};
   const readable=async event=>{
+    detail({stage:'authority'});
     const current=await liveAuth(api,auth);
     if(current.epoch!==auth.epoch)fail(409,'resync_required','authorization changed during event delivery');
     auth=current;return mayReadEvent(api,auth,event);
@@ -196,25 +206,52 @@ async function eventStream({api,req,res,auth,url,requestId}){
     if(cursor!==null&&(cursor<bounds.floor||cursor>Math.max(bounds.last,bounds.floor))){terminal('resync_required','retention');return;}
     if(cursor===null){
       // Subscribe before reading the atomic snapshot. Replay after this watermark repairs all concurrent writes.
-      const [head,resources]=await api.store.snapshot([
+      const [head,resources]=await page('snapshot',()=>api.store.snapshot([
         {kind:'one',sql:'SELECT COALESCE((SELECT max(sequence) FROM org_events WHERE team_id=?),(SELECT pruned_through FROM event_retention WHERE team_id=?),0) last',params:[auth.team_id,auth.team_id]},
         {kind:'query',sql:'SELECT r.sequence FROM event_resources r JOIN org_events e ON e.sequence=r.sequence WHERE r.team_id=? AND (? IS NULL OR r.source_id=?) AND (? IS NULL OR r.resource_kind IN (SELECT value FROM json_each(?)) OR e.kind IN (SELECT value FROM json_each(?))) ORDER BY r.source_id,r.resource_kind,r.resource_id',params:[auth.team_id,query.source,query.source,filterResources,filterResources,filterKinds]},
-      ]);
+      ]),{stage:'query'});
       cursor=head.last;if(!await write({type:'snapshot_begin'}))return;
-      for(let at=0;at<resources.length;at+=32){
-        const sequences=resources.slice(at,at+32).map(row=>row.sequence);
-        const rows=await api.store.query('SELECT * FROM org_events WHERE team_id=? AND sequence IN (SELECT value FROM json_each(?)) ORDER BY sequence',[auth.team_id,JSON.stringify(sequences)]);
-        if(rows.length!==sequences.length){terminal('resync_required','snapshot_retention');return;}
-        for(const row of rows){const event=eventOf(row);if(await readable(event))if(!await write({type:'snapshot_item',sequence:row.sequence,data:event}))return;}
+      for(let at=0;at<resources.length&&!closed;at+=EVENT_REPLAY_BATCH){
+        const sequences=resources.slice(at,at+EVENT_REPLAY_BATCH).map(row=>row.sequence);
+        const complete=await page('snapshotPage',async()=>{
+          const rows=await api.store.query('SELECT * FROM org_events WHERE team_id=? AND sequence IN (SELECT value FROM json_each(?)) ORDER BY sequence',[auth.team_id,JSON.stringify(sequences)]);
+          if(rows.length!==sequences.length){terminal('resync_required','snapshot_retention');return false;}
+          let considered=0,delivered=0;
+          for(const row of rows){
+            if(closed)return false;
+            const event=eventOf(row);considered++;detail({events_considered:considered});
+            if(await readable(event)){if(!await write({type:'snapshot_item',sequence:row.sequence,data:event}))return false;delivered++;detail({events_delivered:delivered});}
+          }
+          detail({stage:'complete'});return true;
+        },{stage:'query',events_considered:0,events_delivered:0,snapshot_resources:resources.length,page_offset:at});
+        if(!complete)return;
       }
       if(!await write({type:'snapshot_end',cursor:codec.encode(cursor)}))return;
     }
-    for(;;){
-      const rows=await api.store.query('SELECT * FROM org_events WHERE team_id=? AND sequence>? AND (? IS NULL OR source_id=?) AND (? IS NULL OR kind IN (SELECT value FROM json_each(?))) ORDER BY sequence LIMIT 128',[auth.team_id,cursor,query.source,query.source,filterKinds,filterKinds]);
-      if(!rows.length){cursor=Math.max(cursor,bounds.last);await write({type:'checkpoint',cursor:codec.encode(cursor)});break;}
-      for(const row of rows){if(closed)return;cursor=row.sequence;const event=eventOf(row);if(await readable(event)&&!await write({type:'event',sequence:row.sequence,cursor:codec.encode(cursor),data:event}))return;}
-      // Advance even when the reader was not authorized for a resource; never leak its data.
-      if(!await write({type:'checkpoint',cursor:codec.encode(cursor)}))return;if(rows.length<128)break;
+    // Finish one observed watermark rather than chasing concurrent publishers
+    // forever. The subscribed dirty flag schedules the next watermark.
+    const through=Math.max(bounds.last,cursor);
+    while(!closed){
+      const state=await page('replayPage',async()=>{
+        // Retention can run during the preceding page's timer yield. Read its
+        // floor and this page in one snapshot before advancing the cursor,
+        // including when filters leave an empty page.
+        const [retention,rows]=await api.store.snapshot([
+          {kind:'one',sql:'SELECT COALESCE((SELECT pruned_through FROM event_retention WHERE team_id=?),0) floor',params:[auth.team_id]},
+          {kind:'query',sql:`SELECT * FROM org_events WHERE team_id=? AND sequence>? AND sequence<=? AND (? IS NULL OR source_id=?) AND (? IS NULL OR kind IN (SELECT value FROM json_each(?))) ORDER BY sequence LIMIT ${EVENT_REPLAY_BATCH}`,params:[auth.team_id,cursor,through,query.source,query.source,filterKinds,filterKinds]},
+        ]);
+        if(cursor<retention.floor){terminal('resync_required','retention');return 'closed';}
+        if(!rows.length){cursor=through;await write({type:'checkpoint',cursor:codec.encode(cursor)});detail({stage:'complete'});return 'done';}
+        let considered=0,delivered=0;
+        for(const row of rows){
+          if(closed)return 'closed';cursor=row.sequence;const event=eventOf(row);considered++;detail({events_considered:considered});
+          if(await readable(event)){if(!await write({type:'event',sequence:row.sequence,cursor:codec.encode(cursor),data:event}))return 'closed';delivered++;detail({events_delivered:delivered});}
+        }
+        // Advance even when the reader was not authorized for a resource; never leak its data.
+        if(!await write({type:'checkpoint',cursor:codec.encode(cursor)}))return 'closed';
+        detail({stage:'complete'});return rows.length<EVENT_REPLAY_BATCH?'done':'more';
+      },{stage:'query',events_considered:0,events_delivered:0});
+      if(state!=='more')break;
     }
   };
   const enqueue=()=>{dirty=true;if(running)return running;running=(async()=>{while(dirty&&!closed){dirty=false;await drain();}})().catch(error=>terminal([401,403].includes(error.status)?'access_revoked':'resync_required',error.code??'unavailable')).finally(()=>{running=null;});return running;};