@volter/teams 0.2.40 → 0.2.41

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/teams",
-  "version": "0.2.40",
+  "version": "0.2.41",
   "type": "module",
   "description": "Volter Teams, the neutral team plane: machine identity and daemon core, the local door and server link, admission from machine grants, product doors, and the Teams server's neutral routes (company RFC 0008).",
   "license": "MIT",

package/server/http.mjs

@@ -23,7 +23,21 @@ const parseJson=req=>parseRequestJson(req,fail);
 const result=(api,auth,data)=>({version:1,context:{server_id:api.serverId,team_id:auth.team_id,principal_id:auth.principal_id},data});
 function sideEffectStatements(api,res,status,body,now,targetId=null,journal=false){const statements=[];if(status<400&&res._idempotencyRecord){const record=res._idempotencyRecord,safe=redactSecretReplay(record.path,body);statements.push({sql:'INSERT INTO idempotency(team_id,principal_id,route,key,request_digest,response_status,response_json,expires_at) VALUES (?,?,?,?,?,?,?,?)',params:[record.teamId,record.principalId,record.route,record.key,record.requestDigest,status,journal?{$journalJsonValue:safe}:JSON.stringify(safe),now+86400000]});}if(res._auditRecord){const record=res._auditRecord;statements.push({sql:'INSERT INTO audit(team_id,id,actor_id,operation,target_id,outcome,revision,created_at,actor_json) VALUES (?,?,?,?,?,?,?,?,?)',params:[record.teamId,api.ids.next('audit'),record.actorId,record.operation,targetId,status<400?'allowed':'denied',journal?{$journalRevision:true}:null,now,JSON.stringify(record.act??[])]});}return statements;}
 async function persistSideEffects(api,res,status,body){const statements=sideEffectStatements(api,res,status,body,api.clock.nowMs());if(statements.length)await api.store.transaction(statements);}
-function send(res,status,body,requestId,headers={}){const deliver=()=>{res.writeHead(status,{'Content-Type':'application/json; charset=utf-8','Cache-Control':'no-store','X-Request-ID':requestId,...headers});res.end(body===null?'':JSON.stringify(body));};if(!res._sideEffectsCommitted&&res._api&&(res._idempotencyRecord||res._auditRecord))return persistSideEffects(res._api,res,status,body).then(deliver,error=>{if(/UNIQUE constraint failed: idempotency\./.test(error?.message??''))return deliver();if(res.headersSent)return;res.writeHead(500,{'Content-Type':'application/json; charset=utf-8','Cache-Control':'no-store','X-Request-ID':requestId});res.end(JSON.stringify({error:{code:'internal',message:'The request was not recorded.'}}));});deliver();}
+function send(res,status,body,requestId,headers={}){
+  const deliver=(actualStatus=status,actualBody=body,actualHeaders=headers)=>{
+    // The host observes the response actually issued, including a refusal
+    // whose audit write failed. It decides which diagnostic fields are safe.
+    res._workResponse?.(actualStatus,actualBody);
+    res.writeHead(actualStatus,{'Content-Type':'application/json; charset=utf-8','Cache-Control':'no-store','X-Request-ID':requestId,...actualHeaders});
+    res.end(actualBody===null?'':JSON.stringify(actualBody));
+  };
+  if(!res._sideEffectsCommitted&&res._api&&(res._idempotencyRecord||res._auditRecord))return persistSideEffects(res._api,res,status,body).then(()=>deliver(),error=>{
+    if(/UNIQUE constraint failed: idempotency\./.test(error?.message??''))return deliver();
+    if(res.headersSent)return;
+    deliver(500,{error:{code:'internal',message:'The request was not recorded.'}},{});
+  });
+  deliver();
+}
 async function guardedMutation(api,res,auth,guards,statements,{status,body,targetId=null,journal=false}){const now=api.clock.nowMs(),all=[...statements,...sideEffectStatements(api,res,status,body,now,targetId,journal)],result=journal?await api.store.journalTransaction(auth.team_id,[...guards,...appAuthGuards(auth,now)],all):await api.store.guardedTransaction([...guards,...appAuthGuards(auth,now)],all);res._sideEffectsCommitted=true;api.events.publish(auth.team_id,{type:'events_available'});return result;}
 const requireRevision=(req,current,label='resource')=>{const expected=Number(String(req.headers['if-match']??'').replaceAll('"',''));if(!Number.isSafeInteger(expected))fail(400,'invalid_request','If-Match is required');if(expected!==current)fail(409,'revision_mismatch',`${label} revision changed`);return expected;};
 const CORS={'Access-Control-Allow-Origin':'*','Access-Control-Expose-Headers':'ETag, X-Request-ID, Content-Range'};

package/server/published-events.mjs

@@ -11,6 +11,7 @@ const stmt = (sql, ...params) => ({sql, params});
 // A Durable Object binds at most 100 parameters to a statement, so id lists go in batches.
 const batches = (list, size = 90) => Array.from({length: Math.ceil(list.length / size)}, (_, at) => list.slice(at * size, (at + 1) * size));
 const holes = list => list.map(() => '?').join(',');
+const EVENT_REPLAY_BATCH = 32;
 const fail = (status,code,message) => {throw Object.assign(new Error(message),{status,code});};
 const text = (value,label) => {if(typeof value!=='string'||!value||value.length>256||/[\x00-\x1f]/.test(value))fail(400,'invalid_request',`${label} must be a nonempty string of at most 256 characters`);return value;};
 const eventOf = row => ({id:row.id,org:row.team_id,at:row.created_at,actor:JSON.parse(row.actor_json),kind:row.kind,resource:{kind:row.resource_kind,id:row.resource_id},data:JSON.parse(row.data_json)});
@@ -179,8 +180,17 @@ async function eventStream({api,req,res,auth,url,requestId}){
   let closed=false,running=null,dirty=false;
   const close=()=>{if(closed)return;closed=true;clearInterval(timer);unsubscribe();req.off('close',close);api.events.off('shutdown',close);release();res.end();};
   const terminal=(type,reason)=>{if(!closed)writeSse(res,{type,reason,terminal:true});close();};
-  const write=async event=>{if(closed)return false;const room=await sseRoom(res,Buffer.byteLength(JSON.stringify(event)));if(closed||room==='closed')return false;if(room==='slow'){terminal('resync_required','slow_reader');return false;}writeSse(res,event);return true;};
+  const detail=values=>api.workDetail?.(values);
+  // SQLite on the object resolves as microtasks. An await per event does not
+  // let another request or the response transport run. The hosted meter yields
+  // at the end of each page; the Node host needs the same timer boundary.
+  const page=async(name,work,values={})=>{
+    if(api.work)return api.work(`publishedEvents.${name}`,work,values);
+    try{return await work();}finally{await new Promise(resolve=>setTimeout(resolve,0));}
+  };
+  const write=async event=>{if(closed)return false;detail({stage:'write',queued_bytes:res.writableLength??0});const room=await sseRoom(res,Buffer.byteLength(JSON.stringify(event)));if(closed||room==='closed')return false;if(room==='slow'){terminal('resync_required','slow_reader');return false;}writeSse(res,event);return true;};
   const readable=async event=>{
+    detail({stage:'authority'});
     const current=await liveAuth(api,auth);
     if(current.epoch!==auth.epoch)fail(409,'resync_required','authorization changed during event delivery');
     auth=current;return mayReadEvent(api,auth,event);
@@ -196,25 +206,52 @@ async function eventStream({api,req,res,auth,url,requestId}){
     if(cursor!==null&&(cursor<bounds.floor||cursor>Math.max(bounds.last,bounds.floor))){terminal('resync_required','retention');return;}
     if(cursor===null){
       // Subscribe before reading the atomic snapshot. Replay after this watermark repairs all concurrent writes.
-      const [head,resources]=await api.store.snapshot([
+      const [head,resources]=await page('snapshot',()=>api.store.snapshot([
         {kind:'one',sql:'SELECT COALESCE((SELECT max(sequence) FROM org_events WHERE team_id=?),(SELECT pruned_through FROM event_retention WHERE team_id=?),0) last',params:[auth.team_id,auth.team_id]},
         {kind:'query',sql:'SELECT r.sequence FROM event_resources r JOIN org_events e ON e.sequence=r.sequence WHERE r.team_id=? AND (? IS NULL OR r.source_id=?) AND (? IS NULL OR r.resource_kind IN (SELECT value FROM json_each(?)) OR e.kind IN (SELECT value FROM json_each(?))) ORDER BY r.source_id,r.resource_kind,r.resource_id',params:[auth.team_id,query.source,query.source,filterResources,filterResources,filterKinds]},
-      ]);
+      ]),{stage:'query'});
       cursor=head.last;if(!await write({type:'snapshot_begin'}))return;
-      for(let at=0;at<resources.length;at+=32){
-        const sequences=resources.slice(at,at+32).map(row=>row.sequence);
-        const rows=await api.store.query('SELECT * FROM org_events WHERE team_id=? AND sequence IN (SELECT value FROM json_each(?)) ORDER BY sequence',[auth.team_id,JSON.stringify(sequences)]);
-        if(rows.length!==sequences.length){terminal('resync_required','snapshot_retention');return;}
-        for(const row of rows){const event=eventOf(row);if(await readable(event))if(!await write({type:'snapshot_item',sequence:row.sequence,data:event}))return;}
+      for(let at=0;at<resources.length&&!closed;at+=EVENT_REPLAY_BATCH){
+        const sequences=resources.slice(at,at+EVENT_REPLAY_BATCH).map(row=>row.sequence);
+        const complete=await page('snapshotPage',async()=>{
+          const rows=await api.store.query('SELECT * FROM org_events WHERE team_id=? AND sequence IN (SELECT value FROM json_each(?)) ORDER BY sequence',[auth.team_id,JSON.stringify(sequences)]);
+          if(rows.length!==sequences.length){terminal('resync_required','snapshot_retention');return false;}
+          let considered=0,delivered=0;
+          for(const row of rows){
+            if(closed)return false;
+            const event=eventOf(row);considered++;detail({events_considered:considered});
+            if(await readable(event)){if(!await write({type:'snapshot_item',sequence:row.sequence,data:event}))return false;delivered++;detail({events_delivered:delivered});}
+          }
+          detail({stage:'complete'});return true;
+        },{stage:'query',events_considered:0,events_delivered:0,snapshot_resources:resources.length,page_offset:at});
+        if(!complete)return;
       }
       if(!await write({type:'snapshot_end',cursor:codec.encode(cursor)}))return;
     }
-    for(;;){
-      const rows=await api.store.query('SELECT * FROM org_events WHERE team_id=? AND sequence>? AND (? IS NULL OR source_id=?) AND (? IS NULL OR kind IN (SELECT value FROM json_each(?))) ORDER BY sequence LIMIT 128',[auth.team_id,cursor,query.source,query.source,filterKinds,filterKinds]);
-      if(!rows.length){cursor=Math.max(cursor,bounds.last);await write({type:'checkpoint',cursor:codec.encode(cursor)});break;}
-      for(const row of rows){if(closed)return;cursor=row.sequence;const event=eventOf(row);if(await readable(event)&&!await write({type:'event',sequence:row.sequence,cursor:codec.encode(cursor),data:event}))return;}
-      // Advance even when the reader was not authorized for a resource; never leak its data.
-      if(!await write({type:'checkpoint',cursor:codec.encode(cursor)}))return;if(rows.length<128)break;
+    // Finish one observed watermark rather than chasing concurrent publishers
+    // forever. The subscribed dirty flag schedules the next watermark.
+    const through=Math.max(bounds.last,cursor);
+    while(!closed){
+      const state=await page('replayPage',async()=>{
+        // Retention can run during the preceding page's timer yield. Read its
+        // floor and this page in one snapshot before advancing the cursor,
+        // including when filters leave an empty page.
+        const [retention,rows]=await api.store.snapshot([
+          {kind:'one',sql:'SELECT COALESCE((SELECT pruned_through FROM event_retention WHERE team_id=?),0) floor',params:[auth.team_id]},
+          {kind:'query',sql:`SELECT * FROM org_events WHERE team_id=? AND sequence>? AND sequence<=? AND (? IS NULL OR source_id=?) AND (? IS NULL OR kind IN (SELECT value FROM json_each(?))) ORDER BY sequence LIMIT ${EVENT_REPLAY_BATCH}`,params:[auth.team_id,cursor,through,query.source,query.source,filterKinds,filterKinds]},
+        ]);
+        if(cursor<retention.floor){terminal('resync_required','retention');return 'closed';}
+        if(!rows.length){cursor=through;await write({type:'checkpoint',cursor:codec.encode(cursor)});detail({stage:'complete'});return 'done';}
+        let considered=0,delivered=0;
+        for(const row of rows){
+          if(closed)return 'closed';cursor=row.sequence;const event=eventOf(row);considered++;detail({events_considered:considered});
+          if(await readable(event)){if(!await write({type:'event',sequence:row.sequence,cursor:codec.encode(cursor),data:event}))return 'closed';delivered++;detail({events_delivered:delivered});}
+        }
+        // Advance even when the reader was not authorized for a resource; never leak its data.
+        if(!await write({type:'checkpoint',cursor:codec.encode(cursor)}))return 'closed';
+        detail({stage:'complete'});return rows.length<EVENT_REPLAY_BATCH?'done':'more';
+      },{stage:'query',events_considered:0,events_delivered:0});
+      if(state!=='more')break;
     }
   };
   const enqueue=()=>{dirty=true;if(running)return running;running=(async()=>{while(dirty&&!closed){dirty=false;await drain();}})().catch(error=>terminal([401,403].includes(error.status)?'access_revoked':'resync_required',error.code??'unavailable')).finally(()=>{running=null;});return running;};