@volter/teams 0.2.25 → 0.2.26

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/teams",
-  "version": "0.2.25",
+  "version": "0.2.26",
   "type": "module",
   "description": "Volter Teams, the neutral team plane: machine identity and daemon core, the local door and server link, admission from machine grants, product doors, and the Teams server's neutral routes (company RFC 0008).",
   "license": "MIT",

package/server/apps.mjs

@@ -363,7 +363,7 @@ export async function appRoute(ctx) {
 
 /** Standard Webhooks: stable delivery id across retries, timestamp refreshed and HMAC over exact body bytes. */
 export async function deliverAppWebhooks(api) {
-  const due = await api.store.query("SELECT id FROM app_webhook_deliveries WHERE state='pending' AND next_at<=? AND (lease_until IS NULL OR lease_until<=?) ORDER BY next_at,id LIMIT 10", [api.clock.nowMs(), api.clock.nowMs()]);
+  const due = await api.store.query("SELECT id FROM app_webhook_deliveries WHERE state='pending' AND next_at<=? AND (lease_until IS NULL OR lease_until<=?) ORDER BY next_at LIMIT 10", [api.clock.nowMs(), api.clock.nowMs()]);
   for (const { id } of due) {
     const now = api.clock.nowMs(), lease = now + 30000;
     try {
@@ -397,6 +397,12 @@ export async function deliverAppWebhooks(api) {
     const state = delivered ? 'delivered' : row.attempts >= 8 ? 'failed' : 'pending';
     await api.store.transaction([statement("UPDATE app_webhook_deliveries SET state=?,last_status=?,lease_until=NULL,next_at=? WHERE id=? AND lease_until=? AND state='pending'", state, status, api.clock.nowMs() + Math.min(3600000, 1000 * 2 ** row.attempts), id, lease)]);
   }
-  await api.store.transaction([statement('DELETE FROM app_install_codes WHERE expires_at<=?', api.clock.nowMs()), statement("DELETE FROM app_webhook_deliveries WHERE state!='pending' AND created_at<?", api.clock.nowMs() - 7 * 86400000)]);
+  // Week-old settled deliveries go from the front (oldest rowid first), a bounded batch per run, stopping at the first
+  // delivery younger than a week or still pending (it settles within its eight attempts). Expired install codes go
+  // with maintain's other expiries (t_6c01c966).
+  const front = await api.store.query('SELECT rowid AS id,state,created_at FROM app_webhook_deliveries ORDER BY rowid LIMIT 500', []);
+  let through = null; for (const row of front) { if (row.state === 'pending' || row.created_at >= api.clock.nowMs() - 7 * 86400000) break; through = row.id; }
+  if (through != null) await api.store.transaction([statement('DELETE FROM app_webhook_deliveries WHERE rowid<=? AND state<>\'pending\'', through)]);
+  if (through != null && front.length >= 500 && front.at(-1).id === through) return true;
   return due.length >= 10;
 }

package/server/core.mjs

@@ -1,5 +1,5 @@
 import { deliverAppWebhooks } from './apps.mjs';
-import { reconcileMachinePresence } from './machine-events.mjs';
+import { reconcileMachinePresence, LINK_GRACE_MS } from './machine-events.mjs';
 // The Teams server, independent of its host. `createTeamsApi` builds the API
 // object every route and door reads; a host attaches a store (`api.attach`),
 // bootstraps the server identity once, serves requests through http.mjs's
@@ -20,7 +20,7 @@ export function createTeamsApi({clock=productionClock,ids=productionIds,publicOr
   if(!Number.isSafeInteger(uploadLimitBytes)||uploadLimitBytes<1||uploadLimitBytes>256*1024*1024)throw new TypeError('uploadLimitBytes must be between 1 and 268435456');
   let store=null,serverId=null,localOrigin=null,harnessRoutes=null;
   const events=new TeamEvents();
-  return {clock,ids,events,publicOrigin,appRootSecret,uploadLimitBytes,rateState:new Map(),rateSpenders:new Map(),rateReported:new Map(),openStreams:new Map(),securityEvents:[],
+  return {clock,ids,events,publicOrigin,appRootSecret,uploadLimitBytes,activatedAt:clock.nowMs(),rateState:new Map(),rateSpenders:new Map(),rateReported:new Map(),openStreams:new Map(),securityEvents:[],
     get store(){return store;},get serverId(){return serverId;},get localOrigin(){return localOrigin;},get harnessRoutes(){return harnessRoutes;},
     attach(next){store=next.store??store;if(next.serverId!==undefined)serverId=next.serverId;if(next.localOrigin!==undefined)localOrigin=next.localOrigin;if(next.harnessRoutes!==undefined)harnessRoutes=next.harnessRoutes;},
   };
@@ -51,74 +51,230 @@ export async function bootstrapServer(api,{initTeam,bootstrap}){
 }
 
 /** Expire cursors, idempotency records, staged captures and uploads, and old journal rows. */
-export async function maintain(api){const now=api.clock.nowMs();
-  // Replay records expire after a day; keeping them forever can fill the object and stop heartbeats.
-  // Delete them before any retention step that may update a surviving version.
-  const expired=await api.store.query('SELECT rowid FROM idempotency WHERE expires_at<=? LIMIT 1000',[now]);
-  if(expired.length)await api.store.transaction([{sql:'DELETE FROM idempotency WHERE rowid IN (SELECT rowid FROM idempotency WHERE expires_at<=? LIMIT 1000)',params:[now]}]);
-  // Deletes go first, each in a transaction of its own: on a full database an insert fails, and a run that begins
-  // with one never reaches the deletes that would make room.
-  // A committed capture's parts are its messages in transit; once committed, the session's version holds them, so
-  // they go (a backlog of them filled the database).
-  const parts=await api.store.query(`SELECT p.team_id,p.capture_id,p.part_index FROM capture_parts p JOIN captures c ON c.team_id=p.team_id AND c.id=p.capture_id WHERE c.state='committed' LIMIT ${PART_PRUNE_BATCH}`,[]);
-  for(let at=0;at<parts.length;at+=50)await api.store.transaction(parts.slice(at,at+50).map(({team_id,capture_id,part_index})=>({sql:'DELETE FROM capture_parts WHERE team_id=? AND capture_id=? AND part_index=?',params:[team_id,capture_id,part_index]})));
-  // Retired versions release their native bundles. Keep every bundle still named by a version,
-  // continuation (including completed receipts), or in-flight capture. Never-committed uploads retain their one-day grace.
-  const orphaned=await api.store.query(`SELECT u.team_id,u.id FROM uploads u WHERE u.committed=1
-    AND NOT EXISTS (SELECT 1 FROM session_artifacts a WHERE a.team_id=u.team_id AND a.upload_id=u.id)
-    AND NOT EXISTS (SELECT 1 FROM continuations c WHERE c.team_id=u.team_id AND instr(c.value_json,u.id)>0)
-    AND NOT EXISTS (SELECT 1 FROM captures c WHERE c.team_id=u.team_id AND c.state='staging' AND instr(c.begin_json,u.id)>0)
-    LIMIT 10`,[]);
-  for(const upload of orphaned)await api.store.transaction([{sql:`DELETE FROM uploads AS u WHERE team_id=? AND id=? AND committed=1
-    AND NOT EXISTS (SELECT 1 FROM session_artifacts a WHERE a.team_id=u.team_id AND a.upload_id=u.id)
-    AND NOT EXISTS (SELECT 1 FROM continuations c WHERE c.team_id=u.team_id AND instr(c.value_json,u.id)>0)
-    AND NOT EXISTS (SELECT 1 FROM captures c WHERE c.team_id=u.team_id AND c.state='staging' AND instr(c.begin_json,u.id)>0)`,params:[upload.team_id,upload.id]}]);
-  const unusedBlobs = await api.store.query(`SELECT team_id,sha256 FROM artifact_blobs b
-    WHERE NOT EXISTS (SELECT 1 FROM artifact_chunk_refs r WHERE r.team_id=b.team_id AND r.sha256=b.sha256) LIMIT 50`);
-  for (const blob of unusedBlobs) await api.store.transaction([{sql:`DELETE FROM artifact_blobs AS b WHERE team_id=? AND sha256=?
-    AND NOT EXISTS (SELECT 1 FROM artifact_chunk_refs r WHERE r.team_id=b.team_id AND r.sha256=b.sha256)`,params:[blob.team_id,blob.sha256]}]);
-  // Every capture writes its session a new version (its messages are carried, not copied: schema 9), so versions pile
-  // up (hundreds per session) until listing them outgrew a Worker's memory. Kept: each session's current version, the newest
-  // version from each of its sources (the next capture and a correlation read that one), and every version an
-  // unexpired list, search or history cursor names. The rest go, a bounded batch per run.
-  const stale=await api.store.query(`SELECT v.team_id,v.session_id,v.revision FROM session_versions v JOIN sessions s ON s.team_id=v.team_id AND s.id=v.session_id
-    WHERE v.revision<>s.revision
-    AND EXISTS (SELECT 1 FROM session_versions n WHERE n.team_id=v.team_id AND n.session_id=v.session_id AND n.revision>v.revision AND json_extract(n.source_projection_json,'$.source_id') IS json_extract(v.source_projection_json,'$.source_id'))
-    AND (NOT EXISTS (SELECT 1 FROM session_artifacts a WHERE a.team_id=v.team_id AND a.session_id=v.session_id AND a.revision=v.revision AND a.provenance='capture')
-      OR EXISTS (SELECT 1 FROM session_artifacts a JOIN session_versions n ON n.team_id=a.team_id AND n.session_id=a.session_id AND n.revision=a.revision
-        WHERE a.team_id=v.team_id AND a.session_id=v.session_id AND a.revision>v.revision AND a.provenance='capture'
-        AND json_extract(n.source_projection_json,'$.source_id') IS json_extract(v.source_projection_json,'$.source_id')))
-    AND NOT EXISTS (SELECT 1 FROM cursors c, json_each(c.snapshot_json) j WHERE c.team_id=v.team_id AND c.kind IN ('list','search') AND c.expires_at>? AND json_extract(j.value,'$[0]')=v.session_id AND json_extract(j.value,'$[1]')=v.revision)
-    AND NOT EXISTS (SELECT 1 FROM cursors c WHERE c.team_id=v.team_id AND c.kind='history' AND c.expires_at>? AND json_extract(c.snapshot_json,'$.session_id')=v.session_id AND json_extract(c.snapshot_json,'$.revision')=v.revision)
-    LIMIT ${VERSION_PRUNE_BATCH}`,[now,now]);
-  // A message row lives from the version that wrote it until one that changed it (schema 9), so a row a later version
-  // still holds moves to the next version that holds it before its own goes; the rest go with the version.
-  for(const {team_id,session_id,revision} of stale)await api.store.transaction([
-    {sql:'UPDATE messages SET revision=(SELECT MIN(k.revision) FROM session_versions k WHERE k.team_id=messages.team_id AND k.session_id=messages.session_id AND k.revision>messages.revision AND k.revision<COALESCE(messages.until_revision,messages.revision+1)) WHERE team_id=? AND session_id=? AND revision=? AND EXISTS (SELECT 1 FROM session_versions k WHERE k.team_id=messages.team_id AND k.session_id=messages.session_id AND k.revision>messages.revision AND k.revision<COALESCE(messages.until_revision,messages.revision+1))',params:[team_id,session_id,revision]},
-    ...['messages','session_artifacts','session_versions'].map(table=>({sql:`DELETE FROM ${table} WHERE team_id=? AND session_id=? AND revision=?`,params:[team_id,session_id,revision]})),
-  ]);
-  await api.store.transaction([
-        {sql:'INSERT INTO event_retention SELECT e.team_id,MAX(e.sequence) FROM org_events e LEFT JOIN (SELECT team_id,MIN(sequence) AS first FROM org_events WHERE created_at>=? GROUP BY team_id) n ON n.team_id=e.team_id WHERE e.created_at<? AND (n.first IS NULL OR e.sequence<n.first) GROUP BY e.team_id ON CONFLICT(team_id) DO UPDATE SET pruned_through=MAX(pruned_through,excluded.pruned_through)',params:[now-7*86400000,now-7*86400000]},
-        {sql:'DELETE FROM org_events WHERE sequence<=(SELECT pruned_through FROM event_retention WHERE team_id=org_events.team_id) AND sequence NOT IN (SELECT sequence FROM event_resources)',params:[]},
-        {sql:'DELETE FROM cursors WHERE expires_at<=?',params:[now]},
-        {sql:'DELETE FROM global_cursors WHERE expires_at<=?',params:[now]},
-        {sql:'DELETE FROM global_idempotency WHERE expires_at<=?',params:[now]},
-        {sql:'DELETE FROM journal WHERE created_at+86400000<=?',params:[now]},
-      ]).catch(()=>{});
-  await api.store.transaction([
-        {sql:'INSERT OR IGNORE INTO capture_expiries(team_id,source_id,id,expired_at) SELECT team_id,source_id,id,? FROM captures WHERE state=\'staging\' AND created_at+86400000<=?',params:[now,now]},
-        {sql:'DELETE FROM captures WHERE state=\'staging\' AND created_at+86400000<=?',params:[now]},
-        {sql:'INSERT OR IGNORE INTO capture_expiries(team_id,source_id,id,expired_at) SELECT team_id,source_id,id,? FROM uploads WHERE committed=0 AND created_at+86400000<=?',params:[now,now]},
-        {sql:'DELETE FROM uploads WHERE committed=0 AND created_at+86400000<=?',params:[now]},
-      ]).catch(()=>{});
-  // Presence reconciliation can INSERT. Never put it ahead of the deletes
-  // that make room when the database is full.
-  await reconcileMachinePresence(api);
-  const webhookBacklog=await deliverAppWebhooks(api);
-  const result={pruned_versions:stale.length,pruned_parts:parts.length,pruned_idempotency:expired.length,pruned_uploads:orphaned.length,pruned_blobs:unusedBlobs.length,backlog:webhookBacklog||unusedBlobs.length>=50||orphaned.length>=10||expired.length>=1000||stale.length>=VERSION_PRUNE_BATCH||parts.length>=PART_PRUNE_BATCH};
-  if(result.pruned_versions||result.pruned_parts||result.pruned_idempotency||result.pruned_uploads)console.log(`maintain: pruned ${result.pruned_uploads} unreachable bundles, ${result.pruned_idempotency} expired replay records, ${result.pruned_parts} parts, ${result.pruned_versions} versions${result.backlog?' (backlog)':''}`);
+/**
+ * The store's retention, run on the object's alarm every minute (and in seconds while a backlog remains). Each step
+ * costs what it removes, never the store (t_6c01c966): it reads through a key, the front of a table it deletes from
+ * the front, or a queue its writes fill (migration 023); it deletes by subquery or key, never a list of bound values
+ * (a Durable Object binds at most 100); it removes from a queue only the generation it read; and a run stops taking
+ * new work once its time budget is spent, saying the rest as `backlog`. Rows from before migration 023 are walked into
+ * its tables a bounded batch per run, resumably, and each step waits for the walks it relies on. A failing step is
+ * logged and the others still run.
+ */
+export async function maintain(api,{budgetMs=MAINTAIN_BUDGET_MS}={}){const now=api.clock.nowMs(),deadline=Date.now()+budgetMs;
+  const failed=[];let more=false;const counts={};
+  const left=()=>Date.now()<deadline;
+  const q=(sql,params=[])=>api.store.query(sql,params),one=(sql,params=[])=>api.store.one(sql,params),tx=(statements)=>api.store.transaction(statements);
+  const step=async(name,work)=>{if(!left()){more=true;return;}try{await work();}catch(error){failed.push(name);console.error(`maintain ${name}: ${error.message}`);}};
+  const count=(name,n)=>{if(n>0)counts[name]=(counts[name]??0)+n;};
+  // Deletes go first, each in a transaction of its own: on a full database an insert fails, and a run that begins with
+  // one never reaches the deletes that would make room.
+  // Rows that expire in the order they were written go from the front: the first rows by rowid, while expired.
+  await step('expiries',async()=>{
+    for(const [table,column,cut] of [['idempotency','expires_at',now],['global_idempotency','expires_at',now],['journal','created_at',now-86400000],['app_install_codes','expires_at',now]]){
+      const front=await q(`SELECT rowid AS id,${column} AS at FROM ${table} ORDER BY rowid LIMIT ${EXPIRY_BATCH}`);
+      let through=null;for(const row of front){if(row.at>cut)break;through=row.id;}
+      if(through!=null){await tx([{sql:`DELETE FROM ${table} WHERE rowid<=?`,params:[through]}]);count(`expired ${table}`,front.findIndex(row=>row.id===through)+1);}
+      if(through!=null&&front.length>=EXPIRY_BATCH&&front.at(-1).id===through)more=true;
+    }
+    // cursors expire through their expiry index
+    for(const table of ['cursors','global_cursors']){
+      await tx([{sql:`DELETE FROM ${table} WHERE rowid IN (SELECT rowid FROM ${table} WHERE expires_at<=? ORDER BY expires_at LIMIT ${EXPIRY_BATCH})`,params:[now]}]);
+      if(await one(`SELECT 1 AS more FROM ${table} WHERE expires_at<=? LIMIT 1`,[now]))more=true;
+    }
+  });
+  // A capture or an upload never committed within a day expires, recorded as expired (they are few: only those in flight).
+  await step('in flight',async()=>{
+    for(const [kept,table] of [['staging_captures','captures'],['pending_uploads','uploads']]){
+      const rows=await q(`SELECT team_id,id,source_id FROM ${kept} WHERE created_at<=? ORDER BY created_at LIMIT ${IN_FLIGHT_BATCH}`,[now-86400000]);
+      for(const row of rows)await tx([
+        {sql:'INSERT OR IGNORE INTO capture_expiries(team_id,source_id,id,expired_at) VALUES (?,?,?,?)',params:[row.team_id,row.source_id,row.id,now]},
+        {sql:`DELETE FROM ${table} WHERE team_id=? AND id=?`,params:[row.team_id,row.id]},
+        {sql:`DELETE FROM ${kept} WHERE team_id=? AND id=?`,params:[row.team_id,row.id]},
+      ]);
+      count(`expired ${table}`,rows.length);if(rows.length>=IN_FLIGHT_BATCH)more=true;
+    }
+  });
+  // A committed capture's parts are its messages in transit; once committed, the session's version holds them.
+  await step('parts',async()=>{
+    for(const {team_id,capture_id,generation} of await q(`SELECT team_id,capture_id,generation FROM parts_prune_queue LIMIT ${PARTS_CAPTURE_BATCH}`)){
+      if(!left()){more=true;break;}
+      await tx([{sql:`DELETE FROM capture_parts WHERE rowid IN (SELECT rowid FROM capture_parts WHERE team_id=? AND capture_id=? LIMIT ${PART_PRUNE_BATCH})`,params:[team_id,capture_id]}]);
+      if(await one('SELECT 1 AS left FROM capture_parts WHERE team_id=? AND capture_id=? LIMIT 1',[team_id,capture_id])){more=true;continue;}
+      await tx([{sql:'DELETE FROM parts_prune_queue WHERE team_id=? AND capture_id=? AND generation=?',params:[team_id,capture_id,generation]}]);
+      count('captures cleared of parts',1);
+    }
+  });
+  // A committed upload nothing names (no artifact, continuation or in-flight capture: upload_refs) releases its bundle.
+  await step('uploads',async()=>{
+    if(!await walked(api,['artifact_refs','continuation_refs','captures','uploads'])){more=true;return;}
+    const queued=await q(`SELECT team_id,upload_id,generation FROM upload_prune_queue LIMIT ${UPLOAD_PRUNE_BATCH}`);
+    for(const {team_id,upload_id,generation} of queued){
+      if(!left()){more=true;break;}
+      const named=await one('SELECT 1 AS named FROM upload_refs WHERE team_id=? AND upload_id=? LIMIT 1',[team_id,upload_id]);
+      const statements=[];
+      if(!named){statements.push({sql:'DELETE FROM uploads WHERE team_id=? AND id=? AND committed=1 AND NOT EXISTS (SELECT 1 FROM upload_refs r WHERE r.team_id=? AND r.upload_id=?)',params:[team_id,upload_id,team_id,upload_id]});count('released uploads',1);}
+      statements.push({sql:'DELETE FROM upload_prune_queue WHERE team_id=? AND upload_id=? AND generation=?',params:[team_id,upload_id,generation]});
+      await tx(statements);
+    }
+    if(queued.length>=UPLOAD_PRUNE_BATCH)more=true;
+  });
+  // A blob no chunk names.
+  await step('blobs',async()=>{
+    if(!await walked(api,['blobs'])){more=true;return;}
+    const queued=await q(`SELECT team_id,sha256,generation FROM blob_prune_queue LIMIT ${BLOB_PRUNE_BATCH}`);
+    for(const {team_id,sha256,generation} of queued){
+      if(!left()){more=true;break;}
+      await tx([
+        {sql:'DELETE FROM artifact_blobs WHERE team_id=? AND sha256=? AND NOT EXISTS (SELECT 1 FROM artifact_chunk_refs r WHERE r.team_id=? AND r.sha256=?)',params:[team_id,sha256,team_id,sha256]},
+        {sql:'DELETE FROM blob_prune_queue WHERE team_id=? AND sha256=? AND generation=?',params:[team_id,sha256,generation]},
+      ]);
+    }
+    if(queued.length>=BLOB_PRUNE_BATCH)more=true;
+  });
+  // Every capture writes its session a new version (its messages are carried, not copied: schema 9). Kept: the
+  // session's current version, the newest version from each of its sources, the newest one from each source that a
+  // capture's artifact names (the next capture and a correlation read those), and every version a live cursor pins.
+  await step('versions',async()=>{
+    if(!await walked(api,['version_sources','cursor_pins'])){more=true;return;}
+    const due=await q(`SELECT team_id,session_id,generation FROM version_prune_queue WHERE due_at<=? ORDER BY due_at LIMIT ${VERSION_SESSION_BATCH}`,[now]);
+    for(const {team_id,session_id,generation} of due){
+      if(!left()){more=true;break;}
+      const session=await one('SELECT revision FROM sessions WHERE team_id=? AND id=?',[team_id,session_id]);
+      let budget=VERSION_PRUNE_BATCH,full=false;
+      for(let source=null;session&&budget>0;){
+        const next=source===null?await one('SELECT source_id FROM version_sources WHERE team_id=? AND session_id=? ORDER BY source_id LIMIT 1',[team_id,session_id])
+          :await one('SELECT source_id FROM version_sources WHERE team_id=? AND session_id=? AND source_id>? ORDER BY source_id LIMIT 1',[team_id,session_id,source]);
+        if(!next)break;source=next.source_id;
+        const newest=(await one('SELECT revision FROM version_sources WHERE team_id=? AND session_id=? AND source_id=? ORDER BY revision DESC LIMIT 1',[team_id,session_id,source])).revision;
+        // the newest version of this source a capture's artifact names, walked down from the newest (the versions it
+        // passes are this source's prunable ones); not found within the walk, versions a capture names are kept
+        let capture=-1;
+        for(let below=newest+1,walkedRows=0;capture<0&&walkedRows<CAPTURE_WALK;){
+          const rows=await q(`SELECT vs.revision,EXISTS (SELECT 1 FROM session_artifacts a WHERE a.team_id=vs.team_id AND a.session_id=vs.session_id AND a.revision=vs.revision AND a.provenance='capture') AS captured
+            FROM version_sources vs WHERE vs.team_id=? AND vs.session_id=? AND vs.source_id=? AND vs.revision<? ORDER BY vs.revision DESC LIMIT 100`,[team_id,session_id,source,below]);
+          if(!rows.length)break;
+          const hit=rows.find(row=>row.captured);if(hit)capture=hit.revision;
+          below=rows.at(-1).revision;walkedRows+=rows.length;
+        }
+        const free=await q(`SELECT vs.revision FROM version_sources vs WHERE vs.team_id=? AND vs.session_id=? AND vs.source_id=? AND vs.revision<? AND vs.revision<>?
+          AND NOT EXISTS (SELECT 1 FROM cursor_pins p WHERE p.team_id=vs.team_id AND p.session_id=vs.session_id AND p.revision=vs.revision AND p.expires_at>?)
+          AND (vs.revision<? OR NOT EXISTS (SELECT 1 FROM session_artifacts a WHERE a.team_id=vs.team_id AND a.session_id=vs.session_id AND a.revision=vs.revision AND a.provenance='capture'))
+          ORDER BY vs.revision LIMIT ${budget}`,[team_id,session_id,source,newest,session.revision,now,capture]);
+        // A message row lives from the version that wrote it until one that changed it (schema 9), so a row a later
+        // version still holds moves to the next version that holds it before its own goes; the rest go with the version.
+        for(const {revision} of free)await tx([
+          {sql:'UPDATE messages SET revision=(SELECT MIN(k.revision) FROM session_versions k WHERE k.team_id=messages.team_id AND k.session_id=messages.session_id AND k.revision>messages.revision AND k.revision<COALESCE(messages.until_revision,messages.revision+1)) WHERE team_id=? AND session_id=? AND revision=? AND EXISTS (SELECT 1 FROM session_versions k WHERE k.team_id=messages.team_id AND k.session_id=messages.session_id AND k.revision>messages.revision AND k.revision<COALESCE(messages.until_revision,messages.revision+1))',params:[team_id,session_id,revision]},
+          ...['messages','session_artifacts','session_versions'].map(table=>({sql:`DELETE FROM ${table} WHERE team_id=? AND session_id=? AND revision=?`,params:[team_id,session_id,revision]})),
+        ]);
+        count('versions',free.length);budget-=free.length;if(budget<=0)full=true;
+      }
+      // more of this session: due now; else due when its first live pin expires (what it pins may then go); else done
+      const pin=full?null:await one('SELECT MIN(expires_at) AS at FROM cursor_pins WHERE team_id=? AND session_id=? AND expires_at>?',[team_id,session_id,now]);
+      const at=full?now:pin?.at??null;
+      if(full)more=true;
+      await tx([at==null
+        ?{sql:'DELETE FROM version_prune_queue WHERE team_id=? AND session_id=? AND generation=?',params:[team_id,session_id,generation]}
+        :{sql:'UPDATE version_prune_queue SET due_at=? WHERE team_id=? AND session_id=? AND generation=?',params:[at,team_id,session_id,generation]}]);
+    }
+    if(due.length>=VERSION_SESSION_BATCH)more=true;
+  });
+  // Events older than a week go, except each resource's current event (its replay). Retention walks forward by
+  // sequence from where it stopped: `pruned_through` over events older than the cut, the deletes behind it from
+  // `deleted_through`, each run examining a bounded window.
+  await step('events',async()=>{
+    const cut=now-7*86400000;
+    for(const {team_id} of await q('SELECT id AS team_id FROM teams')){
+      if(!left()){more=true;break;}
+      const kept=await one('SELECT pruned_through,deleted_through FROM event_retention WHERE team_id=?',[team_id]);
+      const from=kept?.pruned_through??0;
+      const ahead=await q(`SELECT sequence,created_at FROM org_events WHERE team_id=? AND sequence>? ORDER BY sequence LIMIT ${EVENT_BATCH}`,[team_id,from]);
+      let through=from;for(const row of ahead){if(row.created_at>=cut)break;through=row.sequence;}
+      if(ahead.length>=EVENT_BATCH&&ahead.at(-1).sequence===through)more=true;
+      const deletedFrom=kept?.deleted_through??0;
+      const window=await q(`SELECT sequence FROM org_events WHERE team_id=? AND sequence>? AND sequence<=? ORDER BY sequence LIMIT ${EVENT_BATCH}`,[team_id,deletedFrom,through]);
+      const deletedThrough=window.length>=EVENT_BATCH?window.at(-1).sequence:through;
+      if(window.length>=EVENT_BATCH)more=true;
+      await tx([
+        {sql:`DELETE FROM org_events WHERE team_id=? AND sequence>? AND sequence<=? AND NOT EXISTS (SELECT 1 FROM event_resources r WHERE r.team_id=org_events.team_id AND r.source_id=COALESCE(org_events.source_id,'') AND r.resource_kind=org_events.resource_kind AND r.resource_id=org_events.resource_id AND r.sequence=org_events.sequence)`,params:[team_id,deletedFrom,deletedThrough]},
+        {sql:'INSERT INTO event_retention(team_id,pruned_through,deleted_through) VALUES (?,?,?) ON CONFLICT(team_id) DO UPDATE SET pruned_through=MAX(pruned_through,excluded.pruned_through),deleted_through=MAX(deleted_through,excluded.deleted_through)',params:[team_id,through,deletedThrough]},
+      ]);
+    }
+  });
+  // Presence reconciliation can INSERT. Never put it ahead of the deletes that make room when the database is full.
+  await step('presence',()=>reconcileMachinePresence(api));
+  await step('webhooks',async()=>{if(await deliverAppWebhooks(api))more=true;});
+  // Last, with what the budget leaves: the walks of rows written before migration 023 into its tables.
+  await step('backfill',async()=>{if(!await backfill(api,{now,left}))more=true;});
+  const result={counts,failed,backlog:more};
+  if(Object.keys(counts).length)console.log(`maintain: removed ${Object.entries(counts).map(([name,n])=>`${n} ${name}`).join(', ')}${more?' (backlog)':''}`);
   return result;}
-/** Session versions `maintain` removes per run: small enough for one Durable Object alarm; a backlog runs again in seconds. */
-const VERSION_PRUNE_BATCH=20;
-/** Committed capture parts `maintain` removes per run. */
-const PART_PRUNE_BATCH=50;
+
+/** Whether each named walk of migration 023's backfill is done. */
+async function walked(api,names){
+  for(const name of names){const row=await api.store.one('SELECT done FROM maintain_backfill WHERE name=?',[name]);if(!row?.done)return false;}
+  return true;
+}
+/** The walks of rows written before migration 023 into its tables: each a bounded rowid range per batch, in SQL (no
+ *  row comes into memory), resumed from where it stopped. Answers whether every walk is done. */
+const BACKFILLS=[
+  ['version_sources','session_versions',[
+    "INSERT OR IGNORE INTO version_sources SELECT team_id,session_id,COALESCE(json_extract(source_projection_json,'$.source_id'),''),revision FROM session_versions WHERE rowid>? AND rowid<=?",
+    'INSERT INTO version_prune_queue(team_id,session_id,due_at,generation) SELECT DISTINCT team_id,session_id,0,1 FROM session_versions WHERE rowid>? AND rowid<=? ON CONFLICT(team_id,session_id) DO UPDATE SET due_at=0,generation=generation+1']],
+  ['cursor_pins','cursors',[
+    "INSERT INTO cursor_pins SELECT c.id,c.team_id,json_extract(j.value,'$[0]'),json_extract(j.value,'$[1]'),c.expires_at FROM cursors c, json_each(c.snapshot_json) j WHERE c.rowid>? AND c.rowid<=? AND c.expires_at>NOWMS AND c.kind IN ('list','search') AND c.snapshot_json IS NOT NULL AND json_extract(j.value,'$[0]') IS NOT NULL AND json_extract(j.value,'$[1]') IS NOT NULL AND NOT EXISTS (SELECT 1 FROM cursor_pins p WHERE p.cursor_id=c.id)",
+    "INSERT INTO cursor_pins SELECT c.id,c.team_id,json_extract(c.snapshot_json,'$.session_id'),json_extract(c.snapshot_json,'$.revision'),c.expires_at FROM cursors c WHERE c.rowid>? AND c.rowid<=? AND c.expires_at>NOWMS AND c.kind='history' AND json_extract(c.snapshot_json,'$.session_id') IS NOT NULL AND json_extract(c.snapshot_json,'$.revision') IS NOT NULL AND NOT EXISTS (SELECT 1 FROM cursor_pins p WHERE p.cursor_id=c.id)"]],
+  ['artifact_refs','session_artifacts',[
+    "INSERT OR IGNORE INTO upload_refs SELECT team_id,upload_id,'a:'||session_id||':'||revision||':'||bundle_id FROM session_artifacts WHERE rowid>? AND rowid<=?"]],
+  ['continuation_refs','continuations',[
+    "INSERT OR IGNORE INTO upload_refs SELECT c.team_id,substr(t.value,instr(t.value,'upload_'),39),'c:'||c.id FROM continuations c, json_tree(c.value_json) t WHERE c.rowid>? AND c.rowid<=? AND t.type='text' AND instr(t.value,'upload_')>0"]],
+  ['captures','captures',[
+    "INSERT OR IGNORE INTO staging_captures SELECT team_id,id,source_id,created_at FROM captures WHERE rowid>? AND rowid<=? AND state='staging'",
+    "INSERT OR IGNORE INTO upload_refs SELECT c.team_id,substr(t.value,instr(t.value,'upload_'),39),'p:'||c.id FROM captures c, json_tree(c.begin_json) t WHERE c.rowid>? AND c.rowid<=? AND c.state='staging' AND t.type='text' AND instr(t.value,'upload_')>0",
+    "INSERT INTO parts_prune_queue SELECT c.team_id,c.id,1 FROM captures c WHERE c.rowid>? AND c.rowid<=? AND c.state='committed' AND EXISTS (SELECT 1 FROM capture_parts p WHERE p.team_id=c.team_id AND p.capture_id=c.id) ON CONFLICT(team_id,capture_id) DO UPDATE SET generation=generation+1"]],
+  ['uploads','uploads',[
+    'INSERT INTO upload_prune_queue SELECT team_id,id,1 FROM uploads WHERE rowid>? AND rowid<=? AND committed=1 ON CONFLICT(team_id,upload_id) DO UPDATE SET generation=generation+1',
+    'INSERT OR IGNORE INTO pending_uploads SELECT team_id,id,source_id,created_at FROM uploads WHERE rowid>? AND rowid<=? AND committed=0']],
+  ['blobs','artifact_blobs',[
+    'INSERT INTO blob_prune_queue SELECT team_id,sha256,1 FROM artifact_blobs WHERE rowid>? AND rowid<=? ON CONFLICT(team_id,sha256) DO UPDATE SET generation=generation+1']],
+];
+async function backfill(api,{now,left}){
+  let done=true;
+  for(const [name,table,statements] of BACKFILLS){
+    const progress=await api.store.one('SELECT after_rowid,done FROM maintain_backfill WHERE name=?',[name]);
+    if(progress?.done)continue;
+    let after=progress?.after_rowid??0,finished=false;
+    while(left()){
+      const last=await api.store.one(`SELECT MAX(rowid) AS r,COUNT(*) AS n FROM (SELECT rowid FROM ${table} WHERE rowid>? ORDER BY rowid LIMIT ${BACKFILL_BATCH})`,[after]);
+      finished=!last?.n||last.n<BACKFILL_BATCH;
+      const through=last?.r??after;
+      await api.store.transaction([
+        ...(last?.n?statements.map(sql=>({sql:sql.replaceAll('NOWMS',String(Math.floor(now))),params:[after,through]})):[]),
+        {sql:'INSERT INTO maintain_backfill(name,after_rowid,done) VALUES (?,?,?) ON CONFLICT(name) DO UPDATE SET after_rowid=excluded.after_rowid,done=excluded.done',params:[name,through,finished?1:0]},
+      ]);
+      after=through;
+      if(finished)break;
+    }
+    if(!finished)done=false;
+  }
+  return done;
+}
+/** How long the object waits before its next `maintain`: seconds while a backlog remains, else a minute; and never past
+ *  the end of the link grace after it started, so a machine whose link did not dial back is said unlinked on time. */
+export function maintainDelay(api,result,now=api.clock.nowMs()){
+  const base=result?.backlog?3_000:MAINTAIN_MS;
+  const graceEnds=(api.activatedAt??0)+LINK_GRACE_MS+1_000;
+  return graceEnds>now?Math.min(base,graceEnds-now):base;
+}
+/** A minute between runs; a run's own time budget (it stops taking new work after it, saying the rest as backlog). */
+const MAINTAIN_MS=60_000,MAINTAIN_BUDGET_MS=250;
+/** Sessions whose versions a run reads, and versions it removes from each; how far it walks down a source to find the
+ *  newest version a capture names. */
+const VERSION_SESSION_BATCH=10,VERSION_PRUNE_BATCH=20,CAPTURE_WALK=500;
+/** Committed captures whose parts a run removes, and parts per capture per run. */
+const PARTS_CAPTURE_BATCH=20,PART_PRUNE_BATCH=50;
+/** Rows of one expiry a run removes; captures and uploads in flight a run expires. */
+const EXPIRY_BATCH=500,IN_FLIGHT_BATCH=50;
+/** Queued uploads and blobs a run decides. */
+const UPLOAD_PRUNE_BATCH=20,BLOB_PRUNE_BATCH=50;
+/** Events per team retention walks per run; rows per backfill batch. */
+const EVENT_BATCH=1000,BACKFILL_BATCH=500;

package/server/http.mjs

@@ -336,10 +336,10 @@ async function groupRoute(api,req,res,auth,groupId,principalId,url,requestId){
   else{const group=await api.store.one('SELECT revision FROM groups_table WHERE team_id=? AND id=?',[auth.team_id,groupId]);if(!group)fail(404,'not_found','group not found');requireRevision(req,group.revision,'group');guards.push({sql:'SELECT 1 ok FROM groups_table WHERE team_id=? AND id=? AND revision=?',params:[auth.team_id,groupId,group.revision],code:'revision_mismatch',status:409,message:'group revision changed'});statements.push({sql:'DELETE FROM session_grants WHERE team_id=? AND subject_kind=\'group\' AND subject_id=?',params:[auth.team_id,groupId]},{sql:'DELETE FROM group_members WHERE team_id=? AND group_id=?',params:[auth.team_id,groupId]},{sql:'DELETE FROM groups_table WHERE team_id=? AND id=?',params:[auth.team_id,groupId]});}
   statements.push({sql:'UPDATE teams SET auth_epoch=auth_epoch+1 WHERE id=?',params:[auth.team_id]});await guardedMutation(api,res,auth,guards,statements,{status,body:response,targetId:target});api.events.publish(auth.team_id,{type:'authorization_changed',group_id:groupId});if(req.method==='DELETE')closeChannelsFor(api,auth.team_id,{groupId,principalId:principalId??null,reason:'group_changed'});send(res,status,response,requestId);
 }
-async function enroll(api,req,res,auth,requestId){if(auth.role==='viewer')fail(403,'forbidden','viewers cannot enroll machines');const b=await parseJson(req);exact(b,['name','enrollment_key']);if(!b.name||!b.enrollment_key)fail(400,'invalid_request','name and enrollment_key required');const byKey=()=>api.store.one('SELECT m.*,s.id source_id,s.name source_name,s.kind source_kind,s.state source_state,s.visibility,s.revision source_revision,s.last_received_at,s.decoder,s.counters_json,s.last_error FROM machines m JOIN sources s ON s.team_id=m.team_id AND s.machine_id=m.id WHERE m.team_id=? AND m.custodian_id=? AND m.enrollment_key=?',[auth.team_id,auth.principal_id,b.enrollment_key]);let machine=await byKey();
+async function enroll(api,req,res,auth,requestId){if(auth.role==='viewer')fail(403,'forbidden','viewers cannot enroll machines');const b=await parseJson(req);exact(b,['name','enrollment_key']);if(!b.name||!b.enrollment_key)fail(400,'invalid_request','name and enrollment_key required');const byKey=()=>api.store.one('SELECT m.*,(SELECT linked FROM machine_presence p WHERE p.team_id=m.team_id AND p.machine_id=m.id) presence_linked,s.id source_id,s.name source_name,s.kind source_kind,s.state source_state,s.visibility,s.revision source_revision,s.last_received_at,s.decoder,s.counters_json,s.last_error FROM machines m JOIN sources s ON s.team_id=m.team_id AND s.machine_id=m.id WHERE m.team_id=? AND m.custodian_id=? AND m.enrollment_key=?',[auth.team_id,auth.principal_id,b.enrollment_key]);let machine=await byKey();
   // Enrollment is a fresh, authenticated custodial act. A disconnect revoked the
   // old credentials, but must not permanently reserve this custodian's name.
-  if(!machine) machine=await api.store.one('SELECT m.*,s.id source_id,s.name source_name,s.kind source_kind,s.state source_state,s.visibility,s.revision source_revision,s.last_received_at,s.decoder,s.counters_json,s.last_error FROM machines m JOIN sources s ON s.team_id=m.team_id AND s.machine_id=m.id WHERE m.team_id=? AND m.custodian_id=? AND m.name=?',[auth.team_id,auth.principal_id,b.name]);
+  if(!machine) machine=await api.store.one('SELECT m.*,(SELECT linked FROM machine_presence p WHERE p.team_id=m.team_id AND p.machine_id=m.id) presence_linked,s.id source_id,s.name source_name,s.kind source_kind,s.state source_state,s.visibility,s.revision source_revision,s.last_received_at,s.decoder,s.counters_json,s.last_error FROM machines m JOIN sources s ON s.team_id=m.team_id AND s.machine_id=m.id WHERE m.team_id=? AND m.custodian_id=? AND m.name=?',[auth.team_id,auth.principal_id,b.name]);
   if(machine){
     const now=api.clock.nowMs(),reviving=machine.revoked_at!=null;
     const changed=reviving||machine.enrollment_key!==b.enrollment_key;
@@ -363,20 +363,20 @@ async function enroll(api,req,res,auth,requestId){if(auth.role==='viewer')fail(4
   }
   const mid=api.ids.next('machine'),sid=api.ids.next('source'),cid=api.ids.next('credential'),token=makeToken(cid),now=api.clock.nowMs(),data={machine:{id:mid,name:b.name,custodian_id:auth.principal_id,enrollment:'active',connection:'offline',last_seen_at:null,incarnation_id:null,revision:1},source:{id:sid,name:`${b.name} connector`,kind:'connector',state:'active',machine_id:mid,custodian_id:auth.principal_id,default_visibility:'restricted',revision:1,last_received_at:null,decoder:null},credential:{token,kind:'machine',expires_at:now+2592000000}},response=result(api,auth,data);await guardedMutation(api,res,auth,[...authGuards(auth,now),{sql:'SELECT 1 ok WHERE NOT EXISTS (SELECT 1 FROM sources WHERE team_id=? AND name=?)',params:[auth.team_id,`${b.name} connector`],code:'conflict',status:409,message:`a machine named ${b.name} is already enrolled in this team; enroll this one under another --name`}],[{sql:'INSERT INTO machines(team_id,id,name,custodian_id,enrollment_key) VALUES (?,?,?,?,?)',params:[auth.team_id,mid,b.name,auth.principal_id,b.enrollment_key]},{sql:"INSERT INTO sources(team_id,id,name,kind,machine_id,custodian_id,visibility,state) VALUES (?,?,?,?,?,?,?,'active')",params:[auth.team_id,sid,`${b.name} connector`,'connector',mid,auth.principal_id,'restricted']},{sql:'INSERT INTO credentials(id,token_hash,principal_id,kind,team_id,source_id,machine_id,expires_at,created_at) VALUES (?,?,?,?,?,?,?,?,?)',params:[cid,hashToken(token),auth.principal_id,'machine',auth.team_id,sid,mid,now+2592000000,now]}],{status:201,body:response,targetId:mid});send(res,201,response,requestId);}
 // `online` is a machine that heartbeats and holds its link, so callers can reach it; `unlinked` heartbeats without one.
-const machineJson=(m,now,api)=>({id:m.id,name:m.name,custodian_id:m.custodian_id,enrollment:m.revoked_at==null?'active':'revoked',connection:m.last_seen_at!=null&&m.last_seen_at+45000>now?(!api||api.machineLinks?.has(`${m.team_id}\0${m.id}`)?'online':'unlinked'):'offline',last_seen_at:m.last_seen_at,incarnation_id:m.incarnation_id,host_key_id:m.host_key_id??null,revision:m.revision});
+const machineJson=(m,now,api)=>({id:m.id,name:m.name,custodian_id:m.custodian_id,enrollment:m.revoked_at==null?'active':'revoked',connection:m.last_seen_at!=null&&m.last_seen_at+45000>now?(m.presence_linked===1?'online':'unlinked'):'offline',last_seen_at:m.last_seen_at,incarnation_id:m.incarnation_id,host_key_id:m.host_key_id??null,revision:m.revision});
 async function machineAdminRoute(api,req,res,auth,machineId,roots,requestId){
   // Custodians and admins see their machines, revoked ones included; every member sees the team's active machines,
   // each of which they can message (the team's standing mail grant, machine-link.mjs). Roots stay custodial.
   const custody=roots?'(m.custodian_id=? OR ?=1)':'(m.custodian_id=? OR ?=1 OR m.revoked_at IS NULL)',params=[auth.team_id,auth.principal_id,isAdmin(auth)?1:0];
-  if(machineId){const machine=await api.store.one(`SELECT m.* FROM machines m WHERE m.team_id=? AND ${custody} AND m.id=?`,[...params,machineId]);if(!machine)fail(404,'not_found','not found');if(roots){const rows=await api.store.query('SELECT value_json FROM machine_roots WHERE team_id=? AND machine_id=? ORDER BY id',[auth.team_id,machineId]);return send(res,200,{version:1,context:{server_id:api.serverId,team_id:auth.team_id,principal_id:auth.principal_id},items:rows.map(row=>JSON.parse(row.value_json)),next_cursor:null},requestId);}return send(res,200,result(api,auth,machineJson(machine,api.clock.nowMs(),api)),requestId,{ETag:`"${machine.revision}"`});}
-  const rows=await api.store.query(`SELECT m.* FROM machines m WHERE m.team_id=? AND ${custody} ORDER BY m.name,m.id`,params);send(res,200,{version:1,context:{server_id:api.serverId,team_id:auth.team_id,principal_id:auth.principal_id},items:rows.map(row=>machineJson(row,api.clock.nowMs(),api)),next_cursor:null},requestId);
+  if(machineId){const machine=await api.store.one(`SELECT m.*,(SELECT linked FROM machine_presence p WHERE p.team_id=m.team_id AND p.machine_id=m.id) presence_linked FROM machines m WHERE m.team_id=? AND ${custody} AND m.id=?`,[...params,machineId]);if(!machine)fail(404,'not_found','not found');if(roots){const rows=await api.store.query('SELECT value_json FROM machine_roots WHERE team_id=? AND machine_id=? ORDER BY id',[auth.team_id,machineId]);return send(res,200,{version:1,context:{server_id:api.serverId,team_id:auth.team_id,principal_id:auth.principal_id},items:rows.map(row=>JSON.parse(row.value_json)),next_cursor:null},requestId);}return send(res,200,result(api,auth,machineJson(machine,api.clock.nowMs(),api)),requestId,{ETag:`"${machine.revision}"`});}
+  const rows=await api.store.query(`SELECT m.*,(SELECT linked FROM machine_presence p WHERE p.team_id=m.team_id AND p.machine_id=m.id) presence_linked FROM machines m WHERE m.team_id=? AND ${custody} ORDER BY m.name,m.id`,params);send(res,200,{version:1,context:{server_id:api.serverId,team_id:auth.team_id,principal_id:auth.principal_id},items:rows.map(row=>machineJson(row,api.clock.nowMs(),api)),next_cursor:null},requestId);
 }
 async function revokeMachine(api,req,res,auth,machineId,requestId){
   const machine=await api.store.one('SELECT * FROM machines WHERE team_id=? AND id=?',[auth.team_id,machineId]);if(!machine)fail(404,'not_found','not found');if(machine.custodian_id!==auth.principal_id&&!isAdmin(auth))fail(403,'forbidden','forbidden');requireRevision(req,machine.revision,'machine');const now=api.clock.nowMs(),guards=[...authGuards(auth,now),{sql:'SELECT 1 ok FROM machines WHERE team_id=? AND id=? AND revision=? AND revoked_at IS NULL AND (custodian_id=? OR ?=1)',params:[auth.team_id,machineId,machine.revision,auth.principal_id,isAdmin(auth)?1:0],code:'revision_mismatch',status:409,message:'machine changed'}],statements=[{sql:'UPDATE machines SET revoked_at=?,revision=revision+1 WHERE team_id=? AND id=?',params:[now,auth.team_id,machineId]},{sql:'UPDATE credentials SET revoked_at=? WHERE team_id=? AND machine_id=? AND revoked_at IS NULL',params:[now,auth.team_id,machineId]},{sql:"UPDATE sources SET state='revoked',revision=revision+1 WHERE team_id=? AND machine_id=?",params:[auth.team_id,machineId]},{sql:"UPDATE sync_rules SET state='removed',revision=revision+1 WHERE team_id=? AND machine_id=? AND state!='removed'",params:[auth.team_id,machineId]},{sql:"UPDATE attachments SET state='revoked',closed_at=? WHERE team_id=? AND machine_id=? AND state IN ('pending','active')",params:[now,auth.team_id,machineId]},{sql:"UPDATE operations SET state='unknown' WHERE team_id=? AND id IN (SELECT operation_id FROM native_effects WHERE team_id=? AND machine_id=? AND state IN ('pending','running'))",params:[auth.team_id,auth.team_id,machineId]},{sql:"UPDATE native_effects SET state='unknown',result_json=? WHERE team_id=? AND machine_id=? AND state IN ('pending','running')",params:[JSON.stringify({state:'unknown',error:null}),auth.team_id,machineId]},{sql:'UPDATE teams SET auth_epoch=auth_epoch+1 WHERE id=?',params:[auth.team_id]}];await guardedMutation(api,res,auth,guards,statements,{status:204,body:null,targetId:machineId});api.events.publish(auth.team_id,{type:'authorization_changed',machine_id:machineId});closeMachineChannels(api,auth.team_id,machineId,{reason:'machine_revoked'});send(res,204,null,requestId);
 }
 /** This machine's sessions someone is watching live (an open follow stream), as {harness, native_id}. */
 async function watchedFor(api,teamId,machineId){const ids=[...(api.watchers?.keys()??[])].filter(key=>key.startsWith(`${teamId}:`)).map(key=>key.slice(teamId.length+1)).slice(0,90);if(!ids.length)return [];return (await api.store.query(`SELECT o.harness,o.native_id FROM origins o JOIN sources s ON s.team_id=o.team_id AND s.id=o.source_id WHERE o.team_id=? AND s.machine_id=? AND o.session_id IN (${ids.map(()=>'?').join(',')})`,[teamId,machineId,...ids])).map(({harness,native_id})=>({harness,native_id}));}
-async function heartbeat(api,req,res,auth,machineId,requestId){if(auth.kind!=='machine'||auth.machine_id!==machineId)fail(403,'forbidden','exact machine credential required');const machine=await api.store.one('SELECT * FROM machines WHERE team_id=? AND id=?',[auth.team_id,machineId]);if(!machine||machine.revoked_at!=null)fail(403,'forbidden','machine enrollment is not active');const b=await parseJson(req);exact(b,['incarnation_id','sequence','bindings','roots','host_key_id','activity']);if(b.activity!==undefined&&(!Array.isArray(b.activity)||b.activity.length>4096))fail(400,'invalid_request','invalid activity');if(b.host_key_id!==undefined&&(typeof b.host_key_id!=='string'||!/^[A-Za-z0-9_-]{16,128}$/.test(b.host_key_id)))fail(400,'invalid_request','invalid host_key_id');if(typeof b.incarnation_id!=='string'||!Number.isSafeInteger(b.sequence)||b.sequence<1||!Array.isArray(b.bindings)||!Array.isArray(b.roots))fail(400,'invalid_request','invalid heartbeat');if(machine.incarnation_id===b.incarnation_id&&b.sequence<=machine.heartbeat_sequence)return send(res,200,result(api,auth,{...machineJson(machine,api.clock.nowMs(),api),watched:await watchedFor(api,auth.team_id,machineId)}),requestId);const now=api.clock.nowMs(),newIncarnation=machine.incarnation_id!==null&&machine.incarnation_id!==b.incarnation_id,statements=[{sql:'UPDATE machines SET incarnation_id=?,heartbeat_sequence=?,last_seen_at=?,host_key_id=COALESCE(?,host_key_id),revision=revision+1 WHERE team_id=? AND id=?',params:[b.incarnation_id,b.sequence,now,b.host_key_id??null,auth.team_id,machineId]},{sql:'DELETE FROM machine_roots WHERE team_id=? AND machine_id=?',params:[auth.team_id,machineId]}];if(newIncarnation)statements.push({sql:"UPDATE attachments SET state='closed',closed_at=? WHERE team_id=? AND machine_id=? AND state IN ('pending','active')",params:[now,auth.team_id,machineId]},{sql:"UPDATE operations SET state='unknown' WHERE team_id=? AND id IN (SELECT operation_id FROM native_effects WHERE team_id=? AND machine_id=? AND state IN ('pending','running'))",params:[auth.team_id,auth.team_id,machineId]},{sql:"UPDATE native_effects SET state='unknown',result_json=? WHERE team_id=? AND machine_id=? AND state IN ('pending','running')",params:[JSON.stringify({state:'unknown',error:null}),auth.team_id,machineId]});statements.push({sql:'DELETE FROM bindings WHERE team_id=? AND machine_id=? AND id NOT IN (SELECT value FROM json_each(?)) AND id NOT IN (SELECT binding_id FROM attachments WHERE team_id=? AND machine_id=?) AND id NOT IN (SELECT binding_id FROM native_effects WHERE team_id=? AND machine_id=? AND binding_id IS NOT NULL)',params:[auth.team_id,machineId,JSON.stringify(b.bindings.map(binding=>binding.id)),auth.team_id,machineId,auth.team_id,machineId]});// What each synced session's harness is doing now: this heartbeat's report replaces the source's last one.
+async function heartbeat(api,req,res,auth,machineId,requestId){if(auth.kind!=='machine'||auth.machine_id!==machineId)fail(403,'forbidden','exact machine credential required');const machine=await api.store.one('SELECT m.*,(SELECT linked FROM machine_presence p WHERE p.team_id=m.team_id AND p.machine_id=m.id) presence_linked FROM machines m WHERE m.team_id=? AND m.id=?',[auth.team_id,machineId]);if(!machine||machine.revoked_at!=null)fail(403,'forbidden','machine enrollment is not active');const b=await parseJson(req);exact(b,['incarnation_id','sequence','bindings','roots','host_key_id','activity']);if(b.activity!==undefined&&(!Array.isArray(b.activity)||b.activity.length>4096))fail(400,'invalid_request','invalid activity');if(b.host_key_id!==undefined&&(typeof b.host_key_id!=='string'||!/^[A-Za-z0-9_-]{16,128}$/.test(b.host_key_id)))fail(400,'invalid_request','invalid host_key_id');if(typeof b.incarnation_id!=='string'||!Number.isSafeInteger(b.sequence)||b.sequence<1||!Array.isArray(b.bindings)||!Array.isArray(b.roots))fail(400,'invalid_request','invalid heartbeat');if(machine.incarnation_id===b.incarnation_id&&b.sequence<=machine.heartbeat_sequence)return send(res,200,result(api,auth,{...machineJson(machine,api.clock.nowMs(),api),watched:await watchedFor(api,auth.team_id,machineId)}),requestId);const now=api.clock.nowMs(),newIncarnation=machine.incarnation_id!==null&&machine.incarnation_id!==b.incarnation_id,statements=[{sql:'UPDATE machines SET incarnation_id=?,heartbeat_sequence=?,last_seen_at=?,host_key_id=COALESCE(?,host_key_id),revision=revision+1 WHERE team_id=? AND id=?',params:[b.incarnation_id,b.sequence,now,b.host_key_id??null,auth.team_id,machineId]},{sql:'DELETE FROM machine_roots WHERE team_id=? AND machine_id=?',params:[auth.team_id,machineId]}];if(newIncarnation)statements.push({sql:"UPDATE attachments SET state='closed',closed_at=? WHERE team_id=? AND machine_id=? AND state IN ('pending','active')",params:[now,auth.team_id,machineId]},{sql:"UPDATE operations SET state='unknown' WHERE team_id=? AND id IN (SELECT operation_id FROM native_effects WHERE team_id=? AND machine_id=? AND state IN ('pending','running'))",params:[auth.team_id,auth.team_id,machineId]},{sql:"UPDATE native_effects SET state='unknown',result_json=? WHERE team_id=? AND machine_id=? AND state IN ('pending','running')",params:[JSON.stringify({state:'unknown',error:null}),auth.team_id,machineId]});statements.push({sql:'DELETE FROM bindings WHERE team_id=? AND machine_id=? AND id NOT IN (SELECT value FROM json_each(?)) AND id NOT IN (SELECT binding_id FROM attachments WHERE team_id=? AND machine_id=?) AND id NOT IN (SELECT binding_id FROM native_effects WHERE team_id=? AND machine_id=? AND binding_id IS NOT NULL)',params:[auth.team_id,machineId,JSON.stringify(b.bindings.map(binding=>binding.id)),auth.team_id,machineId,auth.team_id,machineId]});// What each synced session's harness is doing now: this heartbeat's report replaces the source's last one.
   if(Array.isArray(b.activity)){const source=await api.store.one("SELECT id FROM sources WHERE team_id=? AND machine_id=? AND kind='connector'",[auth.team_id,machineId]);if(source){statements.push({sql:'DELETE FROM session_activity WHERE team_id=? AND source_id=?',params:[auth.team_id,source.id]});for(const item of b.activity){exact(item,['harness','native_id','turn','since'],'activity');if(typeof item.harness!=='string'||!item.harness||item.harness.length>64||typeof item.native_id!=='string'||!item.native_id||item.native_id.length>256||typeof item.turn!=='string'||!item.turn||item.turn.length>32||(item.since!=null&&!Number.isSafeInteger(item.since)))fail(400,'invalid_request','invalid activity');statements.push({sql:'INSERT OR REPLACE INTO session_activity(team_id,source_id,harness,native_id,turn,since,observed_at) VALUES (?,?,?,?,?,?,?)',params:[auth.team_id,source.id,item.harness,item.native_id,item.turn,item.since??null,now]});}}}
   for(const root of b.roots){exact(root,['id','machine_id','name','harnesses'],'root');if(root.machine_id!==machineId)fail(400,'invalid_request','root machine mismatch');statements.push({sql:'INSERT INTO machine_roots(team_id,machine_id,id,value_json) VALUES (?,?,?,?)',params:[auth.team_id,machineId,root.id,JSON.stringify(root)]});}// A binding is stored against the session its native id names; one naming no session this source published is
   // said back (unbound_bindings), never dropped in silence.
@@ -385,7 +385,7 @@ async function heartbeat(api,req,res,auth,machineId,requestId){if(auth.kind!=='m
 async function machineConfiguration(api,res,auth,machineId,requestId){
   if(auth.kind!=='machine'||auth.machine_id!==machineId)fail(auth.kind==='machine'?404:403,auth.kind==='machine'?'not_found':'forbidden','not found');
   const [machine,rules]=await api.store.snapshot([
-    {kind:'one',sql:'SELECT m.*,s.id source_id,s.name source_name,s.kind source_kind,s.state source_state,s.visibility,s.revision source_revision,s.last_received_at,s.decoder,s.counters_json,s.last_error,t.auth_epoch FROM machines m JOIN sources s ON s.team_id=m.team_id AND s.machine_id=m.id JOIN teams t ON t.id=m.team_id JOIN memberships mem ON mem.team_id=m.team_id AND mem.principal_id=m.custodian_id AND mem.active=1 WHERE m.team_id=? AND m.id=? AND m.revoked_at IS NULL',params:[auth.team_id,machineId]},
+    {kind:'one',sql:'SELECT m.*,(SELECT linked FROM machine_presence p WHERE p.team_id=m.team_id AND p.machine_id=m.id) presence_linked,s.id source_id,s.name source_name,s.kind source_kind,s.state source_state,s.visibility,s.revision source_revision,s.last_received_at,s.decoder,s.counters_json,s.last_error,t.auth_epoch FROM machines m JOIN sources s ON s.team_id=m.team_id AND s.machine_id=m.id JOIN teams t ON t.id=m.team_id JOIN memberships mem ON mem.team_id=m.team_id AND mem.principal_id=m.custodian_id AND mem.active=1 WHERE m.team_id=? AND m.id=? AND m.revoked_at IS NULL',params:[auth.team_id,machineId]},
     {kind:'query',sql:'SELECT r.* FROM sync_rules r JOIN machines m ON m.team_id=r.team_id AND m.id=r.machine_id JOIN sources s ON s.team_id=r.team_id AND s.id=r.source_id WHERE r.team_id=? AND r.machine_id=? AND m.revoked_at IS NULL AND s.machine_id=m.id ORDER BY r.id',params:[auth.team_id,machineId]},
   ]);
   if(!machine)fail(404,'not_found','not found');

package/server/machine-events.mjs

@@ -22,16 +22,38 @@ export function machineTransition(api, team, machine, linked) {
   return next;
 }
 
+/** How long after this object starts a machine's link has to dial back before the store says it unlinked: a reset of the
+ *  object drops every link, and each connector re-dials within its backoff's most (30 s); 15 s more is its grace. */
+export const LINK_GRACE_MS = 45_000;
+/** Presence events retention walks per team per run. */
+const PRESENCE_BATCH = 1000;
+
+/**
+ * The store's presence follows the links this object holds: each link it holds is linked, and a machine the store says
+ * linked whose link is not held is unlinked, once the grace after this object started has passed (until then its link
+ * may be dialling back). Presence is read from the store only (`machine_presence`), by the machine list and the
+ * machine event stream alike, so the two never disagree and no reset reads every machine as unlinked (t_6c01c966).
+ * Week-old presence events go, walking forward by sequence from where the last run stopped.
+ */
 export async function reconcileMachinePresence(api) {
   await api.machineTransitionWork;
-  for(const id of api.machineLinks?.keys()??[]){const [team,machine]=id.split('\0');await machineTransition(api,team,machine,null);}
-  const rows = await api.store.query('SELECT team_id,machine_id FROM machine_presence WHERE linked=1');
-  for (const row of rows) if (!api.machineLinks?.has(`${row.team_id}\0${row.machine_id}`)) await machineTransition(api,row.team_id,row.machine_id,null);
+  for(const id of api.machineLinks?.keys()??[]){const [team,machine]=id.split('\0');await machineTransition(api,team,machine,true);}
+  if (api.clock.nowMs() - (api.activatedAt ?? 0) >= LINK_GRACE_MS) {
+    const rows = await api.store.query('SELECT team_id,machine_id FROM machine_presence WHERE linked=1');
+    for (const row of rows) if (!api.machineLinks?.has(`${row.team_id}\0${row.machine_id}`)) await machineTransition(api,row.team_id,row.machine_id,false);
+  }
   const before=api.clock.nowMs()-7*86400000;
-  await api.store.transaction([
-    {sql:'INSERT INTO machine_presence_cursors SELECT team_id,MAX(sequence) FROM machine_presence_events WHERE created_at<? GROUP BY team_id ON CONFLICT(team_id) DO UPDATE SET pruned_through=MAX(pruned_through,excluded.pruned_through)',params:[before]},
-    {sql:'DELETE FROM machine_presence_events WHERE created_at<?',params:[before]},
-  ]);
+  for (const { team_id } of await api.store.query('SELECT id AS team_id FROM teams')) {
+    const kept = await api.store.one('SELECT pruned_through,deleted_through FROM machine_presence_cursors WHERE team_id=?', [team_id]);
+    const from = kept?.pruned_through ?? 0;
+    const ahead = await api.store.query(`SELECT sequence,created_at FROM machine_presence_events WHERE team_id=? AND sequence>? ORDER BY sequence LIMIT ${PRESENCE_BATCH}`, [team_id, from]);
+    let through = from; for (const row of ahead) { if (row.created_at >= before) break; through = row.sequence; }
+    if (through === from && kept) continue;
+    await api.store.transaction([
+      {sql:'DELETE FROM machine_presence_events WHERE team_id=? AND sequence>? AND sequence<=?',params:[team_id,kept?.deleted_through??0,through]},
+      {sql:'INSERT INTO machine_presence_cursors(team_id,pruned_through,deleted_through) VALUES (?,?,?) ON CONFLICT(team_id) DO UPDATE SET pruned_through=MAX(pruned_through,excluded.pruned_through),deleted_through=MAX(deleted_through,excluded.deleted_through)',params:[team_id,through,through]},
+    ]);
+  }
 }
 
 export async function machineEventStream({api,req,res,auth,url,requestId,fail}) {

package/server/migrations/023.mjs

@@ -0,0 +1,126 @@
+// `maintain` runs on the object's alarm every minute, so none of its steps may cost the store: each costs what it
+// removes (t_6c01c966: the one Durable Object exceeded its memory about once a minute, and each reset dropped every
+// machine's link). This migration runs at activation, before the object serves, so it is constant cost: it adds two
+// columns, new tables and triggers, and builds no index over an existing table and copies no existing row.
+// - What can only be decided across tables (a version no one holds, an upload nothing names, a blob no chunk names, a
+//   committed capture's parts) is queued by the write that may make it so. A queue row carries a generation each write
+//   advances, and `maintain` removes only the generation it read (a write between its read and its removal stays).
+// - What `maintain` would otherwise find by searching (each version's source, an upload's references, a version a live
+//   cursor holds, the captures and uploads still in flight) is kept as rows by the writes that make it so.
+// - Rows written before this migration are walked into those tables by `maintain` itself, a bounded batch per run,
+//   resumably (`maintain_backfill`); each consumer waits until the walks it relies on are done.
+export default `
+ALTER TABLE event_retention ADD COLUMN deleted_through INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE machine_presence_cursors ADD COLUMN deleted_through INTEGER NOT NULL DEFAULT 0;
+
+CREATE TABLE maintain_backfill (name TEXT PRIMARY KEY, after_rowid INTEGER NOT NULL DEFAULT 0, done INTEGER NOT NULL DEFAULT 0) STRICT;
+
+-- each version's source (its projection's source_id), and the versions a live list, search or history cursor holds
+CREATE TABLE version_sources (team_id TEXT NOT NULL, session_id TEXT NOT NULL, source_id TEXT NOT NULL, revision INTEGER NOT NULL, PRIMARY KEY(team_id, session_id, source_id, revision)) STRICT, WITHOUT ROWID;
+CREATE TRIGGER version_sources_insert AFTER INSERT ON session_versions BEGIN
+  INSERT OR IGNORE INTO version_sources VALUES (NEW.team_id, NEW.session_id, COALESCE(json_extract(NEW.source_projection_json,'$.source_id'),''), NEW.revision);
+  INSERT INTO version_prune_queue(team_id,session_id,due_at,generation) VALUES (NEW.team_id,NEW.session_id,0,1)
+    ON CONFLICT(team_id,session_id) DO UPDATE SET due_at=0, generation=generation+1;
+END;
+CREATE TRIGGER version_sources_delete AFTER DELETE ON session_versions BEGIN
+  DELETE FROM version_sources WHERE team_id=OLD.team_id AND session_id=OLD.session_id AND source_id=COALESCE(json_extract(OLD.source_projection_json,'$.source_id'),'') AND revision=OLD.revision;
+END;
+CREATE TABLE cursor_pins (cursor_id TEXT NOT NULL, team_id TEXT NOT NULL, session_id TEXT NOT NULL, revision INTEGER NOT NULL, expires_at INTEGER NOT NULL) STRICT;
+CREATE INDEX cursor_pins_version ON cursor_pins(team_id, session_id, revision, expires_at);
+CREATE INDEX cursor_pins_session ON cursor_pins(team_id, session_id, expires_at);
+CREATE INDEX cursor_pins_cursor ON cursor_pins(cursor_id);
+CREATE TRIGGER cursor_pins_insert AFTER INSERT ON cursors WHEN NEW.snapshot_json IS NOT NULL BEGIN
+  INSERT INTO cursor_pins SELECT NEW.id,NEW.team_id,json_extract(j.value,'$[0]'),json_extract(j.value,'$[1]'),NEW.expires_at
+    FROM json_each(NEW.snapshot_json) j WHERE NEW.kind IN ('list','search') AND json_extract(j.value,'$[0]') IS NOT NULL AND json_extract(j.value,'$[1]') IS NOT NULL;
+  INSERT INTO cursor_pins SELECT NEW.id,NEW.team_id,json_extract(NEW.snapshot_json,'$.session_id'),json_extract(NEW.snapshot_json,'$.revision'),NEW.expires_at
+    WHERE NEW.kind='history' AND json_extract(NEW.snapshot_json,'$.session_id') IS NOT NULL AND json_extract(NEW.snapshot_json,'$.revision') IS NOT NULL;
+END;
+CREATE TRIGGER cursor_pins_update AFTER UPDATE OF snapshot_json, expires_at, kind ON cursors BEGIN
+  DELETE FROM cursor_pins WHERE cursor_id=OLD.id;
+  INSERT INTO cursor_pins SELECT NEW.id,NEW.team_id,json_extract(j.value,'$[0]'),json_extract(j.value,'$[1]'),NEW.expires_at
+    FROM json_each(COALESCE(NEW.snapshot_json,'[]')) j WHERE NEW.kind IN ('list','search') AND json_extract(j.value,'$[0]') IS NOT NULL AND json_extract(j.value,'$[1]') IS NOT NULL;
+  INSERT INTO cursor_pins SELECT NEW.id,NEW.team_id,json_extract(NEW.snapshot_json,'$.session_id'),json_extract(NEW.snapshot_json,'$.revision'),NEW.expires_at
+    WHERE NEW.kind='history' AND json_extract(NEW.snapshot_json,'$.session_id') IS NOT NULL AND json_extract(NEW.snapshot_json,'$.revision') IS NOT NULL;
+END;
+CREATE TRIGGER cursor_pins_delete AFTER DELETE ON cursors BEGIN
+  DELETE FROM cursor_pins WHERE cursor_id=OLD.id;
+END;
+CREATE TABLE version_prune_queue (team_id TEXT NOT NULL, session_id TEXT NOT NULL, due_at INTEGER NOT NULL, generation INTEGER NOT NULL, PRIMARY KEY(team_id, session_id)) STRICT;
+CREATE INDEX version_prune_due ON version_prune_queue(due_at);
+
+-- what names each upload: an artifact, a continuation, an in-flight capture (holder: a:, c:, p: and its key)
+CREATE TABLE upload_refs (team_id TEXT NOT NULL, upload_id TEXT NOT NULL, holder TEXT NOT NULL, PRIMARY KEY(team_id, upload_id, holder)) STRICT, WITHOUT ROWID;
+CREATE INDEX upload_refs_holder ON upload_refs(holder);
+CREATE TABLE upload_prune_queue (team_id TEXT NOT NULL, upload_id TEXT NOT NULL, generation INTEGER NOT NULL, PRIMARY KEY(team_id, upload_id)) STRICT;
+CREATE TRIGGER upload_queue_on_commit AFTER UPDATE OF committed ON uploads WHEN NEW.committed=1 BEGIN
+  INSERT INTO upload_prune_queue VALUES (NEW.team_id,NEW.id,1) ON CONFLICT(team_id,upload_id) DO UPDATE SET generation=generation+1;
+  DELETE FROM pending_uploads WHERE team_id=NEW.team_id AND id=NEW.id;
+END;
+CREATE TRIGGER upload_queue_on_insert AFTER INSERT ON uploads BEGIN
+  INSERT INTO upload_prune_queue SELECT NEW.team_id,NEW.id,1 WHERE NEW.committed=1 ON CONFLICT(team_id,upload_id) DO UPDATE SET generation=generation+1;
+  INSERT OR IGNORE INTO pending_uploads SELECT NEW.team_id,NEW.id,NEW.source_id,NEW.created_at WHERE NEW.committed=0;
+END;
+CREATE TRIGGER upload_queue_on_delete AFTER DELETE ON uploads BEGIN
+  DELETE FROM pending_uploads WHERE team_id=OLD.team_id AND id=OLD.id;
+END;
+CREATE TRIGGER upload_refs_artifact_insert AFTER INSERT ON session_artifacts BEGIN
+  INSERT OR IGNORE INTO upload_refs VALUES (NEW.team_id, NEW.upload_id, 'a:'||NEW.session_id||':'||NEW.revision||':'||NEW.bundle_id);
+END;
+CREATE TRIGGER upload_refs_artifact_delete AFTER DELETE ON session_artifacts BEGIN
+  DELETE FROM upload_refs WHERE team_id=OLD.team_id AND upload_id=OLD.upload_id AND holder='a:'||OLD.session_id||':'||OLD.revision||':'||OLD.bundle_id;
+  INSERT INTO upload_prune_queue VALUES (OLD.team_id,OLD.upload_id,1) ON CONFLICT(team_id,upload_id) DO UPDATE SET generation=generation+1;
+END;
+CREATE TRIGGER upload_refs_continuation_insert AFTER INSERT ON continuations BEGIN
+  INSERT OR IGNORE INTO upload_refs SELECT NEW.team_id, substr(t.value, instr(t.value,'upload_'), 39), 'c:'||NEW.id
+    FROM json_tree(NEW.value_json) t WHERE t.type='text' AND instr(t.value,'upload_')>0;
+END;
+CREATE TRIGGER upload_refs_continuation_update AFTER UPDATE OF value_json ON continuations BEGIN
+  INSERT INTO upload_prune_queue SELECT team_id,upload_id,1 FROM upload_refs WHERE holder='c:'||OLD.id ON CONFLICT(team_id,upload_id) DO UPDATE SET generation=generation+1;
+  DELETE FROM upload_refs WHERE holder='c:'||OLD.id;
+  INSERT OR IGNORE INTO upload_refs SELECT NEW.team_id, substr(t.value, instr(t.value,'upload_'), 39), 'c:'||NEW.id
+    FROM json_tree(NEW.value_json) t WHERE t.type='text' AND instr(t.value,'upload_')>0;
+END;
+CREATE TRIGGER upload_refs_continuation_delete AFTER DELETE ON continuations BEGIN
+  INSERT INTO upload_prune_queue SELECT team_id,upload_id,1 FROM upload_refs WHERE holder='c:'||OLD.id ON CONFLICT(team_id,upload_id) DO UPDATE SET generation=generation+1;
+  DELETE FROM upload_refs WHERE holder='c:'||OLD.id;
+END;
+
+-- captures and uploads still in flight (expired after a day), and a committed capture's parts to remove
+CREATE TABLE staging_captures (team_id TEXT NOT NULL, id TEXT NOT NULL, source_id TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY(team_id, id)) STRICT;
+CREATE INDEX staging_captures_created ON staging_captures(created_at);
+CREATE TABLE pending_uploads (team_id TEXT NOT NULL, id TEXT NOT NULL, source_id TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY(team_id, id)) STRICT;
+CREATE INDEX pending_uploads_created ON pending_uploads(created_at);
+CREATE TABLE parts_prune_queue (team_id TEXT NOT NULL, capture_id TEXT NOT NULL, generation INTEGER NOT NULL, PRIMARY KEY(team_id, capture_id)) STRICT;
+CREATE TRIGGER capture_staging_insert AFTER INSERT ON captures WHEN NEW.state='staging' BEGIN
+  INSERT OR IGNORE INTO staging_captures VALUES (NEW.team_id, NEW.id, NEW.source_id, NEW.created_at);
+  INSERT OR IGNORE INTO upload_refs SELECT NEW.team_id, substr(t.value, instr(t.value,'upload_'), 39), 'p:'||NEW.id
+    FROM json_tree(NEW.begin_json) t WHERE t.type='text' AND instr(t.value,'upload_')>0;
+END;
+CREATE TRIGGER capture_staging_end AFTER UPDATE OF state ON captures WHEN OLD.state='staging' AND NEW.state<>'staging' BEGIN
+  DELETE FROM staging_captures WHERE team_id=OLD.team_id AND id=OLD.id;
+  INSERT INTO upload_prune_queue SELECT team_id,upload_id,1 FROM upload_refs WHERE holder='p:'||OLD.id ON CONFLICT(team_id,upload_id) DO UPDATE SET generation=generation+1;
+  DELETE FROM upload_refs WHERE holder='p:'||OLD.id;
+END;
+CREATE TRIGGER capture_committed AFTER UPDATE OF state ON captures WHEN NEW.state='committed' BEGIN
+  INSERT INTO parts_prune_queue VALUES (NEW.team_id, NEW.id, 1) ON CONFLICT(team_id,capture_id) DO UPDATE SET generation=generation+1;
+END;
+CREATE TRIGGER capture_staging_delete AFTER DELETE ON captures WHEN OLD.state='staging' BEGIN
+  DELETE FROM staging_captures WHERE team_id=OLD.team_id AND id=OLD.id;
+  INSERT INTO upload_prune_queue SELECT team_id,upload_id,1 FROM upload_refs WHERE holder='p:'||OLD.id ON CONFLICT(team_id,upload_id) DO UPDATE SET generation=generation+1;
+  DELETE FROM upload_refs WHERE holder='p:'||OLD.id;
+END;
+
+-- a blob a chunk naming it no longer names
+CREATE TABLE blob_prune_queue (team_id TEXT NOT NULL, sha256 TEXT NOT NULL, generation INTEGER NOT NULL, PRIMARY KEY(team_id, sha256)) STRICT;
+CREATE TRIGGER blob_queue_on_chunk AFTER DELETE ON artifact_chunk_refs BEGIN
+  INSERT INTO blob_prune_queue VALUES (OLD.team_id, OLD.sha256, 1) ON CONFLICT(team_id,sha256) DO UPDATE SET generation=generation+1;
+END;
+
+-- an event kept only as its resource's current one (retention's deletes passed it) goes once a newer event replaces it
+CREATE TRIGGER event_retention_superseded AFTER UPDATE OF sequence ON event_resources WHEN OLD.sequence<>NEW.sequence BEGIN
+  DELETE FROM org_events WHERE sequence=OLD.sequence AND sequence<=COALESCE((SELECT deleted_through FROM event_retention WHERE team_id=OLD.team_id),0);
+END;
+CREATE TRIGGER event_retention_released AFTER DELETE ON event_resources BEGIN
+  DELETE FROM org_events WHERE sequence=OLD.sequence AND sequence<=COALESCE((SELECT deleted_through FROM event_retention WHERE team_id=OLD.team_id),0);
+END;
+`;

package/server/migrations/index.mjs

@@ -21,7 +21,8 @@ import m019 from './019.mjs';
 import m020 from './020.mjs';
 import m021 from './021.mjs';
 import m022 from './022.mjs';
-export const MIGRATIONS = [m001, m002, m003, m004, m005, m006, m007, m008, m009, m010, m011, m012, m013, m014, m015, m016, m017, m018, m019, m020, m021, m022];
+import m023 from './023.mjs';
+export const MIGRATIONS = [m001, m002, m003, m004, m005, m006, m007, m008, m009, m010, m011, m012, m013, m014, m015, m016, m017, m018, m019, m020, m021, m022, m023];
 /** A message row's `until_revision` while its session's current version still holds it (schema 9). */
 export const OPEN_REVISION = Number.MAX_SAFE_INTEGER;
 /** SQL: message row `m` belongs to the version at revision ?1 (bind the revision twice). */