@volter/teams 0.2.38 → 0.2.40

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/teams",
-  "version": "0.2.38",
+  "version": "0.2.40",
   "type": "module",
   "description": "Volter Teams, the neutral team plane: machine identity and daemon core, the local door and server link, admission from machine grants, product doors, and the Teams server's neutral routes (company RFC 0008).",
   "license": "MIT",

package/server/http.mjs

@@ -398,12 +398,54 @@ async function revokeMachine(api,req,res,auth,machineId,requestId){
 }
 /** This machine's sessions someone is watching live (an open follow stream), as {harness, native_id}. */
 async function watchedFor(api,teamId,machineId){const ids=[...(api.watchers?.keys()??[])].filter(key=>key.startsWith(`${teamId}:`)).map(key=>key.slice(teamId.length+1)).slice(0,90);if(!ids.length)return [];return (await api.store.query(`SELECT o.harness,o.native_id FROM origins o JOIN sources s ON s.team_id=o.team_id AND s.id=o.source_id WHERE o.team_id=? AND s.machine_id=? AND o.session_id IN (${ids.map(()=>'?').join(',')})`,[teamId,machineId,...ids])).map(({harness,native_id})=>({harness,native_id}));}
-async function heartbeat(api,req,res,auth,machineId,requestId){if(auth.kind!=='machine'||auth.machine_id!==machineId)fail(403,'forbidden','exact machine credential required');const machine=await api.store.one('SELECT m.*,(SELECT linked FROM machine_presence p WHERE p.team_id=m.team_id AND p.machine_id=m.id) presence_linked FROM machines m WHERE m.team_id=? AND m.id=?',[auth.team_id,machineId]);if(!machine||machine.revoked_at!=null)fail(403,'forbidden','machine enrollment is not active');const b=await parseJson(req);exact(b,['incarnation_id','sequence','bindings','roots','host_key_id','activity']);if(b.activity!==undefined&&(!Array.isArray(b.activity)||b.activity.length>4096))fail(400,'invalid_request','invalid activity');if(b.host_key_id!==undefined&&(typeof b.host_key_id!=='string'||!/^[A-Za-z0-9_-]{16,128}$/.test(b.host_key_id)))fail(400,'invalid_request','invalid host_key_id');if(typeof b.incarnation_id!=='string'||!Number.isSafeInteger(b.sequence)||b.sequence<1||!Array.isArray(b.bindings)||!Array.isArray(b.roots))fail(400,'invalid_request','invalid heartbeat');if(machine.incarnation_id===b.incarnation_id&&b.sequence<=machine.heartbeat_sequence)return send(res,200,result(api,auth,{...machineJson(machine,api.clock.nowMs(),api),watched:await watchedFor(api,auth.team_id,machineId)}),requestId);const now=api.clock.nowMs(),newIncarnation=machine.incarnation_id!==null&&machine.incarnation_id!==b.incarnation_id,statements=[{sql:'UPDATE machines SET incarnation_id=?,heartbeat_sequence=?,last_seen_at=?,host_key_id=COALESCE(?,host_key_id),revision=revision+1 WHERE team_id=? AND id=?',params:[b.incarnation_id,b.sequence,now,b.host_key_id??null,auth.team_id,machineId]},{sql:'DELETE FROM machine_roots WHERE team_id=? AND machine_id=?',params:[auth.team_id,machineId]}];if(newIncarnation)statements.push({sql:"UPDATE attachments SET state='closed',closed_at=? WHERE team_id=? AND machine_id=? AND state IN ('pending','active')",params:[now,auth.team_id,machineId]},{sql:"UPDATE operations SET state='unknown' WHERE team_id=? AND id IN (SELECT operation_id FROM native_effects WHERE team_id=? AND machine_id=? AND state IN ('pending','running'))",params:[auth.team_id,auth.team_id,machineId]},{sql:"UPDATE native_effects SET state='unknown',result_json=? WHERE team_id=? AND machine_id=? AND state IN ('pending','running')",params:[JSON.stringify({state:'unknown',error:null}),auth.team_id,machineId]});statements.push({sql:'DELETE FROM bindings WHERE team_id=? AND machine_id=? AND id NOT IN (SELECT value FROM json_each(?)) AND id NOT IN (SELECT binding_id FROM attachments WHERE team_id=? AND machine_id=?) AND id NOT IN (SELECT binding_id FROM native_effects WHERE team_id=? AND machine_id=? AND binding_id IS NOT NULL)',params:[auth.team_id,machineId,JSON.stringify(b.bindings.map(binding=>binding.id)),auth.team_id,machineId,auth.team_id,machineId]});// What each synced session's harness is doing now: this heartbeat's report replaces the source's last one.
-  if(Array.isArray(b.activity)){const source=await api.store.one("SELECT id FROM sources WHERE team_id=? AND machine_id=? AND kind='connector'",[auth.team_id,machineId]);if(source){statements.push({sql:'DELETE FROM session_activity WHERE team_id=? AND source_id=?',params:[auth.team_id,source.id]});for(const item of b.activity){exact(item,['harness','native_id','turn','since'],'activity');if(typeof item.harness!=='string'||!item.harness||item.harness.length>64||typeof item.native_id!=='string'||!item.native_id||item.native_id.length>256||typeof item.turn!=='string'||!item.turn||item.turn.length>32||(item.since!=null&&!Number.isSafeInteger(item.since)))fail(400,'invalid_request','invalid activity');statements.push({sql:'INSERT OR REPLACE INTO session_activity(team_id,source_id,harness,native_id,turn,since,observed_at) VALUES (?,?,?,?,?,?,?)',params:[auth.team_id,source.id,item.harness,item.native_id,item.turn,item.since??null,now]});}}}
-  for(const root of b.roots){exact(root,['id','machine_id','name','harnesses'],'root');if(root.machine_id!==machineId)fail(400,'invalid_request','root machine mismatch');statements.push({sql:'INSERT INTO machine_roots(team_id,machine_id,id,value_json) VALUES (?,?,?,?)',params:[auth.team_id,machineId,root.id,JSON.stringify(root)]});}// A binding is stored against the session its native id names; one naming no session this source published is
-  // said back (unbound_bindings), never dropped in silence.
-  const unbound=[];for(const binding of b.bindings){const origin=await api.store.one('SELECT 1 ok FROM origins WHERE team_id=? AND source_id=? AND harness=? AND native_id=? AND deleted_at IS NULL',[auth.team_id,auth.source_id,binding.harness,binding.native_session_id]);if(!origin)unbound.push({id:binding.id,harness:binding.harness,native_session_id:binding.native_session_id,reason:'no_session'});}
-  for(const binding of b.bindings)statements.push({sql:'INSERT INTO bindings(team_id,id,machine_id,session_id,incarnation_id,descriptor_json,observed_at) SELECT ?,?,?,o.session_id,?,?,? FROM origins o WHERE o.team_id=? AND o.source_id=? AND o.harness=? AND o.native_id=? AND o.deleted_at IS NULL ON CONFLICT(team_id,id) DO UPDATE SET machine_id=excluded.machine_id,session_id=excluded.session_id,incarnation_id=excluded.incarnation_id,descriptor_json=excluded.descriptor_json,observed_at=excluded.observed_at',params:[auth.team_id,binding.id,machineId,b.incarnation_id,JSON.stringify(binding),now,auth.team_id,auth.source_id,binding.harness,binding.native_session_id]});const updated={...machine,incarnation_id:b.incarnation_id,heartbeat_sequence:b.sequence,last_seen_at:now,revision:machine.revision+1},response=result(api,auth,machineJson(updated,now,api)),guards=[{sql:"SELECT 1 ok FROM credentials c JOIN machines m ON m.team_id=c.team_id AND m.id=c.machine_id JOIN sources s ON s.team_id=c.team_id AND s.id=c.source_id JOIN memberships mem ON mem.team_id=m.team_id AND mem.principal_id=m.custodian_id WHERE c.id=? AND c.kind='machine' AND c.machine_id=? AND c.revoked_at IS NULL AND (c.expires_at IS NULL OR c.expires_at>?) AND m.revoked_at IS NULL AND mem.active=1 AND s.state!='revoked' AND (m.incarnation_id IS NULL OR m.incarnation_id!=? OR m.heartbeat_sequence<?)",params:[auth.credential_id,machineId,now,b.incarnation_id,b.sequence],code:'authentication_required',status:401,message:'machine authorization or heartbeat sequence changed'}];await guardedMutation(api,res,auth,guards,statements,{status:200,body:response,targetId:machineId});send(res,200,response?.data?{...response,data:{...response.data,watched:await watchedFor(api,auth.team_id,machineId),...(unbound.length?{unbound_bindings:unbound}:{})}}:response,requestId);}
+async function heartbeat(api,req,res,auth,machineId,requestId){
+  if(auth.kind!=='machine'||auth.machine_id!==machineId)fail(403,'forbidden','exact machine credential required');
+  const machine=await api.store.one('SELECT m.*,(SELECT linked FROM machine_presence p WHERE p.team_id=m.team_id AND p.machine_id=m.id) presence_linked FROM machines m WHERE m.team_id=? AND m.id=?',[auth.team_id,machineId]);
+  if(!machine||machine.revoked_at!=null)fail(403,'forbidden','machine enrollment is not active');
+  const b=await parseJson(req);
+  exact(b,['incarnation_id','sequence','bindings','roots','host_key_id','activity']);
+  if(b.activity!==undefined&&(!Array.isArray(b.activity)||b.activity.length>4096))fail(400,'invalid_request','invalid activity');
+  if(b.host_key_id!==undefined&&(typeof b.host_key_id!=='string'||!/^[A-Za-z0-9_-]{16,128}$/.test(b.host_key_id)))fail(400,'invalid_request','invalid host_key_id');
+  if(typeof b.incarnation_id!=='string'||!Number.isSafeInteger(b.sequence)||b.sequence<1||!Array.isArray(b.bindings)||!Array.isArray(b.roots))fail(400,'invalid_request','invalid heartbeat');
+  if(machine.incarnation_id===b.incarnation_id&&b.sequence<=machine.heartbeat_sequence)return send(res,200,result(api,auth,{...machineJson(machine,api.clock.nowMs(),api),watched:await watchedFor(api,auth.team_id,machineId)}),requestId);
+  const rootIds=new Set();
+  for(const root of b.roots){
+    exact(root,['id','machine_id','name','harnesses'],'root');
+    if(root.machine_id!==machineId)fail(400,'invalid_request','root machine mismatch');
+    if(rootIds.has(root.id))fail(400,'invalid_request','duplicate root');
+    rootIds.add(root.id);
+  }
+  for(const binding of b.bindings){
+    if(!binding||typeof binding!=='object'||typeof binding.id!=='string'||!binding.id||typeof binding.harness!=='string'||typeof binding.native_session_id!=='string')fail(400,'invalid_request','invalid binding');
+    if(binding.machine_id!==machineId||binding.incarnation_id!==b.incarnation_id)fail(400,'invalid_request','binding machine or incarnation mismatch');
+  }
+  let activity=null;
+  if(Array.isArray(b.activity)){
+    const byNative=new Map();
+    for(const item of b.activity){
+      exact(item,['harness','native_id','turn','since'],'activity');
+      if(typeof item.harness!=='string'||!item.harness||item.harness.length>64||typeof item.native_id!=='string'||!item.native_id||item.native_id.length>256||typeof item.turn!=='string'||!item.turn||item.turn.length>32||(item.since!=null&&!Number.isSafeInteger(item.since)))fail(400,'invalid_request','invalid activity');
+      byNative.set(JSON.stringify([item.harness,item.native_id]),{harness:item.harness,native_id:item.native_id,turn:item.turn,since:item.since??null});
+    }
+    const entries=[...byNative.entries()].sort(([a],[b])=>a<b?-1:a>b?1:0);
+    const json=canonicalJson(entries.map(([,item])=>item));
+    activity={json,digest:sha256(json)};
+  }
+  const now=api.clock.nowMs(),newIncarnation=machine.incarnation_id!==null&&machine.incarnation_id!==b.incarnation_id;
+  const statements=[{sql:'UPDATE machines SET incarnation_id=?,heartbeat_sequence=?,last_seen_at=?,host_key_id=COALESCE(?,host_key_id),revision=revision+1 WHERE team_id=? AND id=?',params:[b.incarnation_id,b.sequence,now,b.host_key_id??null,auth.team_id,machineId]}];
+  if(newIncarnation)statements.push(
+    {sql:"UPDATE attachments SET state='closed',closed_at=? WHERE team_id=? AND machine_id=? AND state IN ('pending','active')",params:[now,auth.team_id,machineId]},
+    {sql:"UPDATE operations SET state='unknown' WHERE team_id=? AND id IN (SELECT operation_id FROM native_effects WHERE team_id=? AND machine_id=? AND state IN ('pending','running'))",params:[auth.team_id,auth.team_id,machineId]},
+    {sql:"UPDATE native_effects SET state='unknown',result_json=? WHERE team_id=? AND machine_id=? AND state IN ('pending','running')",params:[JSON.stringify({state:'unknown',error:null}),auth.team_id,machineId]}
+  );
+  const snapshotIndex=statements.length;
+  statements.push({kind:'heartbeat_snapshot',snapshot:{teamId:auth.team_id,machineId,sourceId:auth.source_id,incarnationId:b.incarnation_id,now,rootsJson:canonicalJson(b.roots),bindingsJson:canonicalJson(b.bindings),activity}});
+  const updated={...machine,incarnation_id:b.incarnation_id,heartbeat_sequence:b.sequence,last_seen_at:now,revision:machine.revision+1},response=result(api,auth,machineJson(updated,now,api));
+  const guards=[{sql:"SELECT 1 ok FROM credentials c JOIN machines m ON m.team_id=c.team_id AND m.id=c.machine_id JOIN sources s ON s.team_id=c.team_id AND s.id=c.source_id JOIN memberships mem ON mem.team_id=m.team_id AND mem.principal_id=m.custodian_id WHERE c.id=? AND c.kind='machine' AND c.machine_id=? AND c.revoked_at IS NULL AND (c.expires_at IS NULL OR c.expires_at>?) AND m.revoked_at IS NULL AND mem.active=1 AND s.state!='revoked' AND (m.incarnation_id IS NULL OR m.incarnation_id!=? OR m.heartbeat_sequence<?)",params:[auth.credential_id,machineId,now,b.incarnation_id,b.sequence],code:'authentication_required',status:401,message:'machine authorization or heartbeat sequence changed'}];
+  const results=await guardedMutation(api,res,auth,guards,statements,{status:200,body:response,targetId:machineId});
+  const {unbound}=results[snapshotIndex];
+  send(res,200,{...response,data:{...response.data,watched:await watchedFor(api,auth.team_id,machineId),...(unbound.length?{unbound_bindings:unbound}:{})}},requestId);
+}
 async function machineConfiguration(api,res,auth,machineId,requestId){
   if(auth.kind!=='machine'||auth.machine_id!==machineId)fail(auth.kind==='machine'?404:403,auth.kind==='machine'?'not_found':'forbidden','not found');
   const [machine,rules]=await api.store.snapshot([

package/server/migrations/026.mjs

@@ -0,0 +1,32 @@
+// Capture prefixes are server-owned SHA-256 continuations. Preparing an old session
+// is resumable; a request never reopens its entire transcript. No message body is copied.
+export default `
+CREATE TABLE capture_prefix_state (
+  team_id TEXT NOT NULL, session_id TEXT NOT NULL, source_id TEXT NOT NULL,
+  revision INTEGER NOT NULL, generation TEXT NOT NULL,
+  count INTEGER NOT NULL, target_count INTEGER NOT NULL, sha_state TEXT NOT NULL,
+  checkpoint_count INTEGER NOT NULL, checkpoint_bytes INTEGER NOT NULL,
+  invalid INTEGER NOT NULL DEFAULT 0,
+  PRIMARY KEY(team_id,session_id,source_id)
+) STRICT;
+CREATE TABLE capture_prefix_checkpoints (
+  team_id TEXT NOT NULL, session_id TEXT NOT NULL, source_id TEXT NOT NULL,
+  generation TEXT NOT NULL, count INTEGER NOT NULL, digest TEXT NOT NULL, sha_state TEXT NOT NULL,
+  PRIMARY KEY(team_id,session_id,source_id,generation,count)
+) STRICT;
+CREATE TABLE capture_prefix_keys (
+  team_id TEXT NOT NULL, session_id TEXT NOT NULL, source_id TEXT NOT NULL,
+  generation TEXT NOT NULL, native_key TEXT NOT NULL, ordinal INTEGER NOT NULL,
+  PRIMARY KEY(team_id,session_id,source_id,generation,native_key)
+) STRICT;
+CREATE INDEX capture_prefix_keys_ordinal ON capture_prefix_keys(team_id,session_id,source_id,generation,ordinal);
+CREATE TABLE capture_prefix_garbage (
+  team_id TEXT NOT NULL, session_id TEXT NOT NULL, source_id TEXT NOT NULL, generation TEXT NOT NULL,
+  PRIMARY KEY(team_id,session_id,source_id,generation)
+) STRICT;
+CREATE TABLE capture_prefix_warm_queue (
+  ticket INTEGER PRIMARY KEY AUTOINCREMENT,
+  team_id TEXT NOT NULL, session_id TEXT NOT NULL, source_id TEXT NOT NULL,
+  UNIQUE(team_id,session_id,source_id)
+) STRICT;
+`;

package/server/migrations/027.mjs

@@ -0,0 +1,13 @@
+// A full heartbeat refreshes one report clock, not every activity/binding row.
+export default `
+CREATE TABLE machine_heartbeat_state (
+  team_id TEXT NOT NULL, machine_id TEXT NOT NULL, activity_source_id TEXT NOT NULL,
+  activity_digest TEXT NOT NULL, activity_observed_at INTEGER NOT NULL,
+  PRIMARY KEY(team_id,machine_id),
+  FOREIGN KEY(team_id,machine_id) REFERENCES machines(team_id,id)
+) STRICT;
+CREATE INDEX bindings_machine ON bindings(team_id,machine_id,id);
+CREATE INDEX machine_roots_machine ON machine_roots(team_id,machine_id,id);
+CREATE INDEX native_effects_binding ON native_effects(team_id,machine_id,binding_id);
+CREATE INDEX sources_machine_kind ON sources(team_id,machine_id,kind);
+`;

package/server/migrations/index.mjs

@@ -24,7 +24,9 @@ import m022 from './022.mjs';
 import m023 from './023.mjs';
 import m024 from './024.mjs';
 import m025 from './025.mjs';
-export const MIGRATIONS = [m001, m002, m003, m004, m005, m006, m007, m008, m009, m010, m011, m012, m013, m014, m015, m016, m017, m018, m019, m020, m021, m022, m023, m024, m025];
+import m026 from './026.mjs';
+import m027 from './027.mjs';
+export const MIGRATIONS = [m001, m002, m003, m004, m005, m006, m007, m008, m009, m010, m011, m012, m013, m014, m015, m016, m017, m018, m019, m020, m021, m022, m023, m024, m025, m026, m027];
 /** A message row's `until_revision` while its session's current version still holds it (schema 9). */
 export const OPEN_REVISION = Number.MAX_SAFE_INTEGER;
 /** SQL: message row `m` belongs to the version at revision ?1 (bind the revision twice). */

package/server/published-events.mjs

@@ -132,6 +132,9 @@ export async function publishedEventRoute(ctx) {
     for(const session of new Set(body.sessions)){
       const natives=(await api.store.query("SELECT DISTINCT json_extract(descriptor_json,'$.native_session_id') AS native FROM bindings WHERE team_id=? AND session_id=?",[auth.team_id,session])).map(row=>row.native).filter(Boolean);
       const statements=[];
+      // Purging text also removes every continuation buffer that can contain
+      // its last SHA block, including retired generations. Never retain a redacted byte there.
+      for(const table of ['capture_prefix_state','capture_prefix_checkpoints','capture_prefix_keys','capture_prefix_garbage','capture_prefix_warm_queue'])statements.push(stmt(`DELETE FROM ${table} WHERE team_id=? AND session_id=?`,auth.team_id,session));
       for(const value of body.values){
         statements.push(stmt('UPDATE messages SET value_json=replace(value_json,?,?),search_text=replace(search_text,?,?) WHERE team_id=? AND session_id=? AND (instr(value_json,?)>0 OR instr(search_text,?)>0)',value,marker,value,marker,auth.team_id,session,value,value));
         if(natives.length)statements.push(stmt(`UPDATE capture_parts SET payload_json=replace(payload_json,?,?) WHERE team_id=? AND instr(payload_json,?)>0 AND capture_id IN (SELECT id FROM captures WHERE team_id=? AND native_id IN (${natives.map(()=>'?').join(',')}))`,value,marker,auth.team_id,value,auth.team_id,...natives));