@volter/supercode-teams 0.2.161 → 0.2.163

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-teams",
-  "version": "0.2.161",
+  "version": "0.2.163",
   "type": "module",
   "description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
   "license": "MIT",

package/server/capture-transactions.mjs

@@ -0,0 +1,31 @@
+import { createHash } from 'node:crypto';
+import { MESSAGE_AT, OPEN_REVISION } from '@volter/teams/server/migrations';
+import { normalizeSearch, textFromContent } from './catalog.mjs';
+
+export function applyCaptureMessages({ get, run }, capture, revision) {
+  const { teamId, sessionId, sourceId, captureId, harness, nativeId, currentRevision, carriedCount, parts, messageNamespace } = capture;
+  let ordinal = carriedCount;
+  for (let index = 0; index < parts; index++) {
+    const heldPart = get('SELECT payload_json FROM capture_parts WHERE team_id=? AND capture_id=? AND part_index=?', [teamId, captureId, index]);
+    if (!heldPart) throw Object.assign(new Error('capture parts incomplete'), { status: 409, code: 'batch_gap' });
+    const part = JSON.parse(heldPart.payload_json);
+    for (const message of part.messages) {
+      const value = JSON.stringify(message.value);
+      const old = currentRevision == null ? null : get(`SELECT revision,source_id,native_key,recorded_at,value_json,until_revision FROM messages m WHERE m.team_id=? AND m.session_id=? AND m.ordinal=? AND ${MESSAGE_AT()} LIMIT 1`, [teamId, sessionId, ordinal, currentRevision, currentRevision]);
+      if (old && old.source_id === sourceId && old.native_key === message.native_key && old.recorded_at === message.recorded_at && old.value_json === value) {
+        if (old.until_revision !== OPEN_REVISION) run('UPDATE messages SET until_revision=? WHERE team_id=? AND session_id=? AND revision=? AND ordinal=?', [OPEN_REVISION, teamId, sessionId, old.revision, ordinal]);
+        ordinal++;
+        continue;
+      }
+      if (old?.until_revision === OPEN_REVISION) run('UPDATE messages SET until_revision=? WHERE team_id=? AND session_id=? AND revision=? AND ordinal=?', [revision, teamId, sessionId, old.revision, ordinal]);
+      let messageId = get('SELECT id FROM message_ids WHERE team_id=? AND source_id=? AND harness=? AND native_id=? AND native_key=?', [teamId, sourceId, harness, nativeId, message.native_key])?.id;
+      if (!messageId) {
+        messageId = `msg_${createHash('sha256').update(`${messageNamespace}\0${message.native_key}`).digest('hex').slice(0, 32)}`;
+        run('INSERT INTO message_ids(team_id,source_id,harness,native_id,native_key,id) VALUES (?,?,?,?,?,?)', [teamId, sourceId, harness, nativeId, message.native_key, messageId]);
+      }
+      run('INSERT INTO messages(team_id,session_id,revision,ordinal,id,source_id,native_key,recorded_at,value_json,search_text,until_revision) VALUES (?,?,?,?,?,?,?,?,?,?,?)', [teamId, sessionId, revision, ordinal++, messageId, sourceId, message.native_key, message.recorded_at, value, normalizeSearch(textFromContent(message.value.content)), OPEN_REVISION]);
+    }
+  }
+  if (currentRevision != null) run('UPDATE messages SET until_revision=? WHERE team_id=? AND session_id=? AND ordinal>=? AND revision<=? AND until_revision=?', [revision, teamId, sessionId, ordinal, currentRevision, OPEN_REVISION]);
+  return { messages: ordinal - carriedCount };
+}

package/server/harness-routes.mjs

@@ -10,8 +10,8 @@ import { createHash } from 'node:crypto';
 import { bearer, hashToken, canRead, canManage, isAdmin } from '@volter/teams/server/auth';
 import { storageReport } from './storage-report.mjs';
 import { validateListQuery, validateSearchQuery, validateReadQuery, validateCaptureBegin, validateCapturePart, canonicalJson } from '../contracts/index.mjs';
-import { normalizeSearch, textFromContent, cursorHash, cursorToken } from './catalog.mjs';
-import { captureDigest, captureMessages, handleOtlpRoute, sha256, validateArtifactName } from './ingest.mjs';
+import { normalizeSearch, cursorHash, cursorToken } from './catalog.mjs';
+import { handleOtlpRoute, sha256, validateArtifactName } from './ingest.mjs';
 import { handleCatalogEvents, handleSessionEvents, sessionNames } from './event-routes.mjs';
 import { exact } from '@volter/teams/server/request-json';
 import { MESSAGE_AT, OPEN_REVISION } from '@volter/teams/server/migrations';
@@ -203,7 +203,10 @@ const ACTIVITY_FRESH_MS=60000;
 /** The connector's latest fresh native turn report; stale or unrecognized reports remain unknown. */
 async function activityOf(api,teamId,sessionId){
   const row=await api.store.one('SELECT a.turn,a.since,a.observed_at FROM session_activity a JOIN origins o ON o.team_id=a.team_id AND o.source_id=a.source_id AND o.harness=a.harness AND o.native_id=a.native_id WHERE a.team_id=? AND o.deleted_at IS NULL AND o.session_id IN (SELECT id FROM sessions WHERE team_id=? AND (id=? OR canonical_id=?)) ORDER BY a.observed_at DESC LIMIT 1',[teamId,teamId,sessionId,sessionId]);
-  return row&&['working','idle','needs_input'].includes(row.turn)&&api.clock.nowMs()-row.observed_at<ACTIVITY_FRESH_MS?{state:row.turn,since:row.since??row.observed_at,observedAt:row.observed_at}:{state:'unknown'};
+  // Unknown is a fresh server determination too: give it an ordering time so
+  // an older known report cannot restore Working while the source is stale.
+  const now=api.clock.nowMs();
+  return row&&['working','idle','needs_input'].includes(row.turn)&&now-row.observed_at<ACTIVITY_FRESH_MS?{state:row.turn,since:row.since??row.observed_at,observedAt:row.observed_at}:{state:'unknown',observedAt:now};
 }
 async function sessionDescriptor(api,res,auth,id,requestId){
   const row=await api.store.authorizedSession({teamId:auth.team_id,principalId:auth.principal_id,credentialId:auth.credential_id,ref:id,now:api.clock.nowMs(),bound:auth.bound});
@@ -325,7 +328,26 @@ async function artifactChunk(api,req,res,auth,sourceId,uploadId,index,requestId)
   ],{status:200,body:response,targetId:uploadId});
   send(res,200,response,requestId);
 }
-async function captureCommit(api,req,res,auth,sourceId,captureId,requestId){const source=await sourceAuth(api,auth,sourceId);const commit=await parseJson(req);exact(commit,['capture_id','parts','digest','title','repository','reported_user','recorded_activity_at','parent_native_id','coverage','artifacts','native_bundle']);if(commit.capture_id!==captureId||!Number.isSafeInteger(commit.parts)||commit.parts<1)fail(400,'invalid_request','invalid commit');const cap=await api.store.one('SELECT * FROM captures WHERE team_id=? AND id=? AND source_id=?',[auth.team_id,captureId,sourceId]);if(await expireStaging(api,auth.team_id,sourceId,captureId,'captures',cap))fail(409,'resync_required','capture reservation expired');if(!cap)fail(404,'not_found','capture not found');if(cap.state==='committed'){const receipt=JSON.parse(cap.receipt_json);if(receipt.digest!==commit.digest)fail(409,'capture_conflict','commit replay differs');receipt.duplicate=true;return send(res,200,result(api,auth,receipt),requestId);}const rows=await api.store.query('SELECT * FROM capture_parts WHERE team_id=? AND capture_id=? ORDER BY part_index',[auth.team_id,captureId]);if(rows.length!==commit.parts)fail(409,'batch_gap','capture parts incomplete');const begin=JSON.parse(cap.begin_json);if(source.kind==='connector'){const rule=await api.store.one('SELECT * FROM sync_rules WHERE team_id=? AND id=? AND source_id=? AND machine_id=?',[auth.team_id,begin.rule_id,sourceId,auth.machine_id]);if(!rule||rule.state!=='active'||rule.revision!==begin.rule_revision||rule.visibility!==begin.visibility)fail(rule?.state==='paused'?409:403,rule?.state==='paused'?'source_paused':'forbidden','sync rule changed before commit');}const parts=rows.map(x=>JSON.parse(x.payload_json));if(captureDigest(begin,parts,commit)!==commit.digest)fail(409,'capture_conflict','capture digest differs');let messages=captureMessages(parts);if(messages.length>100000)fail(413,'request_too_large','capture contains too many messages');
+async function captureCommit(api,req,res,auth,sourceId,captureId,requestId){const source=await sourceAuth(api,auth,sourceId);const commit=await parseJson(req);exact(commit,['capture_id','parts','digest','title','repository','reported_user','recorded_activity_at','parent_native_id','coverage','artifacts','native_bundle']);if(commit.capture_id!==captureId||!Number.isSafeInteger(commit.parts)||commit.parts<1)fail(400,'invalid_request','invalid commit');const cap=await api.store.one('SELECT * FROM captures WHERE team_id=? AND id=? AND source_id=?',[auth.team_id,captureId,sourceId]);if(await expireStaging(api,auth.team_id,sourceId,captureId,'captures',cap))fail(409,'resync_required','capture reservation expired');if(!cap)fail(404,'not_found','capture not found');if(cap.state==='committed'){const receipt=JSON.parse(cap.receipt_json);if(receipt.digest!==commit.digest)fail(409,'capture_conflict','commit replay differs');receipt.duplicate=true;return send(res,200,result(api,auth,receipt),requestId);}const begin=JSON.parse(cap.begin_json);if(source.kind==='connector'){const rule=await api.store.one('SELECT * FROM sync_rules WHERE team_id=? AND id=? AND source_id=? AND machine_id=?',[auth.team_id,begin.rule_id,sourceId,auth.machine_id]);if(!rule||rule.state!=='active'||rule.revision!==begin.rule_revision||rule.visibility!==begin.visibility)fail(rule?.state==='paused'?409:403,rule?.state==='paused'?'source_paused':'forbidden','sync rule changed before commit');}
+  const heldParts = await api.store.one('SELECT COUNT(*) count,MIN(part_index) first,MAX(part_index) last FROM capture_parts WHERE team_id=? AND capture_id=?', [auth.team_id, captureId]);
+  if (heldParts.count !== commit.parts || heldParts.first !== 0 || heldParts.last !== commit.parts - 1) fail(409, 'batch_gap', 'capture parts incomplete');
+  const unsignedCommit = { ...commit };
+  delete unsignedCommit.digest;
+  const captureHash = createHash('sha256').update(`{"begin":${canonicalJson(begin)},"commit":${canonicalJson(unsignedCommit)},"parts":[`);
+  const sentKeys = new Set();
+  let messageCount = 0;
+  for (let index = 0; index < commit.parts; index++) {
+    const part = await api.store.one('SELECT payload_json FROM capture_parts WHERE team_id=? AND capture_id=? AND part_index=?', [auth.team_id, captureId, index]);
+    if (!part) fail(409, 'batch_gap', 'capture parts incomplete');
+    if (index) captureHash.update(',');
+    captureHash.update(part.payload_json);
+    for (const message of JSON.parse(part.payload_json).messages) {
+      if (sentKeys.has(message.native_key)) fail(409, 'capture_conflict', 'duplicate native message key');
+      sentKeys.add(message.native_key);
+      if (++messageCount > 100000) fail(413, 'request_too_large', 'capture contains too many messages');
+    }
+  }
+  if (captureHash.update(']}').digest('hex') !== commit.digest) fail(409, 'capture_conflict', 'capture digest differs');
   // Native transcripts can be larger than the Worker's entire heap. Validate
   // the retained bytes one chunk at a time; never materialize a whole artifact.
   for (const manifest of commit.artifacts) {
@@ -355,9 +377,9 @@ async function captureCommit(api,req,res,auth,sourceId,captureId,requestId){cons
   if (begin.carry) {
     const { count, digest } = begin.carry;
     const session = origin ? await api.store.one('SELECT revision FROM sessions WHERE team_id=? AND id=?', [auth.team_id, origin.session_id]) : null;
-    if (!session || count + messages.length > 100000) fail(409, 'carry_mismatch', 'the carried prefix is unavailable or too large');
+    if (!session || count + messageCount > 100000) fail(409, 'carry_mismatch', 'the carried prefix is unavailable or too large');
     carriedRevision = session.revision;
-    const hash = createHash('sha256').update('['), sent = new Set(messages.map(m => m.native_key));
+    const hash = createHash('sha256').update('[');
     while (carriedCount < count) {
       const page = await api.store.query(`SELECT ordinal,source_id,native_key,recorded_at,value_json FROM messages m
         WHERE m.team_id=? AND m.session_id=? AND m.ordinal>=? AND m.ordinal<? AND ${MESSAGE_AT()} ORDER BY m.ordinal LIMIT 16`,
@@ -365,7 +387,7 @@ async function captureCommit(api,req,res,auth,sourceId,captureId,requestId){cons
       if (!page.length) fail(409, 'carry_mismatch', 'the carried messages are not what this session holds; send the capture whole');
       for (const row of page) {
         if (row.ordinal !== carriedCount || row.source_id !== sourceId) fail(409, 'carry_mismatch', 'the carried messages are not what this session holds; send the capture whole');
-        if (sent.has(row.native_key)) fail(409, 'capture_conflict', 'duplicate native message key');
+        if (sentKeys.has(row.native_key)) fail(409, 'capture_conflict', 'duplicate native message key');
         if (carriedCount) hash.update(',');
         hash.update(canonicalJson({native_key:row.native_key, recorded_at:row.recorded_at ?? null, value:JSON.parse(row.value_json)}));
         carriedCount++;
@@ -382,15 +404,11 @@ async function captureCommit(api,req,res,auth,sourceId,captureId,requestId){cons
   // session that gained one message writes one row, not the whole session again.
   const carryGuards=[],currentRevision=origin?(await api.store.one('SELECT revision FROM sessions WHERE team_id=? AND id=?',[auth.team_id,sid]))?.revision??null:null;
   if (carriedRevision != null && currentRevision !== carriedRevision) fail(409, 'revision_mismatch', 'session changed during prefix verification; retry');
-  const held=new Map();if(currentRevision!=null)for(const row of await api.store.query(`SELECT revision,ordinal,id,source_id,native_key,recorded_at,value_json,until_revision FROM messages m WHERE m.team_id=? AND m.session_id=? AND ${MESSAGE_AT()} AND ordinal>=?`,[auth.team_id,sid,currentRevision,currentRevision,carriedCount]))held.set(row.ordinal,row);
+  sentKeys.clear();
+  carryGuards.push({ sql: 'SELECT 1 ok WHERE (SELECT COUNT(*) FROM capture_parts WHERE team_id=? AND capture_id=?)=?', params: [auth.team_id, captureId, commit.parts], status: 409, code: 'batch_gap', message: 'capture parts changed before commit' });
   // Another commit to this session between this read and the write would make the carried rows wrong: refused, retried.
   if(currentRevision!=null)carryGuards.push({sql:'SELECT 1 ok FROM sessions WHERE team_id=? AND id=? AND revision=?',params:[auth.team_id,sid,currentRevision],code:'revision_mismatch',status:409,message:'session changed during capture; retry'});
-  const close=row=>{if(row.until_revision===OPEN_REVISION)statements.push({sql:'UPDATE messages SET until_revision=? WHERE team_id=? AND session_id=? AND revision=? AND ordinal=?',params:[revisionParam,auth.team_id,sid,row.revision,row.ordinal]});};
-  let ordinal=carriedCount;for(const m of messages){const value=JSON.stringify(m.value),old=held.get(ordinal);held.delete(ordinal);
-    if(old&&old.source_id===sourceId&&old.native_key===m.native_key&&old.recorded_at===m.recorded_at&&old.value_json===value){if(old.until_revision!==OPEN_REVISION)statements.push({sql:'UPDATE messages SET until_revision=? WHERE team_id=? AND session_id=? AND revision=? AND ordinal=?',params:[OPEN_REVISION,auth.team_id,sid,old.revision,old.ordinal]});ordinal++;continue;}
-    if(old)close(old);
-    let mid=(await api.store.one('SELECT id FROM message_ids WHERE team_id=? AND source_id=? AND harness=? AND native_id=? AND native_key=?',[auth.team_id,sourceId,cap.harness,cap.native_id,m.native_key]))?.id;if(!mid){mid=api.ids.next('message');statements.push({sql:'INSERT INTO message_ids(team_id,source_id,harness,native_id,native_key,id) VALUES (?,?,?,?,?,?)',params:[auth.team_id,sourceId,cap.harness,cap.native_id,m.native_key,mid]});}statements.push({sql:'INSERT INTO messages(team_id,session_id,revision,ordinal,id,source_id,native_key,recorded_at,value_json,search_text,until_revision) VALUES (?,?,?,?,?,?,?,?,?,?,?)',params:[auth.team_id,sid,revisionParam,ordinal++,mid,sourceId,m.native_key,m.recorded_at,value,normalizeSearch(textFromContent(m.value.content)),OPEN_REVISION]});}
-  for(const row of held.values())close(row);
+  statements.push({ kind: 'capture_messages', capture: { teamId: auth.team_id, sessionId: sid, sourceId, captureId, harness: cap.harness, nativeId: cap.native_id, currentRevision, carriedCount, parts: commit.parts, messageNamespace: api.ids.next('message') } });
   const bundle=commit.native_bundle??null,legacyBundle=commit.artifacts.length?captureId:null;for(let index=0;index<commit.artifacts.length;index++){const a=commit.artifacts[index],file=bundle?.files.find(x=>x.upload_id===a.upload_id);statements.push({sql:'UPDATE uploads SET manifest_json=?,committed=1 WHERE team_id=? AND id=?',params:[JSON.stringify(a),auth.team_id,a.upload_id]},{sql:'INSERT INTO session_artifacts(team_id,session_id,revision,bundle_id,upload_id,format,provenance,file_role,fidelity,residue_json) VALUES (?,?,?,?,?,?,?,?,?,?)',params:[auth.team_id,sid,revisionParam,bundle?.id??legacyBundle,a.upload_id,bundle?.harness??cap.harness,'capture',file?.role??(index===0?'primary':'bundle'),bundle?.fidelity??'semantic',JSON.stringify(bundle?.residue??['native_bundle_metadata_unavailable'])]});}const storedReceipt={source_id:sourceId,capture_id:captureId,session_id:sid,generation:cap.generation,sequence:cap.sequence,digest:commit.digest,revision:'__JOURNAL_REVISION__',duplicate:false};statements.push({sql:"UPDATE captures SET state='committed',commit_json=?,receipt_json=? WHERE team_id=? AND id=?",params:[JSON.stringify(commit),{$journalJsonValue:storedReceipt},auth.team_id,captureId]},{sql:'DELETE FROM capture_parts WHERE team_id=? AND capture_id=?',params:[auth.team_id,captureId]},{sql:'UPDATE sources SET last_received_at=? WHERE team_id=? AND id=?',params:[boundaryNow,auth.team_id,sourceId]},{sql:'UPDATE teams SET journal_revision=? WHERE id=?',params:[revisionParam,auth.team_id]},{sql:'INSERT INTO journal(team_id,revision,session_id,kind,created_at) VALUES (?,?,?,?,?)',params:[auth.team_id,revisionParam,sid,'capture',boundaryNow]});const guards=[{sql:'SELECT 1 ok FROM credentials WHERE id=? AND source_id=? AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at>?)',params:[auth.credential_id,sourceId,boundaryNow],code:'authentication_required',status:401,message:'source credential expired or revoked'},...lineageGuards,...carryGuards,{sql:"SELECT 1 ok FROM captures c JOIN sources s ON s.team_id=c.team_id AND s.id=c.source_id LEFT JOIN machines m ON m.team_id=s.team_id AND m.id=s.machine_id LEFT JOIN sync_rules r ON r.team_id=c.team_id AND r.id=json_extract(c.begin_json,'$.rule_id') WHERE c.team_id=? AND c.id=? AND c.source_id=? AND c.state='staging' AND s.state='active' AND (s.kind!='connector' OR (m.revoked_at IS NULL AND r.machine_id=m.id AND r.source_id=s.id AND r.state='active' AND r.revision=json_extract(c.begin_json,'$.rule_revision') AND r.visibility=json_extract(c.begin_json,'$.visibility')))",params:[auth.team_id,captureId,sourceId],code:'conflict',status:409,message:'capture authorization changed'}],storedResponse=result(api,auth,storedReceipt),{revision}=await guardedMutation(api,res,auth,guards,statements,{status:200,body:storedResponse,targetId:captureId,journal:true}),receipt={...storedReceipt,revision};api.events.publish(auth.team_id,{type:'session_changed',cursor:String(revision),revision,session_id:sid,reason:'capture',read_required:true});send(res,200,result(api,auth,receipt),requestId);
 }
 

package/server/store-core.mjs

@@ -5,6 +5,7 @@
 // them on the object's own SQLite.
 import { ingestOtlpBatch, correlateSessions } from './arc2-transactions.mjs';
 import { claimEffect, getEffect, listEffects, reportEffect } from './effects-transactions.mjs';
+import { applyCaptureMessages } from './capture-transactions.mjs';
 import { MIGRATIONS, MESSAGE_AT } from '@volter/teams/server/migrations';
 
 /** Bring a database to the newest schema. `version()`/`setVersion(n)` name where the host records it. */
@@ -119,7 +120,7 @@ export function createStoreHandlers({ all, get, run, tx }) {
     transaction: ({statements}) => tx(() => statements.map(s => s.kind === 'query' ? all(s.sql,s.params) : s.kind === 'one' ? get(s.sql,s.params) : run(s.sql,s.params))),
     snapshot: ({statements}) => tx(() => statements.map(s => s.kind === 'one' ? get(s.sql,s.params) : all(s.sql,s.params))),
     guardedTransaction: ({guards=[],statements}) => tx(() => { for(const guard of guards){const row=get(guard.sql,guard.params);if(!row){const error=new Error(guard.message??'transaction predicate changed');error.code=guard.code??'conflict';error.status=guard.status??409;throw error;}}return statements.map(s=>s.kind==='query'?all(s.sql,s.params):s.kind==='one'?get(s.sql,s.params):run(s.sql,s.params));}),
-    journalTransaction: ({teamId,guards=[],statements}) => tx(() => { for(const guard of guards){const row=get(guard.sql,guard.params);if(!row){const error=new Error(guard.message??'transaction predicate changed');error.code=guard.code??'conflict';error.status=guard.status??409;throw error;}}const team=get('SELECT journal_revision FROM teams WHERE id=?',[teamId]);if(!team)throw new Error('team not found');const revision=team.journal_revision+1;const results=statements.map(s=>{const params=(s.params??[]).map(value=>journalParam(value,revision));return s.kind==='query'?all(s.sql,params):s.kind==='one'?get(s.sql,params):run(s.sql,params);});return{revision,results};}),
+    journalTransaction: ({teamId,guards=[],statements}) => tx(() => { for(const guard of guards){const row=get(guard.sql,guard.params);if(!row){const error=new Error(guard.message??'transaction predicate changed');error.code=guard.code??'conflict';error.status=guard.status??409;throw error;}}const team=get('SELECT journal_revision FROM teams WHERE id=?',[teamId]);if(!team)throw new Error('team not found');const revision=team.journal_revision+1;const results=statements.map(s=>{if(s.kind==='capture_messages')return applyCaptureMessages({get,run},s.capture,revision);const params=(s.params??[]).map(value=>journalParam(value,revision));return s.kind==='query'?all(s.sql,params):s.kind==='one'?get(s.sql,params):run(s.sql,params);});return{revision,results};}),
     ingestOtlpBatch: args => tx(() => ingestOtlpBatch({get,all,run},args)),
     correlateSessions: args => tx(() => correlateSessions({get,all,run},args)),
     listEffects: args => tx(() => listEffects({get,all,run},args)),