@volter/supercode-teams 0.3.158 → 0.3.159
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -2
- package/bin/workspace/sync-status.mjs +9 -0
- package/bin/workspace.mjs +2 -2
- package/connector/index.d.ts +2 -1
- package/connector/index.mjs +92 -27
- package/connector/redaction-holds.mjs +59 -0
- package/connector/redactor.mjs +15 -8
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -70,10 +70,23 @@ does not attribute the observed process peak or claim a measured runtime improve
|
|
|
70
70
|
Memo entries are discarded when the observed custody/rules basis changes. A
|
|
71
71
|
capture and its worker use one matching custody snapshot; a basis mismatch
|
|
72
72
|
rejects the capture before queueing and leaves it owed for retry. Older basis
|
|
73
|
-
labels cannot authorize held-digest reuse.
|
|
74
|
-
30-second interval; queued and published captures are not retroactively redacted.
|
|
73
|
+
labels cannot authorize held-digest reuse.
|
|
75
74
|
[Source boundary and qualification](../../docs/evidence/connector-redaction-basis-2026-10-08.md).
|
|
76
75
|
|
|
76
|
+
Session captures reread the custody registry and declared files before each
|
|
77
|
+
transfer, including rate-limit retries. A stale queued capture is retained
|
|
78
|
+
locally and replaced by a fresh full capture; its carried prefix, parts and
|
|
79
|
+
native references are not reused. Original artifact files stay held until a
|
|
80
|
+
replacement commit acknowledges the native bundle too. Other queued sessions
|
|
81
|
+
can proceed. `supercode teams status` and `supercode auth status` show the holds
|
|
82
|
+
and explain that registration does not purge already transferred or published
|
|
83
|
+
data. A Teams administrator uses `TeamsClient.purgeSessionText` / `POST
|
|
84
|
+
/api/v1/teams/:team/sessions/purge` for retrospective session-text cleanup.
|
|
85
|
+
Direct writes after the last check and requests already in flight have **no
|
|
86
|
+
bounded exposure time**. This is capture-transfer source behavior, with runtime
|
|
87
|
+
and single-machine qualification still open.
|
|
88
|
+
[Transfer scope and retained obligations](../../docs/evidence/connector-custody-egress-2026-10-08.md).
|
|
89
|
+
|
|
77
90
|
`supercode teams rpc mail.statistics` reads local storage counts and import/retention receipts. It is refused remotely
|
|
78
91
|
and through the app door. Writer groups count stored envelope copies, not call attempts or quotas; database page
|
|
79
92
|
counts exclude physical WAL size. Successful imports record their actual source reads, bytes, batch sizes and time.
|
|
@@ -8,6 +8,15 @@ const reason = item => `${item.code ? `${item.code}: ` : ''}${item.message}`;
|
|
|
8
8
|
export function syncStatusLines(sync, { indent = ' ' } = {}) {
|
|
9
9
|
if (!sync) return [];
|
|
10
10
|
const lines = [];
|
|
11
|
+
if(sync.custody_notice)lines.push(`Custody ${sync.custody_notice}`);
|
|
12
|
+
const holds=sync.redaction_held??[];
|
|
13
|
+
if(holds.length){
|
|
14
|
+
lines.push(`Custody holds (${holds.length} capture${holds.length===1?'':'s'} retained locally):`);
|
|
15
|
+
const labels={owed:'fresh full capture owed',native_owed:'replacement native bundle owed',blocked:'payload refused; waiting for source, rule or custody change',cleanup:'replacement committed; local cleanup pending'};
|
|
16
|
+
for(const item of holds.slice(0,SHOWN))lines.push(`${indent}${item.harness} ${item.session_id} · ${item.capture_id} · ${when(item.at)} · ${labels[item.state]??item.state}${item.replacement?` · replacement ${item.replacement}`:''}`);
|
|
17
|
+
if(holds.length>SHOWN)lines.push(`${indent}… and ${holds.length-SHOWN} more (--json lists them all)`);
|
|
18
|
+
}
|
|
19
|
+
if(sync.redaction_held_unread?.length)lines.push(`Custody holds unread (${sync.redaction_held_unread.length}); original files and all artifact spills retained, inspect --json`);
|
|
11
20
|
const problems = sync.not_syncing ?? [];
|
|
12
21
|
if (problems.length) {
|
|
13
22
|
lines.push(`Not syncing (${problems.length} session${problems.length === 1 ? '' : 's'}, most recent first):`);
|
package/bin/workspace.mjs
CHANGED
|
@@ -402,9 +402,9 @@ async function authCommand({ flags, sub = 'status', rest, home, contexts, output
|
|
|
402
402
|
if (!account.context) return say(account.others.length ? `Team none current · saved: ${account.others.join(', ')} (supercode auth switch NAME)` : 'Team none (supercode login)');
|
|
403
403
|
say(`Team ${account.team} · ${new URL(account.origin).host}${account.reachable ? '' : ' · unreachable'}${account.others.length ? ` · also: ${account.others.join(', ')} (supercode auth switch)` : ''}`);
|
|
404
404
|
say(account.machine ? `Machine ${account.machine.name} · ${account.machine.connection ?? 'unknown'} · sharing ${account.machine.folder ?? '?'}${account.machine.service ? ` (${account.machine.service})` : ' (runs while `teams connect` does)'}` : 'Machine not connected (supercode teams connect)');
|
|
405
|
-
//
|
|
405
|
+
// Custody's published-history boundary is part of account status even without a failed capture.
|
|
406
406
|
const sync = account.machine?.sync;
|
|
407
|
-
if (sync?.not_syncing.length || sync?.not_bound?.length || sync?.last_error?.at > Date.now() - 86_400_000) for (const line of syncStatusLines(sync)) say(line);
|
|
407
|
+
if (sync?.custody_notice || sync?.not_syncing.length || sync?.not_bound?.length || sync?.last_error?.at > Date.now() - 86_400_000) for (const line of syncStatusLines(sync)) say(line);
|
|
408
408
|
return;
|
|
409
409
|
}
|
|
410
410
|
if (sub === 'switch') {
|
package/connector/index.d.ts
CHANGED
|
@@ -65,8 +65,9 @@ export interface ConnectorConfigurationValue {
|
|
|
65
65
|
|
|
66
66
|
export interface SyncErrorSummary { code: string | null; message: string; at: number }
|
|
67
67
|
export interface SyncProblem extends SyncErrorSummary { harness: string; session_id: string; rule_id: string | null }
|
|
68
|
+
export interface RedactionHoldStatus { capture_id: string; harness: string; session_id: string; rule_id: string | null; at: number | null; state: 'owed' | 'native_owed' | 'blocked' | 'cleanup'; replacement?: string }
|
|
68
69
|
/** What `supercode teams status` shows: sessions not syncing (most recent first) and the last error; null without state. */
|
|
69
|
-
export declare function readSyncStatus(stateDir: string): { not_syncing: SyncProblem[]; not_bound: { harness: string; session_id: string; reason: string }[]; not_bound_count: number; last_error: SyncErrorSummary | null; queued_captures: number } | null;
|
|
70
|
+
export declare function readSyncStatus(stateDir: string): { custody_notice: string; redaction_held: RedactionHoldStatus[]; redaction_held_unread: { name: string; error: string }[]; not_syncing: SyncProblem[]; not_bound: { harness: string; session_id: string; reason: string }[]; not_bound_count: number; last_error: SyncErrorSummary | null; queued_captures: number } | null;
|
|
70
71
|
|
|
71
72
|
export declare class TeamsConnector {
|
|
72
73
|
constructor(options: TeamsConnectorOptions);
|
package/connector/index.mjs
CHANGED
|
@@ -2,6 +2,7 @@ import { mkdirSync, readFileSync, writeFileSync, renameSync, existsSync, chmodSy
|
|
|
2
2
|
import { realpathSync } from './path.mjs';
|
|
3
3
|
import { redactionBasis } from '@volter/teams/redact';
|
|
4
4
|
import { Redactor } from './redactor.mjs';
|
|
5
|
+
import { RedactionHolds } from './redaction-holds.mjs';
|
|
5
6
|
import { basename, dirname, join, sep } from 'node:path';
|
|
6
7
|
|
|
7
8
|
/** A path in its real form: its nearest existing ancestor resolved (a file inside a folder this process may not read is
|
|
@@ -46,8 +47,9 @@ const covers=(root,cwd)=>{
|
|
|
46
47
|
};
|
|
47
48
|
// An error of the connection itself (the outbox, authentication, the server), not of one session.
|
|
48
49
|
const connectionError=error=>Boolean(error?.outbox||error?.status===401||error?.status===403||error?.status>=500);
|
|
49
|
-
const retryCapture=error=>connectionError(error)||error?.code
|
|
50
|
+
const retryCapture=error=>connectionError(error)||['redaction_basis_changed','custody_unreadable'].includes(error?.code);
|
|
50
51
|
const sameError=(a,b)=>a!=null&&b!=null&&a.code===b.code&&a.message===b.message;
|
|
52
|
+
const CUSTODY_NOTICE='Custody registration does not purge data already transferred or published. A Teams administrator can purge session text through POST /api/v1/teams/:team/sessions/purge (TeamsClient.purgeSessionText). A direct file edit after the last transfer check, or a request already in flight, has no bounded exposure time.';
|
|
51
53
|
const errorSummary=(error,at)=>error==null?null:{code:typeof error.code==='string'?error.code:null,message:String(error.message??error).slice(0,500),at};
|
|
52
54
|
/** The sync status a connector keeps in its state directory (see TeamsConnector#syncStatus); null when it has none. */
|
|
53
55
|
export function readSyncStatus(stateDir){
|
|
@@ -55,7 +57,7 @@ export function readSyncStatus(stateDir){
|
|
|
55
57
|
const status=state?.sync_status;
|
|
56
58
|
// Sessions this machine could not bind to the agent their launch named (bindAgentSessions): waiting, or refused for good.
|
|
57
59
|
const agentSessions=Object.entries(state?.agent_sessions??{}).filter(([,item])=>item?.state==='pending'||item?.state==='refused').map(([session,item])=>({session,agent:item.agent,state:item.state,reason:item.error?.message??null}));
|
|
58
|
-
return {agent_sessions_unbound:agentSessions,index_unread:Object.values(status?.index_unread??{}).flat(),not_syncing:Array.isArray(status?.problems)?status.problems:[],not_bound:Array.isArray(status?.unbound)?status.unbound:[],not_bound_count:Number.isSafeInteger(status?.unbound_count)?status.unbound_count:(Array.isArray(status?.unbound)?status.unbound.length:0),last_error:status?.last_error??null,queued_captures:Array.isArray(state?.outbox)?state.outbox.length:(()=>{try{return readdirSync(join(stateDir,'outbox')).filter(name=>name.endsWith('.json')).length;}catch{return 0;}})()};
|
|
60
|
+
return {custody_notice:CUSTODY_NOTICE,redaction_held:status?.redaction_held??[],redaction_held_unread:status?.redaction_held_unread??[],agent_sessions_unbound:agentSessions,index_unread:Object.values(status?.index_unread??{}).flat(),not_syncing:Array.isArray(status?.problems)?status.problems:[],not_bound:Array.isArray(status?.unbound)?status.unbound:[],not_bound_count:Number.isSafeInteger(status?.unbound_count)?status.unbound_count:(Array.isArray(status?.unbound)?status.unbound.length:0),last_error:status?.last_error??null,queued_captures:Array.isArray(state?.outbox)?state.outbox.length:(()=>{try{return readdirSync(join(stateDir,'outbox')).filter(name=>name.endsWith('.json')).length;}catch{return 0;}})()};
|
|
59
61
|
}
|
|
60
62
|
// A tool call still running when the core reads a session has no result yet; the core answers it with a synthesized
|
|
61
63
|
// placeholder (crates/interchange session/helpers.rs ensure_tool_results_paired: a `tool` message for that call id,
|
|
@@ -93,11 +95,14 @@ const itemSpills=item=>(item.artifacts??[]).filter(artifact=>artifact.spill).map
|
|
|
93
95
|
* folded state writes them out again (TeamsConnector#loadEntries).
|
|
94
96
|
*/
|
|
95
97
|
export function foldConnectorState(stateDir){
|
|
98
|
+
if(existsSync(join(stateDir,'redaction-held'))&&readdirSync(join(stateDir,'redaction-held')).some(name=>/^\d+\.json$/.test(name)))throw new Error('retained custody captures prevent folding state for an older connector; resolve the visible redaction holds first');
|
|
96
99
|
const path=join(stateDir,'connector.json'),state=JSON.parse(readFileSync(path,'utf8'));
|
|
97
100
|
const origins=join(stateDir,'origins'),outbox=join(stateDir,'outbox'),files=dir=>existsSync(dir)?readdirSync(dir).filter(name=>name.endsWith('.json')).sort():[];
|
|
101
|
+
if(state.outbox?.some(item=>item.redaction_hold))throw new Error('a custody-flagged capture prevents folding state for an older connector');
|
|
98
102
|
if(state.origins||state.outbox)return {origins:Object.keys(state.origins??{}).length,captures:(state.outbox??[]).length,folded:false};
|
|
99
103
|
state.origins={};for(const name of files(origins)){const {key,origin}=JSON.parse(readFileSync(join(origins,name),'utf8'));state.origins[key]=origin;}
|
|
100
104
|
state.outbox=files(outbox).map(name=>JSON.parse(readFileSync(join(outbox,name),'utf8')));delete state.outbox_sequence;
|
|
105
|
+
if(state.outbox.some(item=>item.redaction_hold))throw new Error('a custody-flagged capture prevents folding state for an older connector');
|
|
101
106
|
atomic(path,JSON.stringify(state,null,2)+'\n');
|
|
102
107
|
for(const dir of [origins,outbox])rmSync(dir,{recursive:true,force:true});
|
|
103
108
|
return {origins:Object.keys(state.origins).length,captures:state.outbox.length,folded:true};
|
|
@@ -112,11 +117,12 @@ export function foldConnectorState(stateDir){
|
|
|
112
117
|
const REDACTED_MEMO=200000,REDACTED_MEMO_BYTES=32*1024*1024,REDACTED_MEMO_ENTRY_BYTES=1024*1024;
|
|
113
118
|
const redactions=new Map();
|
|
114
119
|
let redactionBytes=0,redactionMemoBasis=null;
|
|
120
|
+
const freshRedactionBasis=()=>redactionBasis(undefined,{fresh:true});
|
|
115
121
|
function useRedactionMemo(basis){
|
|
116
122
|
if(redactionMemoBasis!==basis){redactions.clear();redactionBytes=0;redactionMemoBasis=basis;}
|
|
117
123
|
}
|
|
118
124
|
function requireRedactionBasis(basis){
|
|
119
|
-
if(
|
|
125
|
+
if(freshRedactionBasis()!==basis)throw Object.assign(new Error('redaction basis changed; capture needs fresh redaction'),{code:'redaction_basis_changed'});
|
|
120
126
|
}
|
|
121
127
|
// Older queued/committed captures could label a result with the basis read after its worker returned. They do not
|
|
122
128
|
// establish the checked observation needed for held-digest reuse, even when that old label equals today's basis.
|
|
@@ -246,7 +252,7 @@ export class TeamsConnector {
|
|
|
246
252
|
for(const dir of [origins,outbox])rmSync(dir,{recursive:true,force:true});
|
|
247
253
|
for(const dir of [origins,outbox])mkdirSync(dir,{recursive:true,mode:0o700});
|
|
248
254
|
for(const [key,origin] of Object.entries(this.state.origins??{}))atomic(join(origins,`${hash(key).slice(0,32)}.json`),JSON.stringify({key,origin}));
|
|
249
|
-
this.state.outbox_sequence=0;
|
|
255
|
+
this.state.outbox_sequence=existsSync(join(this.stateDir,'redaction-held'))?readdirSync(join(this.stateDir,'redaction-held')).filter(name=>/^\d+\.json$/.test(name)).reduce((sequence,name)=>Math.max(sequence,Number(name.slice(0,-5))),0):0;
|
|
250
256
|
for(const item of this.state.outbox??[])atomic(join(outbox,`${String(++this.state.outbox_sequence).padStart(12,'0')}.json`),JSON.stringify(item));
|
|
251
257
|
delete this.state.origins;delete this.state.outbox;atomic(this.statePath,JSON.stringify(this.state,null,2)+'\n');
|
|
252
258
|
}
|
|
@@ -263,6 +269,9 @@ export class TeamsConnector {
|
|
|
263
269
|
for(const name of readdirSync(outbox).filter(name=>name.endsWith('.json')).sort()){try{this.outbox.push(Object.assign(JSON.parse(readFileSync(join(outbox,name),'utf8')),{[QUEUED_FILE]:name}));}catch(error){this.outboxUnread.push({name,error:String(error?.message??error)});}}
|
|
264
270
|
// the sequence is past every queued capture's name on disk, read or unread, so no new one takes an unread one's
|
|
265
271
|
for(const name of readdirSync(outbox)){const n=Number(/^(\d+)\.json$/.exec(name)?.[1]);if(Number.isSafeInteger(n)&&n>(this.state.outbox_sequence??0))this.state.outbox_sequence=n;}
|
|
272
|
+
this.redactionHolds=new RedactionHolds({stateDir:this.stateDir,atomic,saved:this.state.redaction_holds});
|
|
273
|
+
for(const name of [...this.redactionHolds.items.keys(),...this.redactionHolds.unread.map(item=>item.name)]){const n=Number(name.slice(0,-5));if(Number.isSafeInteger(n)&&n>(this.state.outbox_sequence??0))this.state.outbox_sequence=n;}
|
|
274
|
+
this.updateRedactionHolds();
|
|
266
275
|
}
|
|
267
276
|
/**
|
|
268
277
|
* Where a Codex rollout past the display window is read from: the origin's window_start while it holds (the same
|
|
@@ -291,6 +300,33 @@ export class TeamsConnector {
|
|
|
291
300
|
// every beat does not rewrite connector.json every beat. Its time stays that of its first occurrence in the run.
|
|
292
301
|
set lastError(error){this.lastErrorValue=error;const status=this.syncStatus(),summary=errorSummary(error,this.clock.nowMs());if(sameError(status.last_error,summary))return;status.last_error=summary;this.saveQuietly();}
|
|
293
302
|
syncStatus(){const status=this.state.sync_status??={problems:[],last_error:null};status.problems??=[];return status;}
|
|
303
|
+
updateRedactionHolds(){
|
|
304
|
+
this.state.redaction_holds=this.redactionHolds.saved();
|
|
305
|
+
const status=this.syncStatus();status.redaction_held=this.redactionHolds.status();status.redaction_held_unread=this.redactionHolds.unread;
|
|
306
|
+
}
|
|
307
|
+
observeCustodyBasis(){
|
|
308
|
+
const basis=freshRedactionBasis();
|
|
309
|
+
if(this.state.custody_basis!==basis){this.state.custody_basis=basis;process.stderr.write(`supercode teams: ${CUSTODY_NOTICE}\n`);this.saveQuietly();}
|
|
310
|
+
return basis;
|
|
311
|
+
}
|
|
312
|
+
requireCaptureTransfer(item){
|
|
313
|
+
const basis=this.observeCustodyBasis();
|
|
314
|
+
if(item.redaction_hold||item.redactionBasis!==checkedRedactionBasis(basis))throw Object.assign(new Error('queued capture needs fresh custody redaction; retained locally for fresh capture'),{code:item.redaction_hold?.code??'redaction_basis_changed'});
|
|
315
|
+
return basis;
|
|
316
|
+
}
|
|
317
|
+
async guardCaptureTransfer(item,json){
|
|
318
|
+
const basis=this.requireCaptureTransfer(item);
|
|
319
|
+
this.redactor??=new Redactor();
|
|
320
|
+
await this.redactor.run('custody_assert',[json],{basis,fresh:true});
|
|
321
|
+
// request invokes this synchronously after the worker answer, with no intervening await before fetch.
|
|
322
|
+
return ()=>this.requireCaptureTransfer(item);
|
|
323
|
+
}
|
|
324
|
+
holdQueuedCapture(item,error){
|
|
325
|
+
this.redactionHolds.hold(item,item[QUEUED_FILE],this.clock.nowMs(),error.code);
|
|
326
|
+
this.outbox.shift(); // Its spill references move to the hold; unqueue would release them.
|
|
327
|
+
this.updateRedactionHolds();this.noteSessionProblem({harness:item.begin.harness,nativeSessionId:item.begin.native_session_id,ruleId:item.begin.rule_id},error);
|
|
328
|
+
this.oweCapture({harness:item.begin.harness,nativeSessionId:item.begin.native_session_id},error);this.save();
|
|
329
|
+
}
|
|
294
330
|
saveQuietly(){try{this.save();}catch{}}
|
|
295
331
|
noteSessionProblem({harness,nativeSessionId,ruleId=null},error){
|
|
296
332
|
const status=this.syncStatus(),at=this.clock.nowMs(),summary=errorSummary(error,at);
|
|
@@ -305,7 +341,7 @@ export class TeamsConnector {
|
|
|
305
341
|
clearSessionProblem(harness,nativeSessionId){
|
|
306
342
|
const status=this.state.sync_status;if(!status?.problems?.some(item=>item.harness===harness&&item.session_id===nativeSessionId))return;
|
|
307
343
|
status.problems=status.problems.filter(item=>!(item.harness===harness&&item.session_id===nativeSessionId));this.saveQuietly();
|
|
308
|
-
} async request(method,path,body){const json=body===undefined?undefined:JSON.stringify(body),mutation=['POST','PATCH','DELETE'].includes(method);let response;for(let attempt=0;;attempt++){response=await this.fetch(this.server+`/api/v1/teams/${this.teamId}`+path,{method,redirect:'manual',headers:{Authorization:`Bearer ${this.credential}`,'Content-Type':'application/json',...(mutation?{'Idempotency-Key':hash(`${method}\n${path}\n${json??''}`)}:{})},body:json,signal:AbortSignal.any([this.abort.signal, AbortSignal.timeout(30_000)])});
|
|
344
|
+
} async request(method,path,body,{beforeTransfer}={}){const json=body===undefined?undefined:JSON.stringify(body),mutation=['POST','PATCH','DELETE'].includes(method);let response;for(let attempt=0;;attempt++){if(beforeTransfer){const finalCheck=await beforeTransfer(json);finalCheck();}response=await this.fetch(this.server+`/api/v1/teams/${this.teamId}`+path,{method,redirect:'manual',headers:{Authorization:`Bearer ${this.credential}`,'Content-Type':'application/json',...(mutation?{'Idempotency-Key':hash(`${method}\n${path}\n${json??''}`)}:{})},body:json,signal:AbortSignal.any([this.abort.signal, AbortSignal.timeout(30_000)])});
|
|
309
345
|
// A rate limit is waited out (the server says how long; the same idempotency key makes the retry safe), so a
|
|
310
346
|
// large first sync slows down instead of failing its pass and restarting the connector.
|
|
311
347
|
if(response.status!==429||attempt>=5)break;// The server names the wait in error.details (wire.mjs validationFailure); a Retry-After header is the fallback.
|
|
@@ -664,7 +700,9 @@ export class TeamsConnector {
|
|
|
664
700
|
const {locator,descriptor,harness,nativeSessionId,ruleId,epoch}=options,key=`${harness}:${nativeSessionId}`;
|
|
665
701
|
if(!this.sessionLive(key,ruleId,epoch))return;
|
|
666
702
|
// A session whose store has not changed since this rule last committed it is in sync, whatever it last failed.
|
|
667
|
-
let captured=true
|
|
703
|
+
let captured=true,changed;
|
|
704
|
+
try{changed=this.sessionChanged(options);}catch(error){if(retryCapture(error))this.oweCapture(options,error);else this.noteSessionProblem(options,error);return;}
|
|
705
|
+
if(!changed)this.clearSessionProblem(harness,nativeSessionId);
|
|
668
706
|
// A store this process may not read (macOS denies a launchd job a removable volume, whatever the owner granted the
|
|
669
707
|
// terminal) is said once with its path and skipped.
|
|
670
708
|
if(this.unreadable?.has(key))captured=false;
|
|
@@ -791,6 +829,9 @@ export class TeamsConnector {
|
|
|
791
829
|
}
|
|
792
830
|
sessionChanged(options){
|
|
793
831
|
const stamp=this.sessionStamp(options);options.stamp=stamp;
|
|
832
|
+
const custodyKey=`${options.harness}:${options.nativeSessionId}`;
|
|
833
|
+
if(this.redactionHolds.items.size&&this.redactionHolds.blocked(custodyKey,stamp,options.ruleRevision,checkedRedactionBasis(this.observeCustodyBasis())))return false;
|
|
834
|
+
if(this.redactionHolds.needed(custodyKey))return true;
|
|
794
835
|
if(stamp===null||this.outbox.length)return true;
|
|
795
836
|
const rule=options.configuration?.rules?.find(item=>item.id===options.ruleId&&item.revision===options.ruleRevision&&item.state==='active');if(!rule)return true;
|
|
796
837
|
const configured=rule.backfill??options.backfill??'all',mode=typeof configured==='string'?configured:configured.mode;
|
|
@@ -843,6 +884,7 @@ export class TeamsConnector {
|
|
|
843
884
|
if(this.spillRefs)return this.spillRefs;
|
|
844
885
|
const refs=this.spillRefs=new Map(),add=name=>refs.set(name,(refs.get(name)??0)+1);
|
|
845
886
|
for(const item of this.outbox)for(const name of itemSpills(item))add(name);
|
|
887
|
+
for(const name of this.redactionHolds.spills())add(name);
|
|
846
888
|
return refs;
|
|
847
889
|
}
|
|
848
890
|
retainSpills(names){const refs=this.spillHolders();for(const name of names){refs.set(name,(refs.get(name)??0)+1);this.freshSpills?.delete(name);}}
|
|
@@ -856,7 +898,7 @@ export class TeamsConnector {
|
|
|
856
898
|
* whoever else held it. A file kept so is held by nothing counted and goes at the first start whose inventory is whole.
|
|
857
899
|
*/
|
|
858
900
|
removeSpill(name){
|
|
859
|
-
if(this.outboxUnread?.length){this.spillProblem(`${name} was kept:
|
|
901
|
+
if(this.outboxUnread?.length||this.redactionHolds.unread.length){this.spillProblem(`${name} was kept: a queued or custody-held capture could not be read and may name it`);return false;}
|
|
860
902
|
try{unlinkSync(join(this.spillDir(),name));return true;}
|
|
861
903
|
catch(error){if(error?.code==='ENOENT')return true;this.spillProblem(`${name} could not be removed: ${error?.message??error}`);return false;}
|
|
862
904
|
}
|
|
@@ -874,9 +916,9 @@ export class TeamsConnector {
|
|
|
874
916
|
removeUnheldSpills(){
|
|
875
917
|
const status=this.syncStatus();delete status.spill_cleanup;
|
|
876
918
|
// only against a complete inventory of the outbox: a queued capture this process could not read may name any file
|
|
877
|
-
if(this.outboxUnread.length){
|
|
919
|
+
if(this.outboxUnread.length||this.redactionHolds.unread.length){
|
|
878
920
|
status.outbox_unread=this.outboxUnread.map(({name,error})=>`${name}: ${error}`);
|
|
879
|
-
this.spillProblem(
|
|
921
|
+
this.spillProblem('a queued or custody-held capture could not be read; no spill file is removed until their inventory is complete');
|
|
880
922
|
return;
|
|
881
923
|
}
|
|
882
924
|
delete status.outbox_unread;
|
|
@@ -901,7 +943,7 @@ export class TeamsConnector {
|
|
|
901
943
|
// Each loaded message in the form sent: the memo's; the loaded one itself when the server holds it unchanged under
|
|
902
944
|
// this redaction basis (its digest is among those last observed); else what the redaction worker makes of it, all of
|
|
903
945
|
// a capture's new messages in one request. A digest is taken once per message, of the form sent.
|
|
904
|
-
async sentMessages(raw,previous,basis=
|
|
946
|
+
async sentMessages(raw,previous,basis=freshRedactionBasis()){
|
|
905
947
|
requireRedactionBasis(basis);useRedactionMemo(basis);
|
|
906
948
|
const held=previous?.redaction_basis===checkedRedactionBasis(basis)&&previous?.observed_digests?.length?new Set(previous.observed_digests):null;
|
|
907
949
|
const out=new Array(raw.length),missing=[];
|
|
@@ -915,7 +957,7 @@ export class TeamsConnector {
|
|
|
915
957
|
this.redactor??=new Redactor();
|
|
916
958
|
if(!this.redactionCounted){this.redactionCounted=true;this.syncStatus().redaction={since:new Date(this.clock.nowMs()).toISOString(),batches:0,messages:0,bytes:0,worker_ms:0,main_ms:0,max_main_ms:0};}
|
|
917
959
|
// The worker is sent each new message's canonical JSON and answers only those redaction changed.
|
|
918
|
-
let started=performance.now();const texts=missing.map(([,,text])=>text),pending=this.redactor.run('canonical',texts,{basis});let mainMs=performance.now()-started;
|
|
960
|
+
let started=performance.now();const texts=missing.map(([,,text])=>text),pending=this.redactor.run('canonical',texts,{basis,fresh:true});let mainMs=performance.now()-started;
|
|
919
961
|
const changed=await pending;started=performance.now();
|
|
920
962
|
// Another connector/capture can run while this worker answers. Never store its result under a later basis.
|
|
921
963
|
requireRedactionBasis(basis);useRedactionMemo(basis);
|
|
@@ -938,10 +980,11 @@ export class TeamsConnector {
|
|
|
938
980
|
const files=translated.files?.length?translated.files:[{path:translated.suggested_filename??defaultName,content:translated.content,role:'primary'}];
|
|
939
981
|
if(files.filter(file=>file.role==='primary').length!==1)throw Object.assign(new Error('native codec bundle must contain exactly one primary file'),{code:'fidelity_unavailable'});
|
|
940
982
|
let total=0;const spilled=[];
|
|
941
|
-
this.redactor??=new Redactor();const texts=await this.redactor.run('texts',files.map(file=>file.content),{basis});
|
|
983
|
+
this.redactor??=new Redactor();const texts=await this.redactor.run('texts',files.map(file=>file.content),{basis,fresh:true});
|
|
984
|
+
const [metadata]=await this.redactor.run('values',[{paths:files.map(file=>file.path),residue:translated.residue??[]}],{basis,fresh:true});
|
|
942
985
|
if(basis!==undefined)requireRedactionBasis(basis);
|
|
943
|
-
for(const [index,file] of files.entries()){const bytes=Buffer.from(texts[index],'utf8');total+=bytes.length;if(total>256*1024*1024)throw Object.assign(new Error('native artifact bundle exceeds 256 MiB'),{code:'request_too_large'});spilled.push({path:
|
|
944
|
-
return {fidelity:translated.fidelity,residue:
|
|
986
|
+
for(const [index,file] of files.entries()){const bytes=Buffer.from(texts[index],'utf8');total+=bytes.length;if(total>256*1024*1024)throw Object.assign(new Error('native artifact bundle exceeds 256 MiB'),{code:'request_too_large'});spilled.push({path:metadata.paths[index],role:file.role,...this.spill(bytes)});}
|
|
987
|
+
return {fidelity:translated.fidelity,residue:metadata.residue,files:spilled};
|
|
945
988
|
}
|
|
946
989
|
// The outbox artifacts and native_bundle of one capture from a spilled bundle; every capture gets fresh upload ids.
|
|
947
990
|
bundleArtifacts(bundle,harness){
|
|
@@ -1140,13 +1183,14 @@ export class TeamsConnector {
|
|
|
1140
1183
|
}
|
|
1141
1184
|
// The turns typed through a door here are labelled before redaction, which then reads them as sent.
|
|
1142
1185
|
const raw=(loaded.messages??loaded.turns??[]).map((entry,index)=>{const value=entry.value??Object.fromEntries(Object.entries(entry).filter(([key])=>!['id','native_key','recorded_at','timestamp'].includes(key)));return{native_key:entry.id??entry.native_key??String(index),recorded_at:recordedAt(entry),value};});this.labelTurns(key,loaded.messages??loaded.turns??[],raw);
|
|
1143
|
-
const basis=
|
|
1186
|
+
const basis=this.observeCustodyBasis();let messages=await this.sentMessages(raw,previous,basis);const loadedMessages=messages;
|
|
1187
|
+
const recapturing=this.redactionHolds.needed(key);
|
|
1144
1188
|
if (harness === 'codex' && loaded.total_message_count > messages.length) coverage = { ...coverage, completeness: 'partial', reasons: [...new Set([...coverage.reasons, 'prefix_not_captured'])] };
|
|
1145
1189
|
// A session published without its native bundle is captured again, with the same messages, once the bundle can be built.
|
|
1146
1190
|
const nativeWanted=configuration.native_capture!==false&&backfillPolicy.mode==='all'&&typeof this.core.translateSession==='function'&&!((artifactBefore?.size??0)>NATIVE_MAX_BYTES);
|
|
1147
1191
|
const retryNative=Boolean(previous?.native_deferred&&nativeWanted&&artifactBefore?.isFile()&&!previous?.baseline&&this.nativeReady(key,artifactBefore));
|
|
1148
1192
|
const unchanged=previous?.last_rule_revision===ruleRevision&&previous?.backfill_mode===backfillPolicy.mode&&previous?.observed_keys?.length===messages.length&&messages.every((message,index)=>message.native_key===previous.observed_keys[index]&&digestOf(message)===previous.observed_digests?.[index]);
|
|
1149
|
-
if(unchanged&&!retryNative){requireRedactionBasis(basis);this.origins[key]={...previous,redaction_basis:checkedRedactionBasis(basis)};this.recordStamp(key,ruleId,ruleRevision,backfillPolicy.mode,stamp);await this.publishSessionProfile(key);return null;}
|
|
1193
|
+
if(unchanged&&!retryNative&&!recapturing){requireRedactionBasis(basis);this.origins[key]={...previous,redaction_basis:checkedRedactionBasis(basis)};this.recordStamp(key,ruleId,ruleRevision,backfillPolicy.mode,stamp);await this.publishSessionProfile(key);return null;}
|
|
1150
1194
|
let effectivePrevious=previous;const clearBaseline=backfillPolicy.mode==='all';
|
|
1151
1195
|
// A `none` rule keeps back the history recorded before it became active (the server's capture_after), not
|
|
1152
1196
|
// whatever a session holds when first seen: a session begun after the rule is published from its first message.
|
|
@@ -1185,7 +1229,7 @@ export class TeamsConnector {
|
|
|
1185
1229
|
const begin={native_session_id:nativeSessionId,harness,generation,sequence,capture_id:captureId,previous_digest:effectivePrevious?.digest??null,visibility,rule_id:ruleId,rule_revision:ruleRevision};
|
|
1186
1230
|
// Only what the server does not hold goes over the wire: the longest prefix this source last committed unchanged
|
|
1187
1231
|
// (the outbox is empty here, so that is the session's current version) is named by its digest, not sent again.
|
|
1188
|
-
const carried=this.carriedPrefix(effectivePrevious,messages,{ruleRevision,backfillMode:backfillPolicy.mode,clearBaseline});
|
|
1232
|
+
const carried=recapturing?0:this.carriedPrefix(effectivePrevious,messages,{ruleRevision,backfillMode:backfillPolicy.mode,clearBaseline});
|
|
1189
1233
|
if(carried)begin.carry={count:carried,digest:carryDigest(messages.slice(0,carried))};
|
|
1190
1234
|
const sent=carried?messages.slice(carried):messages;
|
|
1191
1235
|
const parts=captureParts(captureId,sent);
|
|
@@ -1214,7 +1258,7 @@ export class TeamsConnector {
|
|
|
1214
1258
|
const translated=(await this.core.translateSession({locator,target_harness:harness},{timeoutMs:translateTimeoutMs})).artifact;
|
|
1215
1259
|
if(fileStamp(statSync(artifactSource))===onDisk)bundle={stamp:onDisk,...await this.spillTranslated(translated,basename(artifactSource),basis)};
|
|
1216
1260
|
}catch(error){
|
|
1217
|
-
if(this.abort.signal.aborted||error?.code
|
|
1261
|
+
if(this.abort.signal.aborted||['redaction_basis_changed','custody_unreadable'].includes(error?.code))throw error;
|
|
1218
1262
|
this.lastError=error;process.stderr.write(`supercode teams: ${harness} session ${nativeSessionId} published without its native bundle: ${error.message}\n`);
|
|
1219
1263
|
// The same file is not translated again; a write to it clears this.
|
|
1220
1264
|
if(this.sessionLive(key,ruleId,epoch)){this.state.native_failed[key]=onDisk;this.save();}
|
|
@@ -1224,13 +1268,16 @@ export class TeamsConnector {
|
|
|
1224
1268
|
if(bundle&&fileStamp(statSync(artifactSource))!==onDisk)bundle=null;
|
|
1225
1269
|
if(bundle)({artifacts,nativeBundle}=this.bundleArtifacts(bundle,harness));else nativeDeferred=true;
|
|
1226
1270
|
// A retry that still has no bundle would republish the same messages for nothing.
|
|
1227
|
-
if(unchanged&&!nativeBundle){requireRedactionBasis(basis);this.origins[key]={...previous,redaction_basis:checkedRedactionBasis(basis)};this.recordStamp(key,ruleId,ruleRevision,backfillPolicy.mode,stamp);this.armNativeIdle(key);return null;}
|
|
1271
|
+
if(unchanged&&!nativeBundle&&!recapturing){requireRedactionBasis(basis);this.origins[key]={...previous,redaction_basis:checkedRedactionBasis(basis)};this.recordStamp(key,ruleId,ruleRevision,backfillPolicy.mode,stamp);this.armNativeIdle(key);return null;}
|
|
1228
1272
|
}else if(sqliteSource&&nativeWanted&&!effectivePrevious?.baseline){
|
|
1229
1273
|
const translated=(await this.core.translateSession({locator,target_harness:harness},{timeoutMs:translateTimeoutMs})).artifact;
|
|
1230
1274
|
const digest=entries=>hash(canonicalJson((entries.messages??entries.turns??[]).map(entry=>entry.value??entry)));if(digest(await this.core.session(locator).load())!==digest(loaded))throw Object.assign(new Error('native session changed while it was being captured'),{code:'resync_required'});
|
|
1231
1275
|
({artifacts,nativeBundle}=this.bundleArtifacts(await this.spillTranslated(translated,`${nativeSessionId}.json`,basis),harness));
|
|
1232
1276
|
}
|
|
1233
|
-
|
|
1277
|
+
// Descriptive metadata shares the capture's custody snapshot too. Structural identifiers cannot be rewritten;
|
|
1278
|
+
// the request guard refuses them if they themselves contain a custody value.
|
|
1279
|
+
this.redactor??=new Redactor();const [metadata]=await this.redactor.run('values',[{title:descriptor?.title??null,coverage:effectivePrevious?.baseline&&!clearBaseline?{...coverage,completeness:'partial',reasons:[...new Set([...coverage.reasons,'prefix_not_captured'])]}:coverage}],{basis,fresh:true});
|
|
1280
|
+
const recordedTimes=messages.map(message=>message.recorded_at).filter(Number.isSafeInteger),commit={capture_id:captureId,parts:parts.length,digest:'',...metadata,repository:null,reported_user:null,recorded_activity_at:recordedTimes.length?Math.max(...recordedTimes):null,parent_native_id:descriptor?.parent_session_id??null,artifacts:artifacts.map(item=>item.manifest),...(nativeBundle?{native_bundle:nativeBundle}:{})};
|
|
1234
1281
|
const digestCommit={...commit};delete digestCommit.digest;commit.digest=hash(canonicalJson({begin,parts,commit:digestCommit}));
|
|
1235
1282
|
// A session gone or no longer covered while it loaded queues nothing (its spilled bundle, held by nothing, goes at the next start).
|
|
1236
1283
|
if(!this.sessionLive(key,ruleId,epoch))return null;
|
|
@@ -1419,19 +1466,19 @@ export class TeamsConnector {
|
|
|
1419
1466
|
item.commit={...item.commit,capture_id:captureId,parts:item.parts.length};
|
|
1420
1467
|
const digestCommit={...item.commit};delete digestCommit.digest;item.commit.digest=hash(canonicalJson({begin:item.begin,parts:item.parts,commit:digestCommit}));this.rewriteQueued(item);this.save();
|
|
1421
1468
|
}
|
|
1422
|
-
async sendCapture(item){const begun=await
|
|
1469
|
+
async sendCapture(item){const request=(method,path,body)=>this.request(method,path,body,{beforeTransfer:json=>this.guardCaptureTransfer(item,json)});const begun=await request('POST',`/sources/${this.sourceId}/captures`,item.begin);if(begun.state==='committed')return begun.receipt;for(const part of item.parts)await request('PUT',`/sources/${this.sourceId}/captures/${item.begin.capture_id}/parts/${part.index}`,part);
|
|
1423
1470
|
// A capture queued by an older connector holds its chunks inline; a current one reads them from its spill file.
|
|
1424
1471
|
for(const artifact of item.artifacts??[])for(const chunk of artifact.spill?this.spilledChunks(artifact):artifact.chunks){
|
|
1425
1472
|
if(!chunk)throw Object.assign(new Error('a spilled native artifact changed on disk while it was sent'),{code:'resync_required'});
|
|
1426
1473
|
const path=`/sources/${this.sourceId}/uploads/${chunk.upload_id}/chunks/${chunk.index}`;
|
|
1427
1474
|
let reused=false;
|
|
1428
1475
|
if(this.chunkReuse!==false)try{
|
|
1429
|
-
await
|
|
1476
|
+
await request('PUT',path,{upload_id:chunk.upload_id,index:chunk.index,sha256:chunk.sha256,reuse:true});reused=true;this.chunkReuse=true;
|
|
1430
1477
|
}catch(error){
|
|
1431
1478
|
if(error.code==='invalid_request'&&error.status===400)this.chunkReuse=false; // older server
|
|
1432
1479
|
else if(error.code!=='chunk_missing')throw error;
|
|
1433
1480
|
}
|
|
1434
|
-
if(!reused)await
|
|
1481
|
+
if(!reused)await request('PUT',path,chunk);
|
|
1435
1482
|
this.state.native_transfer??={sent_bytes:0,reused_bytes:0,chunks:0};
|
|
1436
1483
|
this.state.native_transfer[reused?'reused_bytes':'sent_bytes']+=Buffer.byteLength(chunk.bytes_base64,'base64');
|
|
1437
1484
|
this.state.native_transfer.chunks++;
|
|
@@ -1442,7 +1489,7 @@ export class TeamsConnector {
|
|
|
1442
1489
|
transfer.chunks++;transfer.at=this.clock.nowMs();
|
|
1443
1490
|
for(const expired of Object.keys(transfers).sort((a,b)=>transfers[b].at-transfers[a].at).slice(128))delete transfers[expired];
|
|
1444
1491
|
}
|
|
1445
|
-
const receipt=await
|
|
1492
|
+
const receipt=await request('POST',`/sources/${this.sourceId}/captures/${item.begin.capture_id}/commit`,item.commit);
|
|
1446
1493
|
const transfer=this.state.native_transfers?.[`${item.begin.harness}:${item.begin.native_session_id}`];
|
|
1447
1494
|
if(transfer&&item.artifacts?.length){transfer.captures++;transfer.last_bundle_bytes=item.commit.artifacts.reduce((sum,a)=>sum+a.byte_length,0);}
|
|
1448
1495
|
return receipt;}
|
|
@@ -1457,15 +1504,33 @@ export class TeamsConnector {
|
|
|
1457
1504
|
if(!Array.isArray(configuration?.rules))return false;
|
|
1458
1505
|
return !configuration.rules.some(rule=>rule.id===item.begin.rule_id&&rule.revision===item.begin.rule_revision&&rule.state==='active');
|
|
1459
1506
|
}
|
|
1460
|
-
async flushOutbox(){let latest=null;while(this.outbox.length){const item=this.outbox[0];let receipt;this.dropLostArtifacts(item);this.refitParts(item);
|
|
1507
|
+
async flushOutbox(){let latest=null;while(this.outbox.length){const item=this.outbox[0];let receipt;try{this.requireCaptureTransfer(item);this.dropLostArtifacts(item);this.refitParts(item);receipt=await this.sendCapture(item);}catch(error){
|
|
1508
|
+
// A failed request may have awaited long enough to observe a different policy. Retain its whole original
|
|
1509
|
+
// before an existing server-error branch can drop a part, native reference or queued item.
|
|
1510
|
+
if(!['redaction_basis_changed','redaction_capture_unsafe'].includes(error.code))try{this.requireCaptureTransfer(item);}catch(changed){error=changed;}
|
|
1511
|
+
if(['redaction_basis_changed','redaction_capture_unsafe'].includes(error.code)){this.holdQueuedCapture(item,error);continue;}
|
|
1512
|
+
if(error.code==='storage_pressure'&&this.deferNativeUnderPressure(item))continue;
|
|
1513
|
+
if(error.status===409&&error.code==='carry_mismatch'){const key=`${item.begin.harness}:${item.begin.native_session_id}`;this.lastError=error;process.stderr.write(`supercode teams: the server does not hold what a capture of ${item.begin.harness} session ${item.begin.native_session_id} carried; it is sent whole again\n`);this.unqueue();if(this.origins[key])this.origins[key]={...this.origins[key],observed_keys:[],observed_digests:[],carry_refused:true,capture_stamps:{}};this.save();this.resee(key);continue;}if(error.status===409&&error.code==='batch_gap'&&error.details&&Number.isSafeInteger(error.details.sequence)){
|
|
1461
1514
|
// The server holds a different last capture of this session than this connector recorded (a capture it
|
|
1462
1515
|
// committed while the connector stopped before recording it, for one): this connector takes the server's
|
|
1463
1516
|
// revision, drops the stale capture and sends the session whole again at once (resee).
|
|
1464
1517
|
const key=`${item.begin.harness}:${item.begin.native_session_id}`,expected=error.details;this.lastError=error;
|
|
1465
1518
|
process.stderr.write(`supercode teams: ${item.begin.harness} session ${item.begin.native_session_id} resumes at the server's revision ${expected.generation}/${expected.sequence}; it is sent whole again\n`);
|
|
1466
1519
|
this.unqueue();this.origins[key]={...this.origins[key],generation:expected.generation,sequence:expected.sequence-1,digest:expected.digest??null,observed_keys:[],observed_digests:[],carry_refused:true,capture_stamps:{}};this.save();this.resee(key);continue;}
|
|
1467
|
-
if(error.status===400&&error.code==='invalid_request'||error.status===413){this.lastError=error;this.noteSessionProblem({harness:item.begin.harness,nativeSessionId:item.begin.native_session_id,ruleId:item.begin.rule_id??null},error);(this.droppedCaptures??=new Set()).add(`${item.begin.harness}:${item.begin.native_session_id}`);if(error.status===413)(this.state.set_aside??={})[`${item.begin.harness}:${item.begin.native_session_id}`]={at:this.clock.nowMs(),rule_revision:item.begin.rule_revision??null,reason:error.message};process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id} the server refused${error.status===413?' (one message is larger than a request may be)':' as malformed'}: ${error.message}\n`);this.unqueue();this.save();continue;}
|
|
1468
|
-
this.
|
|
1520
|
+
if(error.status===400&&error.code==='invalid_request'||error.status===413){this.lastError=error;this.noteSessionProblem({harness:item.begin.harness,nativeSessionId:item.begin.native_session_id,ruleId:item.begin.rule_id??null},error);(this.droppedCaptures??=new Set()).add(`${item.begin.harness}:${item.begin.native_session_id}`);if(error.status===413)(this.state.set_aside??={})[`${item.begin.harness}:${item.begin.native_session_id}`]={at:this.clock.nowMs(),rule_revision:item.begin.rule_revision??null,reason:error.message};process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id} the server refused${error.status===413?' (one message is larger than a request may be)':' as malformed'}: ${error.message}\n`);this.unqueue();this.save();continue;}
|
|
1521
|
+
const rejected=await this.ruleRejected(item,error);
|
|
1522
|
+
try{this.requireCaptureTransfer(item);}catch(changed){if(['redaction_basis_changed','redaction_capture_unsafe'].includes(changed.code)){this.holdQueuedCapture(item,changed);continue;}throw changed;}
|
|
1523
|
+
if(!rejected)throw error;this.lastError=error;process.stderr.write(`supercode teams: dropped a queued capture of ${item.begin.harness} session ${item.begin.native_session_id}; its sync rule changed or was removed: ${error.message}\n`);this.unqueue();this.save();continue;}
|
|
1524
|
+
// A commit already issued may finish after custody changes. Its receipt advances the origin, but its local
|
|
1525
|
+
// original remains held and owed; registration gives this source no authority to purge the published revision.
|
|
1526
|
+
let current=true;
|
|
1527
|
+
try{this.requireCaptureTransfer(item);}catch(error){if(!['redaction_basis_changed','redaction_capture_unsafe'].includes(error.code))throw error;this.holdQueuedCapture(item,error);current=false;}
|
|
1528
|
+
const key=`${item.begin.harness}:${item.begin.native_session_id}`,prior=this.origins[key]??{},next={...prior,generation:item.begin.generation,sequence:item.begin.sequence,digest:item.commit.digest,observed_keys:item.observed?.keys??prior.observed_keys,observed_digests:item.observed?.digests??prior.observed_digests,redaction_basis:item.redactionBasis??null,last_rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode};if(item.clearBaseline){delete next.baseline;delete next.baseline_digests;delete next.baseline_tool_tail;}if(item.nativeDeferred)next.native_deferred=true;else delete next.native_deferred;if(item.windowStart)next.window_start=item.windowStart;else delete next.window_start;if(!item.begin.carry)delete next.carry_refused;if(item.stamp&&item.begin.rule_id)next.capture_stamps={...prior.capture_stamps,[item.begin.rule_id]:{stamp:item.stamp,rule_revision:item.begin.rule_revision,backfill_mode:item.backfillMode}};this.origins[key]=next;
|
|
1529
|
+
if(current){
|
|
1530
|
+
if(receipt?.capture_id===item.begin.capture_id&&receipt.digest===item.commit.digest)this.redactionHolds.settle(key,item,receipt,names=>this.releaseSpills(names));
|
|
1531
|
+
this.updateRedactionHolds();this.unqueue();this.clearSessionProblem(item.begin.harness,item.begin.native_session_id);
|
|
1532
|
+
}
|
|
1533
|
+
this.save();
|
|
1469
1534
|
// The commit is what records (or settles) a native bundle still owed: its wait is armed here.
|
|
1470
1535
|
this.armNativeIdle(key);latest=receipt;}this.flushBackoff=null;return latest;}
|
|
1471
1536
|
stop(){void this.redactor?.close();this.daemon?.off?.('sessionActivityEvent',this.activityEvent);this.daemon?.off?.('sessionIndexEvent',this.indexEvent);this.daemon?.off?.('exit',this.daemonExit);clearTimeout(this.flushTimer);if(this.activitySubscription)void this.daemon?.unsubscribeSessionActivity?.(this.activitySubscription).catch(()=>{});for(const feed of this.feeds?.values()??[])void this.daemon?.unsubscribeSessionIndex?.(feed.subscription).catch(()=>{});for(const timer of this.idleTimers?.values()??[])clearTimeout(timer);try{this.flushState();}catch{}if(this.timer)clearInterval(this.timer);if(this.heartbeatTimer)clearInterval(this.heartbeatTimer);this.abort.abort();return this.nativeEffects.close();}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// Stale captures stay on this machine while a fresh source observation replaces them. Only headers stay in memory;
|
|
2
|
+
// the original parts and artifact references remain in owner-only files and are never a send queue.
|
|
3
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, unlinkSync } from 'node:fs';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
|
|
6
|
+
export class RedactionHolds {
|
|
7
|
+
constructor({ stateDir, atomic, saved = {} }) {
|
|
8
|
+
this.directory = join(stateDir, 'redaction-held'); this.outbox = join(stateDir, 'outbox');
|
|
9
|
+
this.atomic = atomic; this.items = new Map(); this.unread = [];
|
|
10
|
+
mkdirSync(this.directory, { recursive: true, mode: 0o700 });
|
|
11
|
+
for (const name of readdirSync(this.directory).filter(name => /^\d+\.json$/.test(name)).sort()) {
|
|
12
|
+
try {
|
|
13
|
+
const item = JSON.parse(readFileSync(join(this.directory, name), 'utf8'));
|
|
14
|
+
const header = this.header(item, name), prior = saved[name];
|
|
15
|
+
if (prior?.capture_id === header.capture_id && ['native_owed', 'cleanup', 'blocked'].includes(prior.state)) Object.assign(header, { state: prior.state, replacement: prior.replacement });
|
|
16
|
+
this.items.set(name, header);
|
|
17
|
+
} catch (error) { this.unread.push({ name, error: error?.code ?? 'unreadable_hold' }); }
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
header(item, name) {
|
|
22
|
+
return { name, capture_id: item.begin.capture_id, key: `${item.begin.harness}:${item.begin.native_session_id}`,
|
|
23
|
+
harness: item.begin.harness, session_id: item.begin.native_session_id, rule_id: item.begin.rule_id,
|
|
24
|
+
rule_revision: item.begin.rule_revision, at: item.redaction_hold?.at ?? null, state: item.redaction_hold?.code==='redaction_capture_unsafe'?'blocked':'owed',
|
|
25
|
+
stamp: item.stamp??null, basis: item.redactionBasis, code: item.redaction_hold?.code??'redaction_basis_changed',
|
|
26
|
+
native: Boolean(item.artifacts?.length || item.nativeDeferred),
|
|
27
|
+
spills: (item.artifacts ?? []).filter(artifact => artifact.spill).map(artifact => artifact.spill.file) };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
hold(item, name, at, code) {
|
|
31
|
+
// Persist the no-send flag before moving: a crash between these steps restarts with a flagged outbox item, which
|
|
32
|
+
// the transfer guard also refuses. The move transfers its spill ownership; it must not release any references.
|
|
33
|
+
item.redaction_hold = { at, code };
|
|
34
|
+
this.atomic(join(this.outbox, name), JSON.stringify(item));
|
|
35
|
+
if(existsSync(join(this.directory,name)))throw new Error('a custody-held capture already owns this filename; original outbox item retained');
|
|
36
|
+
renameSync(join(this.outbox, name), join(this.directory, name));
|
|
37
|
+
this.items.set(name, this.header(item, name));
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
needed(key) { return [...this.items.values()].some(item => item.key === key && ['owed','blocked'].includes(item.state)); }
|
|
41
|
+
blocked(key, stamp, revision, basis) { return [...this.items.values()].some(item => item.key === key && item.state === 'blocked' && item.stamp === stamp && item.rule_revision === revision && item.basis === basis); }
|
|
42
|
+
spills() { return [...this.items.values()].flatMap(item => item.spills); }
|
|
43
|
+
saved() { return Object.fromEntries(this.items); }
|
|
44
|
+
status() { return [...this.items.values()].map(({ capture_id, harness, session_id, rule_id, at, state, replacement }) => ({ capture_id, harness, session_id, rule_id, at, state, replacement })); }
|
|
45
|
+
|
|
46
|
+
settle(key, capture, receipt, releaseSpills) {
|
|
47
|
+
for (const [name, held] of this.items) {
|
|
48
|
+
if (held.key !== key || held.rule_id !== capture.begin.rule_id || held.rule_revision !== capture.begin.rule_revision) continue;
|
|
49
|
+
if(capture.begin.carry&&['owed','blocked'].includes(held.state))continue;
|
|
50
|
+
held.replacement = receipt.capture_id;
|
|
51
|
+
// A receipt for messages alone does not discharge a held native bundle. The normal native-deferred retry
|
|
52
|
+
// remains owed; its original files stay held until a replacement artifact is acknowledged too.
|
|
53
|
+
if (held.native && !(capture.artifacts?.length)) { held.state = 'native_owed'; continue; }
|
|
54
|
+
try { unlinkSync(join(this.directory, name)); }
|
|
55
|
+
catch (error) { if (error?.code !== 'ENOENT') { held.state = 'cleanup'; continue; } }
|
|
56
|
+
this.items.delete(name); releaseSpills(held.spills);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
package/connector/redactor.mjs
CHANGED
|
@@ -19,16 +19,22 @@ if (!isMainThread && parentPort) {
|
|
|
19
19
|
return changed;
|
|
20
20
|
};
|
|
21
21
|
const { performance } = await import('node:perf_hooks');
|
|
22
|
-
|
|
22
|
+
const holdsCustody = (value, custody) => {
|
|
23
|
+
if (typeof value === 'string') return custody.some(({ value: secret }) => value.includes(secret));
|
|
24
|
+
if (Array.isArray(value)) return value.some((item) => holdsCustody(item, custody));
|
|
25
|
+
return value && typeof value === 'object' && Object.entries(value).some(([key, item]) => holdsCustody(key, custody) || holdsCustody(item, custody));
|
|
26
|
+
};
|
|
27
|
+
parentPort.on('message', ({ id, kind, items, basis, fresh }) => {
|
|
23
28
|
// Its own busy time rides back with each answer: what redaction costs, apart from the connector's other work.
|
|
24
29
|
const started = performance.now();
|
|
25
|
-
const requireBasis=()=>{if(basis!==undefined&&redactionBasis()!==basis)throw Object.assign(new Error('redaction worker basis differs from the capture; capture needs fresh redaction'),{code:'redaction_basis_changed'});};
|
|
30
|
+
const requireBasis=()=>{if(basis!==undefined&&redactionBasis(undefined,{fresh})!==basis)throw Object.assign(new Error('redaction worker basis differs from the capture; capture needs fresh redaction'),{code:'redaction_basis_changed'});};
|
|
26
31
|
try {
|
|
27
32
|
// All strings in this checked batch use the same custody observation, even if a batch crosses its refresh interval.
|
|
28
|
-
const snapshot=basis===undefined?null:redactionSnapshot();
|
|
33
|
+
const snapshot=basis===undefined?null:redactionSnapshot(undefined,{fresh});
|
|
29
34
|
if(snapshot&&snapshot.basis!==basis)throw Object.assign(new Error('redaction worker basis differs from the capture; capture needs fresh redaction'),{code:'redaction_basis_changed'});
|
|
30
35
|
requireBasis();
|
|
31
|
-
|
|
36
|
+
if(kind==='custody_assert'&&(!snapshot||items.some(text=>holdsCustody(JSON.parse(text),snapshot.values))))throw Object.assign(new Error('capture still contains a registered custody value; fresh capture required'),{code:'redaction_capture_unsafe'});
|
|
37
|
+
const answer = kind === 'custody_assert' ? [] : kind === 'canonical' ? changedOf(items,snapshot?.values) : items.map((item) => (kind === 'texts' ? redactText(item,undefined,snapshot?.values) : redactValue(item,undefined,snapshot?.values)));
|
|
32
38
|
requireBasis();
|
|
33
39
|
parentPort.postMessage({ id, items: answer, busy: performance.now() - started });
|
|
34
40
|
} catch (error) { parentPort.postMessage({ id, error: String(error?.message ?? error), code: error?.code }); }
|
|
@@ -45,7 +51,7 @@ export class Redactor {
|
|
|
45
51
|
const waiter = this.waiting.get(id); if (!waiter) return;
|
|
46
52
|
this.waiting.delete(id);
|
|
47
53
|
this.workerMs += busy ?? 0;
|
|
48
|
-
if (error) waiter.reject(Object.assign(new Error(`redaction failed: ${error}`), { code:
|
|
54
|
+
if (error) waiter.reject(Object.assign(new Error(`redaction failed: ${error}`), { code: ['redaction_basis_changed','redaction_capture_unsafe','custody_unreadable'].includes(code) ? code : 'redaction_failed' }));
|
|
49
55
|
else waiter.resolve(items);
|
|
50
56
|
if (!this.waiting.size) worker.unref();
|
|
51
57
|
});
|
|
@@ -63,13 +69,14 @@ export class Redactor {
|
|
|
63
69
|
|
|
64
70
|
/** `items` redacted, in order: `kind` 'values' (JSON values, every string in them) or 'texts'; for 'canonical' (canonical
|
|
65
71
|
* JSON texts) only the changed ones, as `{ index, value, digest }`. An optional basis must match the worker before
|
|
66
|
-
* and after redaction; a mismatch rejects the request rather than returning stale decisions.
|
|
67
|
-
|
|
72
|
+
* and after redaction; a mismatch rejects the request rather than returning stale decisions. `custody_assert`
|
|
73
|
+
* accepts serialized request bodies and refuses a custody value anywhere in their parsed strings or keys. */
|
|
74
|
+
run(kind, items, { basis, fresh = false } = {}) {
|
|
68
75
|
if (!items.length) return Promise.resolve([]);
|
|
69
76
|
if (!this.worker) this.start();
|
|
70
77
|
const id = this.next++;
|
|
71
78
|
this.worker.ref();
|
|
72
|
-
return new Promise((resolve, reject) => { this.waiting.set(id, { resolve, reject }); this.worker.postMessage({ id, kind, items, basis }); });
|
|
79
|
+
return new Promise((resolve, reject) => { this.waiting.set(id, { resolve, reject }); this.worker.postMessage({ id, kind, items, basis, fresh }); });
|
|
73
80
|
}
|
|
74
81
|
|
|
75
82
|
close() { const worker = this.worker; this.worker = null; return worker ? worker.terminate().then(() => {}) : Promise.resolve(); }
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-teams",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.159",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -75,7 +75,7 @@
|
|
|
75
75
|
"@volter/supercode-health": "^0.1.0",
|
|
76
76
|
"@volter/supercode-terminal": "^0.2.54",
|
|
77
77
|
"@volter/supercode-trace": "^0.1.0",
|
|
78
|
-
"@volter/teams": "0.2.
|
|
78
|
+
"@volter/teams": "0.2.44",
|
|
79
79
|
"@xterm/addon-serialize": "^0.14.0",
|
|
80
80
|
"@xterm/headless": "^6.0.0",
|
|
81
81
|
"protobufjs": "8.8.0",
|