@volter/supercode-teams 0.3.142 → 0.3.144
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/bin/workspace.mjs +6 -0
- package/contracts/index.d.ts +1 -0
- package/contracts/index.mjs +1 -1
- package/package.json +2 -2
- package/server/arc2-transactions.mjs +9 -1
- package/server/capture-prefixes.mjs +113 -0
- package/server/capture-transactions.mjs +27 -1
- package/server/event-routes.mjs +1 -1
- package/server/harness-routes.mjs +18 -21
- package/server/index.mjs +23 -6
- package/server/store-core.mjs +3 -0
- package/server/work-meter.mjs +84 -0
- package/worker/index.mjs +82 -20
package/README.md
CHANGED
|
@@ -188,3 +188,11 @@ Only the session custodian can grant this binding, in person or through an app t
|
|
|
188
188
|
no session-backed channel is created. The agent DM uses the exact resource `${ROOM}:dm:${PRINCIPAL}`.
|
|
189
189
|
`unbind` accepts the same flags. `--conversation` remains available for existing grants; it is mutually exclusive
|
|
190
190
|
with `--agent`. Opening a DM reads the existing session; sending a turn is a separate action.
|
|
191
|
+
|
|
192
|
+
## Read server work diagnostics
|
|
193
|
+
|
|
194
|
+
An administrator can run `supercode teams storage work --json` to read recent handler records and retained slow work. Records name normalized HTTP routes, catalog refreshes, transitions and maintenance steps, with SQL call and cursor scan/write counts. The Worker retains up to 32 slow records across a restart. Elapsed milliseconds are not CPU time; hosted per-query timing is unavailable. [Recorded private Worker proof](../../docs/evidence/teams-handler-work-2026-10-07.md).
|
|
195
|
+
|
|
196
|
+
Diagnostic persistence writes at most once every five seconds; the latest window can be lost on restart. A malformed retained history does not stop activation, and the report names that condition in `persistence.restore_error`.
|
|
197
|
+
|
|
198
|
+
Carried capture commits use durable prefix checkpoints. An old session may first return HTTP 503 `carry_checkpoint_pending` with `Retry-After: 1`; retry the same staged commit while the host prepares bounded batches. A ready commit verifies at most 63 old message bodies. `carry_mismatch` refuses a forged prefix, and `carry_checkpoint_unavailable` refuses an unsupported internal checkpoint. The canonical wire digest is unchanged. See [the design](../../docs/adr/0024-capture-prefix-checkpoints.md) and [the measured proof](../../docs/evidence/teams-capture-prefixes-2026-10-07.md).
|
package/bin/workspace.mjs
CHANGED
|
@@ -95,6 +95,11 @@ export async function workspaceMain(argv = process.argv.slice(2), env = process.
|
|
|
95
95
|
if (verb === 'sessions') return sessionsCommand(sub, command);
|
|
96
96
|
if (verb === 'calendar') return calendarCommand(sub, { ...command, teamPath });
|
|
97
97
|
if (verb === 'settings') return settingsCommand(sub, command);
|
|
98
|
+
if (verb === 'storage') {
|
|
99
|
+
only(flags, ['json', 'help', 'context']);requireArgs(rest, 0, 0, 'storage work');
|
|
100
|
+
if(sub!=='work')usage('storage work');
|
|
101
|
+
return output(await client.request('GET',teamPath(context,'/storage/work')));
|
|
102
|
+
}
|
|
98
103
|
if (verb === 'members') return membersCommand(sub, command);
|
|
99
104
|
if (verb === 'groups') return groupsCommand(sub, command);
|
|
100
105
|
if (verb === 'sources') return sourcesCommand(sub, command);
|
|
@@ -1434,6 +1439,7 @@ function printHelp() {
|
|
|
1434
1439
|
identity link
|
|
1435
1440
|
identities observations|sightings|resolve|observe|declare-person|link|infer|revoke|revoke-inference --body-file FILE
|
|
1436
1441
|
server start [--data-dir DIR] [--listen HOST:PORT] [--public-origin ORIGIN] [--init-team NAME] | server token
|
|
1442
|
+
storage work [--context NAME] [--json] administrators: recent handler work and retained slow records
|
|
1437
1443
|
publish pane|root TARGET --room ROOM --workplace ORIGIN --organization ORG [--scope view|control] [--name] [--key] [--machine] [--token-stdin] [--record-only]
|
|
1438
1444
|
unpublish SURFACE --room ROOM --workplace ORIGIN --organization ORG [--machine] [--token-stdin] [--record-only|--workplace-only]
|
|
1439
1445
|
connect|disconnect|status|fold (fold: the connector's state back into connector.json for an older daemon, with it stopped)
|
package/contracts/index.d.ts
CHANGED
|
@@ -23,6 +23,7 @@ export type ErrorCode =
|
|
|
23
23
|
| 'forbidden' | 'unsupported' | 'machine_offline' | 'controller_busy' | 'conflict'
|
|
24
24
|
| 'resync_required' | 'revision_mismatch' | 'idempotency_conflict' | 'source_revoked'
|
|
25
25
|
| 'source_paused' | 'session_deleted' | 'batch_gap' | 'capture_conflict'
|
|
26
|
+
| 'carry_mismatch' | 'carry_checkpoint_pending' | 'carry_checkpoint_unavailable'
|
|
26
27
|
| 'fidelity_unavailable' | 'rate_limited' | 'request_too_large' | 'internal_error';
|
|
27
28
|
export interface Failure {
|
|
28
29
|
version: 1;
|
package/contracts/index.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { ContractError, canonicalJson, sha256, validationFailure } from '@volter/teams/server/wire';
|
|
2
2
|
export { ContractError, canonicalJson, validationFailure };
|
|
3
3
|
const ID = /^(srv|team|usr|svc|grp|ses|msg|src|mach|bind|cap|upload|grant|map|op|cont|inv|cred|req|root|inc|rule|att|evt|audit)_[0-9a-f]{32}$/;
|
|
4
|
-
export const ERROR_CODES = Object.freeze(['invalid_request','ambiguous_reference','authentication_required','not_found','forbidden','unsupported','machine_offline','controller_busy','conflict','resync_required','revision_mismatch','idempotency_conflict','source_revoked','source_paused','session_deleted','batch_gap','capture_conflict','fidelity_unavailable','rate_limited','request_too_large','internal_error']);
|
|
4
|
+
export const ERROR_CODES = Object.freeze(['invalid_request','ambiguous_reference','authentication_required','not_found','forbidden','unsupported','machine_offline','controller_busy','conflict','resync_required','revision_mismatch','idempotency_conflict','source_revoked','source_paused','session_deleted','batch_gap','capture_conflict','carry_mismatch','carry_checkpoint_pending','carry_checkpoint_unavailable','fidelity_unavailable','rate_limited','request_too_large','internal_error']);
|
|
5
5
|
export const OPERATIONS = Object.freeze(['me','teams_list','teams_create','list','search','read','follow','descriptor','access','grant','invite','accept_invite','source_create','source_config','enroll','heartbeat','machine_configuration','capture_begin','capture_part','capture_commit','correlate','live_grant','attach','prepare','launch','export']);
|
|
6
6
|
const object = (value, name) => { if (!value || typeof value !== 'object' || Array.isArray(value)) throw new ContractError(`${name} must be an object`); return value; };
|
|
7
7
|
const exact = (value, allowed, name) => { for (const key of Object.keys(object(value, name))) if (!allowed.includes(key)) throw new ContractError(`${name}.${key} is unknown`); };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-teams",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.144",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -70,7 +70,7 @@
|
|
|
70
70
|
"@volter/supercode-health": "^0.1.0",
|
|
71
71
|
"@volter/supercode-terminal": "^0.2.54",
|
|
72
72
|
"@volter/supercode-trace": "^0.1.0",
|
|
73
|
-
"@volter/teams": "0.2.
|
|
73
|
+
"@volter/teams": "0.2.39",
|
|
74
74
|
"@xterm/addon-serialize": "^0.14.0",
|
|
75
75
|
"@xterm/headless": "^6.0.0",
|
|
76
76
|
"protobufjs": "8.8.0",
|
|
@@ -34,7 +34,15 @@ export function correlateSessions(db,args){
|
|
|
34
34
|
if(members.some(row=>!manageable(row)))fail('caller must manage every constituent','forbidden',403);
|
|
35
35
|
if(members.some(row=>row.deleted_at!=null)){
|
|
36
36
|
const deletedAt=members.map(row=>row.deleted_at).filter(value=>value!=null).reduce((a,b)=>Math.min(a,b)),team=get('SELECT journal_revision FROM teams WHERE id=?',[args.teamId]),revision=team.journal_revision+1,canonical=rootOf(members.reduce((best,row)=>row.created_at<best.created_at?row:best,members[0]));
|
|
37
|
-
for(const member of members){
|
|
37
|
+
for(const member of members){
|
|
38
|
+
run('UPDATE sessions SET deleted_at=?,revision=? WHERE team_id=? AND id=?',[deletedAt,revision,args.teamId,member.id]);
|
|
39
|
+
run('UPDATE origins SET deleted_at=? WHERE team_id=? AND session_id=?',[deletedAt,args.teamId,member.id]);
|
|
40
|
+
// Correlation propagates a tombstone without going through DELETE /sessions.
|
|
41
|
+
// Retire only metadata here; the alarm removes each generation in bounded batches.
|
|
42
|
+
run('INSERT OR IGNORE INTO capture_prefix_garbage SELECT team_id,session_id,source_id,generation FROM capture_prefix_state WHERE team_id=? AND session_id=?',[args.teamId,member.id]);
|
|
43
|
+
run('DELETE FROM capture_prefix_state WHERE team_id=? AND session_id=?',[args.teamId,member.id]);
|
|
44
|
+
run('DELETE FROM capture_prefix_warm_queue WHERE team_id=? AND session_id=?',[args.teamId,member.id]);
|
|
45
|
+
}
|
|
38
46
|
run('UPDATE teams SET journal_revision=?,auth_epoch=auth_epoch+1 WHERE id=?',[revision,args.teamId]);run('INSERT INTO journal(team_id,revision,session_id,kind,created_at) VALUES (?,?,?,?,?)',[args.teamId,revision,canonical.id,'delete',now]);run('INSERT INTO audit(team_id,id,actor_id,operation,target_id,outcome,revision,created_at) VALUES (?,?,?,?,?,?,?,?)',[args.teamId,args.auditId,args.principalId,args.operation,args.correlationId,'denied',revision,now]);return{tombstoned:true,revision,canonicalId:canonical.id};
|
|
39
47
|
}
|
|
40
48
|
const viewBySource=new Map();for(const row of members)for(const version of all('SELECT * FROM session_versions WHERE team_id=? AND session_id=? ORDER BY revision',[args.teamId,row.id])){const sourceId=JSON.parse(version.source_projection_json).source_id??'';if(sourceId&&(!viewBySource.has(sourceId)||version.revision>viewBySource.get(sourceId).revision))viewBySource.set(sourceId,{...row,...version,source_id:sourceId});}const views=[...viewBySource.values()];if(!views.length)fail('constituent projection missing','conflict',409);
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { sha256 } from '@noble/hashes/sha2.js';
|
|
3
|
+
import { canonicalJson } from '@volter/teams/server/wire';
|
|
4
|
+
import { MESSAGE_AT } from '@volter/teams/server/migrations';
|
|
5
|
+
|
|
6
|
+
// This is a versioned, internal continuation of the exactly pinned noble 2.4.0
|
|
7
|
+
// SHA-256 engine, not a wire digest change. Its get/set state and HashMD metadata
|
|
8
|
+
// are stored together; a digest closes a clone with ']', leaving the state resumable.
|
|
9
|
+
const FORMAT = 'sha256-noble-2.4.0-v1';
|
|
10
|
+
const encoder = new TextEncoder();
|
|
11
|
+
const CHECKPOINT_MESSAGES = 64, CHECKPOINT_BYTES = 64 * 1024;
|
|
12
|
+
const WARM_MESSAGES = 256, WARM_BYTES = 1024 * 1024, GARBAGE_ROWS = 512;
|
|
13
|
+
const error = (status, code, message) => Object.assign(new Error(message), { status, code });
|
|
14
|
+
export function startPrefix() { return sha256.create().update(encoder.encode('[')); }
|
|
15
|
+
export function savePrefix(hash) {
|
|
16
|
+
return JSON.stringify({ format: FORMAT, words: hash.get(), length: hash.length, pos: hash.pos, buffer: Buffer.from(hash.buffer.subarray(0, hash.pos)).toString('base64') });
|
|
17
|
+
}
|
|
18
|
+
export function loadPrefix(state) {
|
|
19
|
+
const value = JSON.parse(state);
|
|
20
|
+
if (value.format !== FORMAT || !Array.isArray(value.words) || value.words.length !== 8 || value.words.some(x => !Number.isInteger(x) || x < -2147483648 || x > 2147483647) || !Number.isSafeInteger(value.length) || value.length < 1 || !Number.isInteger(value.pos) || value.pos < 0 || value.pos >= 64 || value.pos !== value.length % 64 || typeof value.buffer !== 'string')
|
|
21
|
+
throw error(503, 'carry_checkpoint_unavailable', 'capture prefix checkpoint has an unsupported format');
|
|
22
|
+
const bytes = Buffer.from(value.buffer, 'base64');
|
|
23
|
+
if (bytes.length !== value.pos) throw error(503, 'carry_checkpoint_unavailable', 'capture prefix checkpoint has an invalid buffer');
|
|
24
|
+
const hash = sha256.create(); hash.set(...value.words); hash.length = value.length; hash.pos = value.pos; hash.buffer.set(bytes); return hash;
|
|
25
|
+
}
|
|
26
|
+
export function prefixDigest(hash) { return Buffer.from(hash.clone().update(encoder.encode(']')).digest()).toString('hex'); }
|
|
27
|
+
export function addPrefixMessage(hash, count, message) {
|
|
28
|
+
if (count) hash.update(encoder.encode(','));
|
|
29
|
+
hash.update(encoder.encode(canonicalJson({ native_key: message.native_key, recorded_at: message.recorded_at ?? null, value: message.value })));
|
|
30
|
+
}
|
|
31
|
+
const identity = row => [row.team_id, row.session_id, row.source_id, row.generation];
|
|
32
|
+
export function writeCheckpoint(run, state, hash, count) {
|
|
33
|
+
run('INSERT OR REPLACE INTO capture_prefix_checkpoints(team_id,session_id,source_id,generation,count,digest,sha_state) VALUES (?,?,?,?,?,?,?)', [...identity(state), count, prefixDigest(hash), savePrefix(hash)]);
|
|
34
|
+
}
|
|
35
|
+
export function shouldCheckpoint(state, hash, count) { return count - state.checkpoint_count >= CHECKPOINT_MESSAGES || hash.length - state.checkpoint_bytes >= CHECKPOINT_BYTES; }
|
|
36
|
+
|
|
37
|
+
/** Retire by generation: changing a session does not delete its whole prefix ledger on the request loop. */
|
|
38
|
+
export function retirePrefix({ get, run }, teamId, sessionId, sourceId) {
|
|
39
|
+
const state = get('SELECT generation FROM capture_prefix_state WHERE team_id=? AND session_id=? AND source_id=?', [teamId, sessionId, sourceId]);
|
|
40
|
+
if (!state) return;
|
|
41
|
+
run('INSERT OR IGNORE INTO capture_prefix_garbage VALUES (?,?,?,?)', [teamId, sessionId, sourceId, state.generation]);
|
|
42
|
+
run('DELETE FROM capture_prefix_state WHERE team_id=? AND session_id=? AND source_id=?', [teamId, sessionId, sourceId]);
|
|
43
|
+
run('DELETE FROM capture_prefix_warm_queue WHERE team_id=? AND session_id=? AND source_id=?', [teamId, sessionId, sourceId]);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Verify a prepared prefix with at most 63 message reads / one checkpoint byte interval. Never a full scan. */
|
|
47
|
+
export function resolveCapturePrefix({ get, run }, { teamId, sessionId, sourceId, revision, count, digest, sentKeys }) {
|
|
48
|
+
let state = get('SELECT * FROM capture_prefix_state WHERE team_id=? AND session_id=? AND source_id=?', [teamId, sessionId, sourceId]);
|
|
49
|
+
if (!state || state.revision !== revision) {
|
|
50
|
+
retirePrefix({ get, run }, teamId, sessionId, sourceId);
|
|
51
|
+
const hash = startPrefix();
|
|
52
|
+
state = { team_id: teamId, session_id: sessionId, source_id: sourceId, revision, generation: randomUUID(), count: 0, target_count: count, sha_state: savePrefix(hash), checkpoint_count: 0, checkpoint_bytes: hash.length, invalid: 0 };
|
|
53
|
+
run('INSERT INTO capture_prefix_state VALUES (?,?,?,?,?,?,?,?,?,?,?)', [teamId, sessionId, sourceId, revision, state.generation, 0, count, state.sha_state, 0, hash.length, 0]);
|
|
54
|
+
writeCheckpoint(run, state, hash, 0);
|
|
55
|
+
}
|
|
56
|
+
if (state.invalid) throw error(409, 'carry_mismatch', 'the carried messages are not what this session holds; send the capture whole');
|
|
57
|
+
if (state.count < count) {
|
|
58
|
+
run('UPDATE capture_prefix_state SET target_count=MAX(target_count,?) WHERE team_id=? AND session_id=? AND source_id=?', [count, teamId, sessionId, sourceId]);
|
|
59
|
+
run('INSERT OR IGNORE INTO capture_prefix_warm_queue(team_id,session_id,source_id) VALUES (?,?,?)', [teamId, sessionId, sourceId]);
|
|
60
|
+
return { pending: true, prepared: state.count, requested: count };
|
|
61
|
+
}
|
|
62
|
+
const checkpoint = get('SELECT * FROM capture_prefix_checkpoints WHERE team_id=? AND session_id=? AND source_id=? AND generation=? AND count<=? ORDER BY count DESC LIMIT 1', [...identity(state), count]);
|
|
63
|
+
if (!checkpoint || count - checkpoint.count >= CHECKPOINT_MESSAGES) throw error(503, 'carry_checkpoint_unavailable', 'capture prefix checkpoint is incomplete');
|
|
64
|
+
const hash = loadPrefix(checkpoint.sha_state);
|
|
65
|
+
for (let ordinal = checkpoint.count; ordinal < count; ordinal++) {
|
|
66
|
+
const row = get(`SELECT ordinal,source_id,native_key,recorded_at,value_json FROM messages m WHERE m.team_id=? AND m.session_id=? AND m.ordinal=? AND ${MESSAGE_AT()} LIMIT 1`, [teamId, sessionId, ordinal, revision, revision]);
|
|
67
|
+
if (!row || row.source_id !== sourceId) throw error(409, 'carry_mismatch', 'the carried messages are not what this session holds; send the capture whole');
|
|
68
|
+
addPrefixMessage(hash, ordinal, { ...row, value: JSON.parse(row.value_json) });
|
|
69
|
+
}
|
|
70
|
+
if (prefixDigest(hash) !== digest) throw error(409, 'carry_mismatch', 'the carried messages are not what this session holds; send the capture whole');
|
|
71
|
+
for (const key of sentKeys) {
|
|
72
|
+
const held = get('SELECT ordinal FROM capture_prefix_keys WHERE team_id=? AND session_id=? AND source_id=? AND generation=? AND native_key=?', [...identity(state), key]);
|
|
73
|
+
if (held && held.ordinal < count) throw error(409, 'capture_conflict', 'duplicate native message key');
|
|
74
|
+
}
|
|
75
|
+
return { pending: false, generation: state.generation, sha_state: savePrefix(hash), checkpoint_count: state.checkpoint_count <= count ? state.checkpoint_count : checkpoint.count, checkpoint_bytes: state.checkpoint_count <= count ? state.checkpoint_bytes : loadPrefix(checkpoint.sha_state).length };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Stop starting messages at the row/byte bound, independent of a frozen clock.
|
|
79
|
+
* A single message may cross the byte bound; its size is bounded by capture admission. */
|
|
80
|
+
export function warmCapturePrefixes({ get, run }) {
|
|
81
|
+
let messages = 0, bytes = 0;
|
|
82
|
+
const garbage = get('SELECT * FROM capture_prefix_garbage ORDER BY rowid LIMIT 1');
|
|
83
|
+
if (garbage) {
|
|
84
|
+
for (const table of ['capture_prefix_keys', 'capture_prefix_checkpoints']) run(`DELETE FROM ${table} WHERE rowid IN (SELECT rowid FROM ${table} WHERE team_id=? AND session_id=? AND source_id=? AND generation=? LIMIT ${GARBAGE_ROWS})`, identity(garbage));
|
|
85
|
+
if (!get('SELECT 1 FROM capture_prefix_keys WHERE team_id=? AND session_id=? AND source_id=? AND generation=? LIMIT 1', identity(garbage)) && !get('SELECT 1 FROM capture_prefix_checkpoints WHERE team_id=? AND session_id=? AND source_id=? AND generation=? LIMIT 1', identity(garbage))) run('DELETE FROM capture_prefix_garbage WHERE team_id=? AND session_id=? AND source_id=? AND generation=?', identity(garbage));
|
|
86
|
+
}
|
|
87
|
+
const queued = get('SELECT * FROM capture_prefix_warm_queue ORDER BY ticket LIMIT 1');
|
|
88
|
+
const state = queued ? get('SELECT * FROM capture_prefix_state WHERE team_id=? AND session_id=? AND source_id=?', [queued.team_id, queued.session_id, queued.source_id]) : null;
|
|
89
|
+
if (queued) run('DELETE FROM capture_prefix_warm_queue WHERE ticket=?', [queued.ticket]);
|
|
90
|
+
if (state) {
|
|
91
|
+
const session = get('SELECT s.revision,s.deleted_at,src.state source_state FROM sessions s JOIN sources src ON src.team_id=s.team_id AND src.id=? WHERE s.team_id=? AND s.id=?', [state.source_id, state.team_id, state.session_id]);
|
|
92
|
+
if (!session || session.source_state !== 'active' || session.deleted_at != null || session.revision !== state.revision) retirePrefix({ get, run }, state.team_id, state.session_id, state.source_id);
|
|
93
|
+
else {
|
|
94
|
+
const hash = loadPrefix(state.sha_state), startBytes = hash.length;
|
|
95
|
+
if (state.count !== state.checkpoint_count) run('DELETE FROM capture_prefix_checkpoints WHERE team_id=? AND session_id=? AND source_id=? AND generation=? AND count=?', [...identity(state), state.count]);
|
|
96
|
+
while (state.count < state.target_count && messages < WARM_MESSAGES && hash.length - startBytes < WARM_BYTES) {
|
|
97
|
+
const row = get(`SELECT ordinal,source_id,native_key,recorded_at,value_json FROM messages m WHERE m.team_id=? AND m.session_id=? AND m.ordinal=? AND ${MESSAGE_AT()} LIMIT 1`, [state.team_id, state.session_id, state.count, state.revision, state.revision]);
|
|
98
|
+
if (!row || row.source_id !== state.source_id || get('SELECT 1 FROM capture_prefix_keys WHERE team_id=? AND session_id=? AND source_id=? AND generation=? AND native_key=?', [...identity(state), row?.native_key ?? ''])) { state.invalid = 1; break; }
|
|
99
|
+
addPrefixMessage(hash, state.count, { ...row, value: JSON.parse(row.value_json) });
|
|
100
|
+
run('INSERT INTO capture_prefix_keys VALUES (?,?,?,?,?,?)', [...identity(state), row.native_key, state.count]);
|
|
101
|
+
state.count++; messages++;
|
|
102
|
+
if (shouldCheckpoint(state, hash, state.count)) { writeCheckpoint(run, state, hash, state.count); state.checkpoint_count = state.count; state.checkpoint_bytes = hash.length; }
|
|
103
|
+
}
|
|
104
|
+
// Keep the exact last state too; it lets a full-prefix append avoid every body read.
|
|
105
|
+
if (state.count === state.target_count) writeCheckpoint(run, state, hash, state.count);
|
|
106
|
+
run('UPDATE capture_prefix_state SET count=?,sha_state=?,checkpoint_count=?,checkpoint_bytes=?,invalid=? WHERE team_id=? AND session_id=? AND source_id=? AND generation=?', [state.count, savePrefix(hash), state.checkpoint_count, state.checkpoint_bytes, state.invalid, ...identity(state)]);
|
|
107
|
+
bytes = hash.length - startBytes;
|
|
108
|
+
// FIFO across sessions, including ones whose warming began after this one.
|
|
109
|
+
if (!state.invalid && state.count < state.target_count) run('INSERT OR IGNORE INTO capture_prefix_warm_queue(team_id,session_id,source_id) VALUES (?,?,?)', [state.team_id, state.session_id, state.source_id]);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return { messages, bytes, backlog: !!get('SELECT 1 FROM capture_prefix_warm_queue LIMIT 1') || !!get('SELECT 1 FROM capture_prefix_garbage LIMIT 1') };
|
|
113
|
+
}
|
|
@@ -1,15 +1,36 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { MESSAGE_AT, OPEN_REVISION } from '@volter/teams/server/migrations';
|
|
3
3
|
import { normalizeSearch, textFromContent } from './catalog.mjs';
|
|
4
|
+
import { addPrefixMessage, loadPrefix, retirePrefix, savePrefix, shouldCheckpoint, writeCheckpoint } from './capture-prefixes.mjs';
|
|
4
5
|
|
|
5
6
|
export function applyCaptureMessages({ get, run }, capture, revision) {
|
|
6
|
-
const { teamId, sessionId, sourceId, captureId, harness, nativeId, currentRevision, carriedCount, parts, messageNamespace } = capture;
|
|
7
|
+
const { teamId, sessionId, sourceId, captureId, harness, nativeId, currentRevision, carriedCount, parts, messageNamespace, prefix } = capture;
|
|
8
|
+
let prefixState = null, prefixHash = null;
|
|
9
|
+
if (prefix) {
|
|
10
|
+
prefixState = get('SELECT * FROM capture_prefix_state WHERE team_id=? AND session_id=? AND source_id=?', [teamId, sessionId, sourceId]);
|
|
11
|
+
if (!prefixState || prefixState.revision !== currentRevision || prefixState.generation !== prefix.generation) throw Object.assign(new Error('session prefix changed before commit; retry'), { status: 409, code: 'revision_mismatch' });
|
|
12
|
+
prefixHash = loadPrefix(prefix.sha_state);
|
|
13
|
+
if (prefixState.count !== prefixState.checkpoint_count) run('DELETE FROM capture_prefix_checkpoints WHERE team_id=? AND session_id=? AND source_id=? AND generation=? AND count=?', [teamId, sessionId, sourceId, prefix.generation, prefixState.count]);
|
|
14
|
+
prefixState.checkpoint_count = prefix.checkpoint_count;
|
|
15
|
+
prefixState.checkpoint_bytes = prefix.checkpoint_bytes;
|
|
16
|
+
// Remove only a tail the capture replaces, through its ordinal/count index.
|
|
17
|
+
run('DELETE FROM capture_prefix_keys WHERE team_id=? AND session_id=? AND source_id=? AND generation=? AND ordinal>=?', [teamId, sessionId, sourceId, prefix.generation, carriedCount]);
|
|
18
|
+
run('DELETE FROM capture_prefix_checkpoints WHERE team_id=? AND session_id=? AND source_id=? AND generation=? AND count>?', [teamId, sessionId, sourceId, prefix.generation, carriedCount]);
|
|
19
|
+
} else retirePrefix({ get, run }, teamId, sessionId, sourceId);
|
|
7
20
|
let ordinal = carriedCount;
|
|
8
21
|
for (let index = 0; index < parts; index++) {
|
|
9
22
|
const heldPart = get('SELECT payload_json FROM capture_parts WHERE team_id=? AND capture_id=? AND part_index=?', [teamId, captureId, index]);
|
|
10
23
|
if (!heldPart) throw Object.assign(new Error('capture parts incomplete'), { status: 409, code: 'batch_gap' });
|
|
11
24
|
const part = JSON.parse(heldPart.payload_json);
|
|
12
25
|
for (const message of part.messages) {
|
|
26
|
+
if (prefixHash) {
|
|
27
|
+
addPrefixMessage(prefixHash, ordinal, message);
|
|
28
|
+
run('INSERT INTO capture_prefix_keys VALUES (?,?,?,?,?,?) ON CONFLICT(team_id,session_id,source_id,generation,native_key) DO UPDATE SET ordinal=excluded.ordinal', [teamId, sessionId, sourceId, prefix.generation, message.native_key, ordinal]);
|
|
29
|
+
if (shouldCheckpoint(prefixState, prefixHash, ordinal + 1)) {
|
|
30
|
+
writeCheckpoint(run, prefixState, prefixHash, ordinal + 1);
|
|
31
|
+
prefixState.checkpoint_count = ordinal + 1; prefixState.checkpoint_bytes = prefixHash.length;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
13
34
|
const value = JSON.stringify(message.value);
|
|
14
35
|
const old = currentRevision == null ? null : get(`SELECT revision,source_id,native_key,recorded_at,value_json,until_revision FROM messages m WHERE m.team_id=? AND m.session_id=? AND m.ordinal=? AND ${MESSAGE_AT()} LIMIT 1`, [teamId, sessionId, ordinal, currentRevision, currentRevision]);
|
|
15
36
|
if (old && old.source_id === sourceId && old.native_key === message.native_key && old.recorded_at === message.recorded_at && old.value_json === value) {
|
|
@@ -27,5 +48,10 @@ export function applyCaptureMessages({ get, run }, capture, revision) {
|
|
|
27
48
|
}
|
|
28
49
|
}
|
|
29
50
|
if (currentRevision != null) run('UPDATE messages SET until_revision=? WHERE team_id=? AND session_id=? AND ordinal>=? AND revision<=? AND until_revision=?', [revision, teamId, sessionId, ordinal, currentRevision, OPEN_REVISION]);
|
|
51
|
+
if (prefixHash) {
|
|
52
|
+
writeCheckpoint(run, prefixState, prefixHash, ordinal);
|
|
53
|
+
run('UPDATE capture_prefix_state SET revision=?,count=?,target_count=?,sha_state=?,checkpoint_count=?,checkpoint_bytes=? WHERE team_id=? AND session_id=? AND source_id=? AND generation=?', [revision, ordinal, ordinal, savePrefix(prefixHash), prefixState.checkpoint_count, prefixState.checkpoint_bytes, teamId, sessionId, sourceId, prefix.generation]);
|
|
54
|
+
run('DELETE FROM capture_prefix_warm_queue WHERE team_id=? AND session_id=? AND source_id=?', [teamId, sessionId, sourceId]);
|
|
55
|
+
}
|
|
30
56
|
return { messages: ordinal - carriedCount };
|
|
31
57
|
}
|
package/server/event-routes.mjs
CHANGED
|
@@ -221,7 +221,7 @@ export async function handleCatalogEvents({ api, req, res, auth, url, requestId,
|
|
|
221
221
|
const stream = await openStream(api, req, res, auth, requestId);
|
|
222
222
|
let lastCursor = null;
|
|
223
223
|
const enqueue = streamQueue(stream, async () => {
|
|
224
|
-
const page = await catalogWindow(api, auth, query);
|
|
224
|
+
const page = await (api.work ? api.work('catalogWindow', () => catalogWindow(api, auth, query)) : catalogWindow(api, auth, query));
|
|
225
225
|
// Authorization and expiry can change a window without creating a session revision.
|
|
226
226
|
// A binding's observed_at moves with every heartbeat; only a change of what the window says is a reset.
|
|
227
227
|
const digest = createHash('sha256').update(JSON.stringify(page.data, (key, value) => key === 'bindings' && Array.isArray(value) ? value.map(({ observed_at: _seen, ...binding }) => binding) : value)).digest('hex');
|
|
@@ -88,6 +88,11 @@ async function beforeIdempotency({ api, req, res, auth, url, path, sub, requestI
|
|
|
88
88
|
|
|
89
89
|
async function team({ api, req, res, auth, url, path, sub, requestId }) {
|
|
90
90
|
let m;
|
|
91
|
+
if(req.method==='GET'&&sub==='/storage/work'){
|
|
92
|
+
if(!isAdmin(auth))fail(403,'forbidden','Teams work diagnostics require an administrator');
|
|
93
|
+
if(!api.workReport)fail(501,'unsupported','this host does not record Teams work diagnostics');
|
|
94
|
+
send(res,200,result(api,auth,api.workReport()),requestId);return true;
|
|
95
|
+
}
|
|
91
96
|
if(req.method==='GET'&&sub==='/storage/uploads'){await storageReport({api,res,auth,url,requestId});return true;}
|
|
92
97
|
if(req.method==='GET'&&sub==='/sessions'){await listSessions(api,res,auth,url,requestId,false);return true;}
|
|
93
98
|
if(req.method==='GET'&&sub==='/sessions/search'){await listSessions(api,res,auth,url,requestId,true);return true;}
|
|
@@ -240,13 +245,14 @@ async function correlateRoute(api,req,res,auth,path,requestId){
|
|
|
240
245
|
if(body.evidence!=='custodian_attestation'||typeof body.harness!=='string'||typeof body.left_source_id!=='string'||typeof body.right_source_id!=='string'||typeof body.left_native_id!=='string'||typeof body.right_native_id!=='string'||!Number.isSafeInteger(body.left_revision)||!Number.isSafeInteger(body.right_revision))fail(400,'invalid_request','invalid correlation attestation');
|
|
241
246
|
const correlationId=api.ids.next('correlation'),record=res._idempotencyRecord;
|
|
242
247
|
const outcome=await api.store.correlateSessions({teamId:auth.team_id,principalId:auth.principal_id,credentialId:auth.credential_id,now:api.clock.nowMs(),correlationId,auditId:api.ids.next('audit'),operation:`POST ${path}`,context:{server_id:api.serverId,team_id:auth.team_id,principal_id:auth.principal_id},idempotency:record?{route:record.route,key:record.key,requestDigest:record.requestDigest}:null,harness:body.harness,evidence:body,left:{sourceId:body.left_source_id,nativeId:body.left_native_id,revision:body.left_revision},right:{sourceId:body.right_source_id,nativeId:body.right_native_id,revision:body.right_revision}});
|
|
243
|
-
if(outcome.tombstoned){res._sideEffectsCommitted=true;api.events.publish(auth.team_id,{type:'session_deleted',cursor:String(outcome.revision),revision:outcome.revision,session_id:outcome.canonicalId});fail(410,'session_deleted','a correlated constituent was deleted');}
|
|
248
|
+
if(outcome.tombstoned){res._sideEffectsCommitted=true;api.events.publish(auth.team_id,{type:'session_deleted',cursor:String(outcome.revision),revision:outcome.revision,session_id:outcome.canonicalId});await api.scheduleCapturePrefixWork?.();fail(410,'session_deleted','a correlated constituent was deleted');}
|
|
244
249
|
res._sideEffectsCommitted=true;api.events.publish(auth.team_id,{type:'session_changed',cursor:String(outcome.revision),revision:outcome.revision,session_id:outcome.canonicalId,reason:'metadata',read_required:true});send(res,200,outcome.response,requestId,{ETag:`"${outcome.revision}"`});
|
|
245
250
|
}
|
|
246
251
|
async function mutateSession(api,req,res,auth,id,requestId){const seed=await api.store.one('SELECT * FROM sessions WHERE team_id=? AND id=?',[auth.team_id,id]),root=await canonicalSession(api,auth.team_id,seed),canonicalId=root?.id??id;const row=await api.store.one('SELECT s.*,v.descriptor_json,v.source_projection_json,v.search_title,v.sort_at FROM sessions s JOIN session_versions v ON v.team_id=s.team_id AND v.session_id=s.id AND v.revision=s.revision WHERE s.team_id=? AND s.id=?',[auth.team_id,canonicalId]);if(!row)fail(404,'not_found','not found');const members=await constituents(api,auth.team_id,row);for(const member of members)if(!isAdmin(auth)&&!canManage(auth,member,await grants(api,auth.team_id,member.id)))fail(404,'not_found','not found');const expected=Number(String(req.headers['if-match']??'').replaceAll('"',''));if(!Number.isSafeInteger(expected))fail(400,'invalid_request','If-Match is required');if(expected!==row.revision)fail(409,'revision_mismatch','session revision changed');const boundaryNow=api.clock.nowMs(),revisionParam={$journalRevision:true},guards=[...authGuards(auth,boundaryNow),{sql:'SELECT 1 ok FROM sessions WHERE team_id=? AND id=? AND revision=? AND deleted_at IS NULL',params:[auth.team_id,row.id,expected],code:'revision_mismatch',status:409,message:'session revision changed'}];
|
|
247
252
|
for(const member of members)guards.push(managerGuard(auth,member.id,boundaryNow,{governance:true}));
|
|
248
|
-
if(req.method==='DELETE'){const deletion=[];for(const member of members)deletion.push({sql:'UPDATE sessions SET deleted_at=?,revision=? WHERE team_id=? AND id=?',params:[boundaryNow,revisionParam,auth.team_id,member.id]},{sql:'UPDATE origins SET deleted_at=? WHERE team_id=? AND session_id=?',params:[boundaryNow,auth.team_id,member.id]},{sql:'DELETE FROM session_versions WHERE team_id=? AND session_id=?',params:[auth.team_id,member.id]});deletion.push({sql:'UPDATE teams SET journal_revision=?,auth_epoch=auth_epoch+1 WHERE id=?',params:[revisionParam,auth.team_id]},{sql:'INSERT INTO journal(team_id,revision,session_id,kind,created_at) VALUES (?,?,?,?,?)',params:[auth.team_id,revisionParam,row.id,'delete',boundaryNow]});const {revision}=await guardedMutation(api,res,auth,guards,deletion,{status:204,body:null,targetId:row.id,journal:true});api.events.publish(auth.team_id,{type:'session_deleted',cursor:String(revision),revision,session_id:row.id});return send(res,204,null,requestId);}
|
|
253
|
+
if(req.method==='DELETE'){const deletion=[];for(const member of members)deletion.push({sql:'UPDATE sessions SET deleted_at=?,revision=? WHERE team_id=? AND id=?',params:[boundaryNow,revisionParam,auth.team_id,member.id]},{sql:'UPDATE origins SET deleted_at=? WHERE team_id=? AND session_id=?',params:[boundaryNow,auth.team_id,member.id]},{sql:'DELETE FROM session_versions WHERE team_id=? AND session_id=?',params:[auth.team_id,member.id]},{sql:'INSERT OR IGNORE INTO capture_prefix_garbage SELECT team_id,session_id,source_id,generation FROM capture_prefix_state WHERE team_id=? AND session_id=?',params:[auth.team_id,member.id]},{sql:'DELETE FROM capture_prefix_state WHERE team_id=? AND session_id=?',params:[auth.team_id,member.id]},{sql:'DELETE FROM capture_prefix_warm_queue WHERE team_id=? AND session_id=?',params:[auth.team_id,member.id]});deletion.push({sql:'UPDATE teams SET journal_revision=?,auth_epoch=auth_epoch+1 WHERE id=?',params:[revisionParam,auth.team_id]},{sql:'INSERT INTO journal(team_id,revision,session_id,kind,created_at) VALUES (?,?,?,?,?)',params:[auth.team_id,revisionParam,row.id,'delete',boundaryNow]});const {revision}=await guardedMutation(api,res,auth,guards,deletion,{status:204,body:null,targetId:row.id,journal:true});api.events.publish(auth.team_id,{type:'session_deleted',cursor:String(revision),revision,session_id:row.id});await api.scheduleCapturePrefixWork?.();return send(res,204,null,requestId);}
|
|
249
254
|
const body=await parseJson(req);exact(body,['title']);if(body.title!==null&&typeof body.title!=='string')fail(400,'invalid_request','title must be string or null');const descriptor={...projection(row),title:body.title};delete descriptor.revision;const statements=[{sql:'UPDATE sessions SET revision=? WHERE team_id=? AND id=? AND revision=?',params:[revisionParam,auth.team_id,row.id,expected]},{sql:'INSERT INTO session_versions(team_id,session_id,revision,descriptor_json,source_projection_json,search_title,sort_at,created_at) VALUES (?,?,?,?,?,?,?,?)',params:[auth.team_id,row.id,revisionParam,{$journalJson:descriptor},row.source_projection_json,normalizeSearch(body.title??''),row.sort_at,boundaryNow]}];// The new version holds the same messages: the current version's rows stay open into it (schema 9), none copied.
|
|
255
|
+
statements.push({sql:'UPDATE capture_prefix_state SET revision=? WHERE team_id=? AND session_id=? AND revision=?',params:[revisionParam,auth.team_id,row.id,expected]});
|
|
250
256
|
statements.push({sql:`UPDATE messages SET until_revision=? WHERE team_id=? AND session_id=? AND ${MESSAGE_AT('messages')}`,params:[OPEN_REVISION,auth.team_id,row.id,row.revision,row.revision]});statements.push({sql:'UPDATE teams SET journal_revision=? WHERE id=?',params:[revisionParam,auth.team_id]},{sql:'INSERT INTO journal(team_id,revision,session_id,kind,created_at) VALUES (?,?,?,?,?)',params:[auth.team_id,revisionParam,row.id,'metadata',boundaryNow]});const storedResponse=result(api,auth,{...descriptor,revision:'__JOURNAL_REVISION__'}),{revision}=await guardedMutation(api,res,auth,guards,statements,{status:200,body:storedResponse,targetId:row.id,journal:true}),responseDescriptor={...descriptor,revision};api.events.publish(auth.team_id,{type:'session_changed',cursor:String(revision),revision,session_id:row.id,reason:'metadata',read_required:true});send(res,200,result(api,auth,responseDescriptor),requestId,{ETag:`"${revision}"`});}
|
|
251
257
|
async function readSession(api,res,auth,id,url,requestId){
|
|
252
258
|
const query=qobj(url);
|
|
@@ -409,30 +415,21 @@ async function captureCommit(api,req,res,auth,sourceId,captureId,requestId){cons
|
|
|
409
415
|
}
|
|
410
416
|
if(commit.native_bundle!=null){const bundle=commit.native_bundle;exact(bundle,['id','harness','fidelity','residue','files'],'native_bundle');if(typeof bundle.id!=='string'||bundle.harness!==cap.harness||!['byte_lossless','value_lossless','semantic'].includes(bundle.fidelity)||!Array.isArray(bundle.residue)||!bundle.residue.every(x=>typeof x==='string')||!Array.isArray(bundle.files))fail(400,'invalid_request','invalid native bundle');const artifactIds=new Set(commit.artifacts.map(x=>x.upload_id)),seen=new Set();for(const file of bundle.files){exact(file,['upload_id','role'],'native_bundle file');if(!artifactIds.has(file.upload_id)||seen.has(file.upload_id)||!['primary','subagent','bundle','source_recovery'].includes(file.role))fail(400,'invalid_request','native bundle files must uniquely reference captured artifacts');seen.add(file.upload_id);}if(seen.size!==artifactIds.size||bundle.files.filter(x=>x.role==='primary').length!==1||new Set(commit.artifacts.map(x=>x.relative_name)).size!==commit.artifacts.length)fail(400,'invalid_request','native bundle must describe every unique artifact and one primary');}
|
|
411
417
|
let origin=await api.store.one('SELECT * FROM origins WHERE team_id=? AND source_id=? AND harness=? AND native_id=?',[auth.team_id,sourceId,cap.harness,cap.native_id]);
|
|
412
|
-
//
|
|
413
|
-
//
|
|
414
|
-
|
|
415
|
-
let carriedCount = 0, carriedRevision = null;
|
|
418
|
+
// Prepared digests are owned by this source and fenced by the session revision.
|
|
419
|
+
// A cold prefix queues bounded background work; no request falls back to a whole transcript read.
|
|
420
|
+
let carriedCount = 0, carriedRevision = null, prefix = null;
|
|
416
421
|
if (begin.carry) {
|
|
417
422
|
const { count, digest } = begin.carry;
|
|
418
423
|
const session = origin ? await api.store.one('SELECT revision FROM sessions WHERE team_id=? AND id=?', [auth.team_id, origin.session_id]) : null;
|
|
419
424
|
if (!session || count + messageCount > 100000) fail(409, 'carry_mismatch', 'the carried prefix is unavailable or too large');
|
|
420
425
|
carriedRevision = session.revision;
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
if (!page.length) fail(409, 'carry_mismatch', 'the carried messages are not what this session holds; send the capture whole');
|
|
427
|
-
for (const row of page) {
|
|
428
|
-
if (row.ordinal !== carriedCount || row.source_id !== sourceId) fail(409, 'carry_mismatch', 'the carried messages are not what this session holds; send the capture whole');
|
|
429
|
-
if (sentKeys.has(row.native_key)) fail(409, 'capture_conflict', 'duplicate native message key');
|
|
430
|
-
if (carriedCount) hash.update(',');
|
|
431
|
-
hash.update(canonicalJson({native_key:row.native_key, recorded_at:row.recorded_at ?? null, value:JSON.parse(row.value_json)}));
|
|
432
|
-
carriedCount++;
|
|
433
|
-
}
|
|
426
|
+
prefix = await api.store.call('resolveCapturePrefix', { teamId: auth.team_id, sessionId: origin.session_id, sourceId, revision: carriedRevision, count, digest, sentKeys: [...sentKeys] });
|
|
427
|
+
if (prefix.pending) {
|
|
428
|
+
await api.scheduleCapturePrefixWork?.();
|
|
429
|
+
res.setHeader('Retry-After', '1');
|
|
430
|
+
fail(503, 'carry_checkpoint_pending', 'the session prefix is preparing; retry this capture without resending its messages');
|
|
434
431
|
}
|
|
435
|
-
|
|
432
|
+
carriedCount = count;
|
|
436
433
|
}
|
|
437
434
|
let lineage=null,lineageValue=null,lineageMembers=[],lineageGuards=[];if(!origin&&source.kind==='connector'&&source.machine_id){const candidates=await api.store.query("SELECT * FROM continuations WHERE team_id=? AND state='running' AND json_extract(value_json,'$.new_session_id') IS NULL AND json_extract(value_json,'$.harness')=? AND json_extract(value_json,'$.destination.machine_id')=? AND json_extract(value_json,'$.remote_result.state')='succeeded' AND json_extract(value_json,'$.remote_result.receipt.native_session_id')=?",[auth.team_id,cap.harness,source.machine_id,cap.native_id]);if(candidates.length===1){const candidate=candidates[0],membership=await api.store.one('SELECT role,active FROM memberships WHERE team_id=? AND principal_id=?',[auth.team_id,candidate.requester_id]),reader={kind:'user',team_id:auth.team_id,principal_id:candidate.requester_id,role:membership?.role,active:membership?.active===1,now:api.clock.nowMs(),group_ids:(await api.store.query('SELECT group_id FROM group_members WHERE team_id=? AND principal_id=?',[auth.team_id,candidate.requester_id])).map(x=>x.group_id)};if(await canReadMerged(api,reader,await api.store.one('SELECT * FROM sessions WHERE team_id=? AND id=?',[auth.team_id,candidate.source_session_id]))){lineage=candidate;lineageValue=JSON.parse(candidate.value_json);lineageMembers=await constituents(api,auth.team_id,await api.store.one('SELECT * FROM sessions WHERE team_id=? AND id=?',[auth.team_id,candidate.source_session_id]));}}if(lineageValue){const lineageNow=api.clock.nowMs();lineageGuards=[{sql:'SELECT 1 ok FROM memberships WHERE team_id=? AND principal_id=? AND active=1',params:[auth.team_id,lineage.requester_id],code:'conflict',status:409,message:'continuation requester membership changed'},{sql:'SELECT 1 ok FROM credentials WHERE id=? AND principal_id=? AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at>?)',params:[lineageValue.issuer_credential_id,lineage.requester_id,lineageNow],code:'conflict',status:409,message:'continuation issuer credential changed'},...lineageMembers.map(member=>({sql:`SELECT 1 ok FROM sessions s WHERE s.team_id=? AND s.id=? AND s.deleted_at IS NULL AND (?=0 OR s.access_revision=?) AND (s.visibility='team' OR s.custodian_id=? OR EXISTS (SELECT 1 FROM session_grants g WHERE g.team_id=s.team_id AND g.session_id=s.id AND (g.expires_at IS NULL OR g.expires_at>?) AND (g.subject_kind='principal' AND g.subject_id=? OR g.subject_kind='group' AND EXISTS (SELECT 1 FROM group_members gm WHERE gm.team_id=s.team_id AND gm.group_id=g.subject_id AND gm.principal_id=?))))`,params:[auth.team_id,member.id,member.id===lineage.source_session_id?1:0,lineageValue.access_revision,lineage.requester_id,lineageNow,lineage.requester_id,lineage.requester_id],code:'conflict',status:409,message:'continuation source authorization changed'}))];}}
|
|
438
435
|
const sid=origin?.session_id??api.ids.next('session'),parentOrigin=commit.parent_native_id?await api.store.one('SELECT session_id FROM origins WHERE team_id=? AND source_id=? AND harness=? AND native_id=? AND deleted_at IS NULL',[auth.team_id,sourceId,cap.harness,commit.parent_native_id]):null,revisionParam={$journalRevision:true},recorded=commit.recorded_activity_at,boundaryNow=api.clock.nowMs(),sortAt=recorded??boundaryNow,descriptor={id:sid,revision:'__JOURNAL_REVISION__',title:commit.title,harness:cap.harness,user:{principal_id:source.kind==='connector'?source.custodian_id:null,reported_user:commit.reported_user,status:source.kind==='connector'?'verified':commit.reported_user?'reported':'unknown',mapping_id:null},repository:commit.repository,parent_id:parentOrigin?.session_id??null,continued_from:lineage?.source_session_id??null,recorded_activity_at:recorded,received_at:boundaryNow,sort_at:sortAt,activity_basis:recorded==null?'received':'recorded',source_ids:[sourceId],machine_ids:source.machine_id?[source.machine_id]:[],coverage:commit.coverage,bindings:[],actions:{}},statements=[];
|
|
@@ -447,7 +444,7 @@ async function captureCommit(api,req,res,auth,sourceId,captureId,requestId){cons
|
|
|
447
444
|
carryGuards.push({ sql: 'SELECT 1 ok WHERE (SELECT COUNT(*) FROM capture_parts WHERE team_id=? AND capture_id=?)=?', params: [auth.team_id, captureId, commit.parts], status: 409, code: 'batch_gap', message: 'capture parts changed before commit' });
|
|
448
445
|
// Another commit to this session between this read and the write would make the carried rows wrong: refused, retried.
|
|
449
446
|
if(currentRevision!=null)carryGuards.push({sql:'SELECT 1 ok FROM sessions WHERE team_id=? AND id=? AND revision=?',params:[auth.team_id,sid,currentRevision],code:'revision_mismatch',status:409,message:'session changed during capture; retry'});
|
|
450
|
-
statements.push({ kind: 'capture_messages', capture: { teamId: auth.team_id, sessionId: sid, sourceId, captureId, harness: cap.harness, nativeId: cap.native_id, currentRevision, carriedCount, parts: commit.parts, messageNamespace: api.ids.next('message') } });
|
|
447
|
+
statements.push({ kind: 'capture_messages', capture: { teamId: auth.team_id, sessionId: sid, sourceId, captureId, harness: cap.harness, nativeId: cap.native_id, currentRevision, carriedCount, prefix, parts: commit.parts, messageNamespace: api.ids.next('message') } });
|
|
451
448
|
const bundle=commit.native_bundle??null,legacyBundle=commit.artifacts.length?captureId:null;for(let index=0;index<commit.artifacts.length;index++){const a=commit.artifacts[index],file=bundle?.files.find(x=>x.upload_id===a.upload_id);statements.push({sql:'UPDATE uploads SET manifest_json=?,committed=1 WHERE team_id=? AND id=?',params:[JSON.stringify(a),auth.team_id,a.upload_id]},{sql:'INSERT INTO session_artifacts(team_id,session_id,revision,bundle_id,upload_id,format,provenance,file_role,fidelity,residue_json) VALUES (?,?,?,?,?,?,?,?,?,?)',params:[auth.team_id,sid,revisionParam,bundle?.id??legacyBundle,a.upload_id,bundle?.harness??cap.harness,'capture',file?.role??(index===0?'primary':'bundle'),bundle?.fidelity??'semantic',JSON.stringify(bundle?.residue??['native_bundle_metadata_unavailable'])]});}const storedReceipt={source_id:sourceId,capture_id:captureId,session_id:sid,generation:cap.generation,sequence:cap.sequence,digest:commit.digest,revision:'__JOURNAL_REVISION__',duplicate:false};statements.push({sql:"UPDATE captures SET state='committed',commit_json=?,receipt_json=? WHERE team_id=? AND id=?",params:[JSON.stringify(commit),{$journalJsonValue:storedReceipt},auth.team_id,captureId]},{sql:'DELETE FROM capture_parts WHERE team_id=? AND capture_id=?',params:[auth.team_id,captureId]},{sql:'UPDATE sources SET last_received_at=? WHERE team_id=? AND id=?',params:[boundaryNow,auth.team_id,sourceId]},{sql:'UPDATE teams SET journal_revision=? WHERE id=?',params:[revisionParam,auth.team_id]},{sql:'INSERT INTO journal(team_id,revision,session_id,kind,created_at) VALUES (?,?,?,?,?)',params:[auth.team_id,revisionParam,sid,'capture',boundaryNow]});const guards=[{sql:'SELECT 1 ok FROM credentials WHERE id=? AND source_id=? AND revoked_at IS NULL AND (expires_at IS NULL OR expires_at>?)',params:[auth.credential_id,sourceId,boundaryNow],code:'authentication_required',status:401,message:'source credential expired or revoked'},...lineageGuards,...carryGuards,{sql:"SELECT 1 ok FROM captures c JOIN sources s ON s.team_id=c.team_id AND s.id=c.source_id LEFT JOIN machines m ON m.team_id=s.team_id AND m.id=s.machine_id LEFT JOIN sync_rules r ON r.team_id=c.team_id AND r.id=json_extract(c.begin_json,'$.rule_id') WHERE c.team_id=? AND c.id=? AND c.source_id=? AND c.state='staging' AND s.state='active' AND (s.kind!='connector' OR (m.revoked_at IS NULL AND r.machine_id=m.id AND r.source_id=s.id AND r.state='active' AND r.revision=json_extract(c.begin_json,'$.rule_revision') AND r.visibility=json_extract(c.begin_json,'$.visibility')))",params:[auth.team_id,captureId,sourceId],code:'conflict',status:409,message:'capture authorization changed'}],storedResponse=result(api,auth,storedReceipt),{revision}=await guardedMutation(api,res,auth,guards,statements,{status:200,body:storedResponse,targetId:captureId,journal:true}),receipt={...storedReceipt,revision};api.events.publish(auth.team_id,{type:'session_changed',cursor:String(revision),revision,session_id:sid,reason:'capture',read_required:true});send(res,200,result(api,auth,receipt),requestId);
|
|
452
449
|
}
|
|
453
450
|
|
package/server/index.mjs
CHANGED
|
@@ -24,7 +24,14 @@ export function createTeamsServer({dataDir,listen={host:'127.0.0.1',port:7331},p
|
|
|
24
24
|
// The harness's own routes on this server (harness-routes.mjs); the handler owns the neutral ones.
|
|
25
25
|
api.attach({harnessRoutes});
|
|
26
26
|
const events=api.events;
|
|
27
|
-
let store,http,upgrade,lockFd,maintenanceTimer,maintenanceWork,started=false;
|
|
27
|
+
let store,http,upgrade,lockFd,maintenanceTimer,maintenanceWork,maintenanceRunning=false,prefixRequested=false,maintenanceDue=0,started=false;
|
|
28
|
+
api.scheduleCapturePrefixWork=async()=>{
|
|
29
|
+
prefixRequested=true;
|
|
30
|
+
if(!started||maintenanceRunning)return;
|
|
31
|
+
if(maintenanceTimer)clearTimeout(maintenanceTimer);
|
|
32
|
+
maintenanceTimer=setTimeout(()=>{maintenanceWork=maintenance();},100);maintenanceTimer.unref?.();
|
|
33
|
+
};
|
|
34
|
+
let maintenance;
|
|
28
35
|
api.start=async()=>{
|
|
29
36
|
if(started)throw new Error('Teams server already started'); mkdirSync(dataDir,{recursive:true,mode:0o700});chmodSync(dataDir,0o700);
|
|
30
37
|
const lock=join(dataDir,'server.lock'); const acquire=()=>{lockFd=openSync(lock,'wx',0o600);writeFileSync(lockFd,JSON.stringify({pid:process.pid,started_at:clock.nowMs()}));};try{acquire();}catch(e){let recover=false;try{const owner=JSON.parse(readFileSync(lock,'utf8'));if(Number.isInteger(owner.pid)&&owner.pid>1){try{process.kill(owner.pid,0);}catch(error){recover=error.code==='ESRCH';}}}catch{}if(!recover)throw new Error(`Teams data directory is already owned: ${dataDir}`,{cause:e});unlinkSync(lock);acquire();}
|
|
@@ -43,14 +50,24 @@ export function createTeamsServer({dataDir,listen={host:'127.0.0.1',port:7331},p
|
|
|
43
50
|
}
|
|
44
51
|
api.attach({serverId});
|
|
45
52
|
if(existsSync(credentialPath))api.appRootSecret=JSON.parse(readFileSync(credentialPath,'utf8')).token;
|
|
46
|
-
|
|
53
|
+
maintenance=async()=>{
|
|
54
|
+
maintenanceRunning=true;prefixRequested=false;
|
|
47
55
|
let next=60000;
|
|
48
56
|
try{
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
if(
|
|
57
|
+
const prefixes=await store.call('warmCapturePrefixes').catch(error=>{console.error(`Teams capture prefix preparation failed: ${error.message}`);return {backlog:true};});
|
|
58
|
+
// Prefix preparation runs in small batches; it must not pull retention,
|
|
59
|
+
// compaction or filesystem probes forward on every batch.
|
|
60
|
+
if(Date.now()>=maintenanceDue){
|
|
61
|
+
const result=await maintain(api);maintenanceDue=Date.now()+maintainDelay(api,result);
|
|
62
|
+
if(!result?.backlog){const reclaimed=await store.call('reclaimStorage');if(reclaimed.after.allocated<reclaimed.before.allocated)console.log(`Teams storage reclaimed: ${JSON.stringify(reclaimed)}`);}
|
|
63
|
+
const usage=await api.storageUsage();
|
|
64
|
+
if(usage.used>=usage.limit*0.8||usage.available_bytes<usage.min_free_bytes+usage.limit*0.2)console.warn(`Teams storage pressure: ${JSON.stringify(usage)}; native admission stops at90% or the configured disk reserve`);
|
|
65
|
+
}
|
|
66
|
+
next=Math.max(1,maintenanceDue-Date.now());
|
|
67
|
+
if(prefixes.backlog)next=Math.min(next,100);
|
|
53
68
|
}catch(error){console.error(`Teams maintenance failed: ${error.message}`);next=3000;}
|
|
69
|
+
maintenanceRunning=false;
|
|
70
|
+
if(prefixRequested)next=Math.min(next,100);
|
|
54
71
|
if(started){maintenanceTimer=setTimeout(()=>{maintenanceWork=maintenance();},next);maintenanceTimer.unref?.();}
|
|
55
72
|
return next;
|
|
56
73
|
};
|
package/server/store-core.mjs
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { ingestOtlpBatch, correlateSessions } from './arc2-transactions.mjs';
|
|
7
7
|
import { claimEffect, getEffect, listEffects, reportEffect } from './effects-transactions.mjs';
|
|
8
8
|
import { applyCaptureMessages } from './capture-transactions.mjs';
|
|
9
|
+
import { resolveCapturePrefix, warmCapturePrefixes } from './capture-prefixes.mjs';
|
|
9
10
|
import { MIGRATIONS, MESSAGE_AT } from '@volter/teams/server/migrations';
|
|
10
11
|
import { AGENT_ACTIVE, AGENT_DESCENDANTS } from '@volter/teams/server/agent-authority';
|
|
11
12
|
import { APP_LINEAGE_REVOKED, appGrantAllows } from '@volter/teams/server/auth';
|
|
@@ -157,6 +158,8 @@ export function createStoreHandlers({ all, get, run, tx }) {
|
|
|
157
158
|
guardedTransaction: ({guards=[],statements}) => tx(() => { for(const guard of guards){const row=get(guard.sql,guard.params);if(!row){const error=new Error(guard.message??'transaction predicate changed');error.code=guard.code??'conflict';error.status=guard.status??409;throw error;}}return statements.map(s=>s.kind==='query'?all(s.sql,s.params):s.kind==='one'?get(s.sql,s.params):run(s.sql,s.params));}),
|
|
158
159
|
journalTransaction: ({teamId,guards=[],statements}) => tx(() => { for(const guard of guards){const row=get(guard.sql,guard.params);if(!row){const error=new Error(guard.message??'transaction predicate changed');error.code=guard.code??'conflict';error.status=guard.status??409;throw error;}}const team=get('SELECT journal_revision FROM teams WHERE id=?',[teamId]);if(!team)throw new Error('team not found');const revision=team.journal_revision+1;const results=statements.map(s=>{if(s.kind==='capture_messages')return applyCaptureMessages({get,run},s.capture,revision);const params=(s.params??[]).map(value=>journalParam(value,revision));return s.kind==='query'?all(s.sql,params):s.kind==='one'?get(s.sql,params):run(s.sql,params);});return{revision,results};}),
|
|
159
160
|
ingestOtlpBatch: args => tx(() => ingestOtlpBatch({get,all,run},args)),
|
|
161
|
+
resolveCapturePrefix: args => tx(() => resolveCapturePrefix({get,run},args)),
|
|
162
|
+
warmCapturePrefixes: () => tx(() => warmCapturePrefixes({get,run})),
|
|
160
163
|
correlateSessions: args => tx(() => correlateSessions({get,all,run},args)),
|
|
161
164
|
listEffects: args => tx(() => listEffects({get,all,run},args)),
|
|
162
165
|
getEffect: args => tx(() => getEffect({get,all,run},args)),
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
|
|
4
|
+
// Only contract path words survive. Never record a caller's query, identity, body or SQL parameters.
|
|
5
|
+
const WORDS = new Set('api v1 me health teams sessions messages search events catalogue machines heartbeat configuration roots sources captures parts commit uploads chunks members groups credentials permissions bindings agents publications attachments operations continuations sync-rules view-bindings exports audit storage work'.split(' '));
|
|
6
|
+
export function workRoute(input) {
|
|
7
|
+
try { return new URL(input, 'http://local').pathname.split('/').map(part => !part || WORDS.has(part) ? part : ':id').join('/'); }
|
|
8
|
+
catch { return '/:invalid'; }
|
|
9
|
+
}
|
|
10
|
+
const rounded = value => Math.round(value * 1000) / 1000;
|
|
11
|
+
|
|
12
|
+
/** Attribute synchronous SQLite work to its handler, rather than the invocation a timer happened to inherit. */
|
|
13
|
+
export class WorkMeter {
|
|
14
|
+
constructor({ emit = record => console.log(JSON.stringify(record)), now = () => performance.now(), retainSlow = null, advanceClock = null, sqlTiming = true } = {}) {
|
|
15
|
+
this.now = now; this.advanceClock = advanceClock; this.sqlTiming = sqlTiming;
|
|
16
|
+
this.context = new AsyncLocalStorage(); this.recent = []; this.slow = [];
|
|
17
|
+
this.emit = record => {
|
|
18
|
+
emit(record);
|
|
19
|
+
if (!['teams_work', 'teams_sql_work'].includes(record.event)) return;
|
|
20
|
+
this.recent.push(record); if (this.recent.length > 64) this.recent.shift();
|
|
21
|
+
if ((record.elapsed_ms ?? 0) < 1000 && (record.rows_read ?? 0) + (record.rows_written ?? 0) < 5000) return;
|
|
22
|
+
this.slow.push(record); if (this.slow.length > 32) this.slow.shift();
|
|
23
|
+
try { retainSlow?.(this.slow); }
|
|
24
|
+
catch { emit({ event: 'teams_work_retention_failed', work_id: record.work_id ?? null }); }
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
restoreSlow(records) { if (Array.isArray(records)) this.slow = records.slice(-32); }
|
|
28
|
+
report() { return { clock: this.advanceClock ? 'elapsed milliseconds through an end-of-work timer boundary; SQL timing unavailable on the hosted Worker' : 'elapsed milliseconds; sql_ms measures synchronous SQL, not CPU', recent: this.recent.slice(), slow: this.slow.slice() }; }
|
|
29
|
+
async run(work, fn, detail = {}) {
|
|
30
|
+
const parent = this.context.getStore();
|
|
31
|
+
const record = { work, ...detail, work_id: crypto.randomUUID(), parent_work_id: parent?.work_id ?? null, started_at: new Date().toISOString(), started: this.now(), sql_ms: 0, sql_calls: 0, rows_read: 0, rows_written: 0, rows_known: true, returned_rows: 0, slowest_sql: null, most_rows_sql: null, finished: false };
|
|
32
|
+
this.emit({ event: 'teams_work_start', work, ...detail, work_id: record.work_id, parent_work_id: record.parent_work_id, started_at: record.started_at });
|
|
33
|
+
return this.context.run(record, async () => {
|
|
34
|
+
let outcome = 'ok';
|
|
35
|
+
try { return await fn(); }
|
|
36
|
+
catch (error) { outcome = 'error'; throw error; }
|
|
37
|
+
finally {
|
|
38
|
+
record.finished = true;
|
|
39
|
+
// Hosted Workers freeze clocks during CPU work. Yield once after the work, not once per SQL call.
|
|
40
|
+
// This reports elapsed time, including the timer's wait; it does not manufacture a CPU reading.
|
|
41
|
+
if (this.advanceClock) await this.advanceClock();
|
|
42
|
+
const { started, finished: _finished, rows_known, ...data } = record;
|
|
43
|
+
this.emit({ event: 'teams_work', ...data, slowest_sql: this.sqlTiming ? record.slowest_sql : null, rows_read: rows_known ? record.rows_read : null, rows_written: rows_known ? record.rows_written : null, elapsed_ms: rounded(this.now() - started), sql_ms: this.sqlTiming ? rounded(record.sql_ms) : null, outcome });
|
|
44
|
+
// Nested steps are separately named, and their SQL remains counted in the enclosing handler.
|
|
45
|
+
if (parent && !parent.finished) {
|
|
46
|
+
for (const field of ['sql_ms', 'sql_calls', 'rows_read', 'rows_written', 'returned_rows']) parent[field] += record[field];
|
|
47
|
+
parent.rows_known &&= record.rows_known;
|
|
48
|
+
if (record.slowest_sql && (!parent.slowest_sql || record.slowest_sql.elapsed_ms > parent.slowest_sql.elapsed_ms)) parent.slowest_sql = record.slowest_sql;
|
|
49
|
+
if (record.most_rows_sql && (!parent.most_rows_sql || record.most_rows_sql.rows_read + record.most_rows_sql.rows_written > parent.most_rows_sql.rows_read + parent.most_rows_sql.rows_written)) parent.most_rows_sql = record.most_rows_sql;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
detail(values) {
|
|
55
|
+
const record = this.context.getStore();
|
|
56
|
+
if (record && !record.finished) Object.assign(record, values);
|
|
57
|
+
}
|
|
58
|
+
sql(query, fn) {
|
|
59
|
+
const record = this.context.getStore();
|
|
60
|
+
if (!record || record.finished) {
|
|
61
|
+
const start = this.now(); let result;
|
|
62
|
+
try { result = fn(); return result; }
|
|
63
|
+
finally {
|
|
64
|
+
const elapsed = this.now() - start;
|
|
65
|
+
if (elapsed >= 100 || (result?.rowsRead ?? 0) >= 5000) this.emit({ event: 'teams_sql_work', work: 'unscoped', origin_work: record?.work ?? null, fingerprint: createHash('sha256').update(query).digest('hex').slice(0, 16), elapsed_ms: this.sqlTiming ? rounded(elapsed) : null, rows_read: result?.rowsRead ?? null, rows_written: result?.rowsWritten ?? null, returned_rows: result?.returnedRows ?? null });
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
const start = this.now(); let result;
|
|
69
|
+
try { result = fn(); return result; }
|
|
70
|
+
finally {
|
|
71
|
+
const elapsed = this.now() - start;
|
|
72
|
+
record.sql_calls++; record.sql_ms += elapsed;
|
|
73
|
+
record.rows_known &&= Number.isFinite(result?.rowsRead) && Number.isFinite(result?.rowsWritten);
|
|
74
|
+
record.rows_read += result?.rowsRead ?? 0; record.rows_written += result?.rowsWritten ?? 0;
|
|
75
|
+
record.returned_rows += result?.returnedRows ?? 0;
|
|
76
|
+
if (!record.slowest_sql || elapsed > record.slowest_sql.elapsed_ms) {
|
|
77
|
+
record.slowest_sql = { fingerprint: createHash('sha256').update(query).digest('hex').slice(0, 16), operation: /^\s*(\w+)/.exec(query)?.[1]?.toUpperCase() ?? 'UNKNOWN', elapsed_ms: rounded(elapsed), rows_read: result?.rowsRead ?? null, rows_written: result?.rowsWritten ?? null, returned_rows: result?.returnedRows ?? null };
|
|
78
|
+
}
|
|
79
|
+
if (Number.isFinite(result?.rowsRead) && Number.isFinite(result?.rowsWritten) && (!record.most_rows_sql || result.rowsRead + result.rowsWritten > record.most_rows_sql.rows_read + record.most_rows_sql.rows_written)) {
|
|
80
|
+
record.most_rows_sql = { fingerprint: createHash('sha256').update(query).digest('hex').slice(0, 16), operation: /^\s*(\w+)/.exec(query)?.[1]?.toUpperCase() ?? 'UNKNOWN', rows_read: result.rowsRead, rows_written: result.rowsWritten, returned_rows: result?.returnedRows ?? null };
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
package/worker/index.mjs
CHANGED
|
@@ -18,6 +18,7 @@ import { setOtlpSchemaSources } from '../server/otlp/transport.mjs';
|
|
|
18
18
|
import { setViewerSources } from '@volter/teams/server/viewer';
|
|
19
19
|
import { serveFetch } from './fetch-adapter.mjs';
|
|
20
20
|
import { webSocketDuplex } from './websocket-duplex.mjs';
|
|
21
|
+
import { WorkMeter, workRoute } from '../server/work-meter.mjs';
|
|
21
22
|
// The hosted viewer's files, bundled as text (wrangler.toml rules) and served at /view.
|
|
22
23
|
import viewerPage from '../../volter-teams/viewer/index.html';
|
|
23
24
|
import viewerScript from '../../volter-teams/viewer/viewer.mjs';
|
|
@@ -56,34 +57,72 @@ const MAINTAIN_MS = 60_000;
|
|
|
56
57
|
const BOOTSTRAP_TOKEN = /^sct_(cred_[0-9a-f]{32})\.[A-Za-z0-9_-]{16,}$/;
|
|
57
58
|
|
|
58
59
|
/** SQLite primitives over the object's storage, in store-core's shape. */
|
|
59
|
-
function durableSql(storage) {
|
|
60
|
+
function durableSql(storage, meter) {
|
|
60
61
|
const sql = storage.sql;
|
|
61
62
|
const param = (value) => (value instanceof Uint8Array ? value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength) : value);
|
|
62
63
|
const row = (value) => { for (const key of Object.keys(value)) if (value[key] instanceof ArrayBuffer) value[key] = Buffer.from(value[key]); return value; };
|
|
63
|
-
const
|
|
64
|
+
const measured = (query, fn) => meter.sql(query, fn);
|
|
65
|
+
const all = (query, params = []) => measured(query, () => {
|
|
66
|
+
const cursor = sql.exec(query, ...params.map(param));
|
|
67
|
+
const values = cursor.toArray().map(row);
|
|
68
|
+
return { values, rowsRead: cursor.rowsRead, rowsWritten: cursor.rowsWritten, returnedRows: values.length };
|
|
69
|
+
}).values;
|
|
64
70
|
const get = (query, params = []) => all(query, params)[0] ?? null;
|
|
65
|
-
const run = (query, params = []) => {
|
|
71
|
+
const run = (query, params = []) => {
|
|
72
|
+
const counts = measured(query, () => {
|
|
73
|
+
const cursor = sql.exec(query, ...params.map(param));
|
|
74
|
+
return { rowsRead: cursor.rowsRead, rowsWritten: cursor.rowsWritten, returnedRows: 0 };
|
|
75
|
+
});
|
|
76
|
+
return { ...counts, changes: get('SELECT changes() AS c').c };
|
|
77
|
+
};
|
|
66
78
|
const tx = (fn) => storage.transactionSync(fn);
|
|
67
|
-
return { all, get, run, tx, exec: (query) =>
|
|
79
|
+
return { all, get, run, tx, exec: (query) => { all(query); } };
|
|
68
80
|
}
|
|
69
81
|
|
|
70
82
|
export class TeamsServer extends DurableObject {
|
|
71
83
|
constructor(ctx, env) {
|
|
72
84
|
super(ctx, env);
|
|
73
|
-
this.
|
|
85
|
+
this.nextMaintenanceAt = Date.now() + 5_000;
|
|
86
|
+
this.diagnosticPersistence = { interval_ms: 5_000, writes_since_activation: 0, last_saved_at: null, restored: false, restore_error: false };
|
|
87
|
+
this.work = new WorkMeter({ retainSlow: records => {
|
|
88
|
+
const now = Date.now(), state = this.diagnosticPersistence;
|
|
89
|
+
// Keep every slow record in memory, but never rewrite the persisted ring
|
|
90
|
+
// on each slow request. A reset can lose the most recent cooldown window.
|
|
91
|
+
if (state.last_saved_at != null && now - state.last_saved_at < state.interval_ms) return;
|
|
92
|
+
ctx.storage.kv.put('teams_work_slow_v1', JSON.stringify(records));
|
|
93
|
+
state.last_saved_at = now; state.writes_since_activation++;
|
|
94
|
+
}, advanceClock: () => scheduler.wait(0), sqlTiming: false });
|
|
95
|
+
this.ready = ctx.blockConcurrencyWhile(() => this.work.run('activation', async () => {
|
|
74
96
|
// Read before migrations/maintenance: allocation failures must not hide
|
|
75
97
|
// the table responsible, including hosts with a lower storage limit.
|
|
76
98
|
console.log(`storage: database_bytes=${ctx.storage.sql.databaseSize}`);
|
|
77
|
-
const db = durableSql(ctx.storage);
|
|
99
|
+
const db = durableSql(ctx.storage, this.work);
|
|
78
100
|
migrate({ exec: db.exec, tx: db.tx, version: () => ctx.storage.kv.get('schema_version') ?? 0, setVersion: (n) => ctx.storage.kv.put('schema_version', n) });
|
|
101
|
+
const retained = ctx.storage.kv.get('teams_work_slow_v1');
|
|
102
|
+
if (retained) {
|
|
103
|
+
try {
|
|
104
|
+
const records = JSON.parse(retained);
|
|
105
|
+
if (!Array.isArray(records)) throw new Error('invalid diagnostic history');
|
|
106
|
+
this.work.restoreSlow(records); this.diagnosticPersistence.restored = true;
|
|
107
|
+
} catch {
|
|
108
|
+
this.diagnosticPersistence.restore_error = true;
|
|
109
|
+
console.warn('Teams diagnostic history could not be read; serving with an empty diagnostic history');
|
|
110
|
+
}
|
|
111
|
+
}
|
|
79
112
|
this.api = createTeamsApi({ publicOrigin: normalizePublicOrigin(env.PUBLIC_ORIGIN), appRootSecret: env.SUPERCODE_TEAMS_BOOTSTRAP_TOKEN || null });
|
|
113
|
+
this.api.work = (name, fn, detail) => this.work.run(name, fn, detail);
|
|
114
|
+
this.api.workReport = () => ({ ...this.work.report(), persistence: { ...this.diagnosticPersistence } });
|
|
80
115
|
this.api.storageUsage = () => {
|
|
81
116
|
const sql = ctx.storage.sql;
|
|
82
117
|
let freeBytes = null;
|
|
83
|
-
try { freeBytes =
|
|
118
|
+
try { freeBytes = db.get('PRAGMA freelist_count').freelist_count * db.get('PRAGMA page_size').page_size; } catch { /* preserve an explicit unknown if the host omits these pragmas */ }
|
|
84
119
|
return { allocated: sql.databaseSize, free_bytes: freeBytes, used: Math.max(0, sql.databaseSize - (freeBytes ?? 0)), used_is_upper_bound: freeBytes === null, limit: 10 * 1024 ** 3 };
|
|
85
120
|
};
|
|
86
121
|
this.api.attach({ store: new DirectStore(createStoreHandlers(db)), harnessRoutes });
|
|
122
|
+
this.api.scheduleCapturePrefixWork = async () => {
|
|
123
|
+
const soon = Date.now() + 100, armed = await ctx.storage.getAlarm();
|
|
124
|
+
if (armed == null || armed > soon) await ctx.storage.setAlarm(soon);
|
|
125
|
+
};
|
|
87
126
|
const existing = await this.api.store.one("SELECT value FROM server_meta WHERE key='server_id'");
|
|
88
127
|
if (existing) this.api.attach({ serverId: existing.value });
|
|
89
128
|
else {
|
|
@@ -93,7 +132,16 @@ export class TeamsServer extends DurableObject {
|
|
|
93
132
|
this.api.attach({ serverId: await bootstrapServer(this.api, { initTeam: env.INIT_TEAM, bootstrap: { server_id: ids.next('server'), team_id: ids.next('team'), principal_id: ids.next('user'), credential_id: match[1], token: env.SUPERCODE_TEAMS_BOOTSTRAP_TOKEN } }) });
|
|
94
133
|
}
|
|
95
134
|
}
|
|
96
|
-
|
|
135
|
+
const handler = createHttpHandler(this.api);
|
|
136
|
+
this.handler = (req, res) => {
|
|
137
|
+
const pending = this.work.run('http', async () => {
|
|
138
|
+
const writeHead = res.writeHead;
|
|
139
|
+
res.writeHead = (...args) => { this.work.detail({ status: args[0], request_id: args[1]?.['X-Request-ID'] ?? null }); return writeHead.apply(res, args); };
|
|
140
|
+
return handler(req, res);
|
|
141
|
+
}, { method: req.method, route: workRoute(req.url) });
|
|
142
|
+
ctx.waitUntil(pending);
|
|
143
|
+
return pending;
|
|
144
|
+
};
|
|
97
145
|
// Retention never runs here: an activation serves its first request as soon as the store is open, and a reset is
|
|
98
146
|
// never followed by a pass that holds every request back (t_6c01c966). A capacity failure can exhaust an old
|
|
99
147
|
// alarm's retries, and an exhausted alarm still reads its old time, already past, while it never fires again: so
|
|
@@ -101,13 +149,13 @@ export class TeamsServer extends DurableObject {
|
|
|
101
149
|
const armed = await ctx.storage.getAlarm().catch(() => null);
|
|
102
150
|
const now = Date.now(), soon = now + 5_000;
|
|
103
151
|
if (armed == null || armed > soon || armed < now) await ctx.storage.setAlarm(soon).catch(error => console.error(`maintain alarm scheduling failed: ${error.message}`));
|
|
104
|
-
});
|
|
152
|
+
}));
|
|
105
153
|
}
|
|
106
154
|
|
|
107
155
|
async fetch(request) {
|
|
108
156
|
await this.ready;
|
|
109
157
|
if (!this.api.serverId) return Response.json({ version: 1, error: { code: 'unavailable', message: 'set the SUPERCODE_TEAMS_BOOTSTRAP_TOKEN secret (supercode teams server token) before first use' } }, { status: 503 });
|
|
110
|
-
if (request.headers.get('upgrade')?.toLowerCase() === 'websocket') return this.upgrade(request);
|
|
158
|
+
if (request.headers.get('upgrade')?.toLowerCase() === 'websocket') return this.work.run('websocket.upgrade', () => this.upgrade(request), { method: request.method, route: workRoute(request.url) });
|
|
111
159
|
return serveFetch(this.handler, request, { remoteAddress: request.headers.get('cf-connecting-ip') });
|
|
112
160
|
}
|
|
113
161
|
|
|
@@ -128,18 +176,31 @@ export class TeamsServer extends DurableObject {
|
|
|
128
176
|
|
|
129
177
|
async alarm() {
|
|
130
178
|
await this.ready;
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
179
|
+
let prefixes = null;
|
|
180
|
+
if (this.api.serverId) prefixes = await this.work.run('capturePrefix', async () => {
|
|
181
|
+
const result = await this.api.store.call('warmCapturePrefixes');
|
|
182
|
+
this.work.detail({ prepared_messages: result.messages, prepared_bytes: result.bytes, backlog: result.backlog });
|
|
183
|
+
return result;
|
|
184
|
+
}).catch(error => { console.error(`capture prefix preparation failed: ${error.message}`); return { backlog: true }; });
|
|
185
|
+
let done = null;
|
|
186
|
+
if (this.api.serverId && Date.now() >= this.nextMaintenanceAt) {
|
|
187
|
+
done = await this.work.run('alarm', () => maintain(this.api)).catch(error => { console.error(`maintain failed: ${error.message}`); return null; });
|
|
188
|
+
this.nextMaintenanceAt = Date.now() + (done ? maintainDelay(this.api, done) : MAINTAIN_MS);
|
|
189
|
+
const usage = this.api.storageUsage();
|
|
190
|
+
if (usage.used >= usage.limit * 0.8) console.warn(`Teams storage pressure: ${JSON.stringify(usage)}; native upload admission stops at 90%`);
|
|
191
|
+
}
|
|
134
192
|
// The database's size on every full pass, where `wrangler tail` reads it.
|
|
135
|
-
if (!done
|
|
193
|
+
if (done && !done.backlog) {
|
|
136
194
|
// What grows: free pages (space deletes left for reuse), and each busy table's newest rowid (its insert rate).
|
|
137
195
|
const sql = this.ctx.storage.sql, one = (query) => { try { return [...sql.exec(query)][0] ?? {}; } catch { return {}; } };
|
|
138
196
|
const tables = ['messages', 'capture_parts', 'captures', 'session_versions', 'idempotency', 'journal'].map((name) => `${name}=${one(`SELECT MAX(rowid) m FROM ${name}`).m ?? '-'}`).join(' ');
|
|
139
197
|
console.log(`maintain: database ${(sql.databaseSize / 1048576).toFixed(1)} MB free_pages=${one('PRAGMA freelist_count').freelist_count ?? '-'} ${tables}`);
|
|
140
198
|
}
|
|
141
199
|
// A backlog (versions or parts left over) is worked again in seconds, not a minute.
|
|
142
|
-
|
|
200
|
+
// A prefix batch does not accelerate retention. The one alarm owns two
|
|
201
|
+
// separate deadlines, so warm-up never repeats maintenance every 100 ms.
|
|
202
|
+
const next = Math.max(1, this.nextMaintenanceAt - Date.now());
|
|
203
|
+
await this.ctx.storage.setAlarm(Date.now() + (prefixes?.backlog ? Math.min(100, next) : next));
|
|
143
204
|
}
|
|
144
205
|
}
|
|
145
206
|
|
|
@@ -156,6 +217,7 @@ const objectFailure = (error) => ({
|
|
|
156
217
|
export default {
|
|
157
218
|
async fetch(request, env) {
|
|
158
219
|
const repeatable = request.method === 'GET' || request.method === 'HEAD';
|
|
220
|
+
const route = { method: request.method, route: workRoute(request.url) };
|
|
159
221
|
const call = () => env.TEAMS_SERVER.get(env.TEAMS_SERVER.idFromName('server')).fetch(repeatable ? request.clone() : request);
|
|
160
222
|
// A request safe to repeat (a read, or a link's WebSocket upgrade: every machine's connection) is tried once more
|
|
161
223
|
// when the runtime says the failure is retryable and the object is not overloaded: a reset object (its code
|
|
@@ -166,10 +228,10 @@ export default {
|
|
|
166
228
|
try { return await call(); }
|
|
167
229
|
catch (error) {
|
|
168
230
|
first = objectFailure(error);
|
|
169
|
-
if (!(repeatable && first.retryable && !first.overloaded)) return unavailable(first, null);
|
|
231
|
+
if (!(repeatable && first.retryable && !first.overloaded)) return unavailable(first, null, route);
|
|
170
232
|
}
|
|
171
233
|
try { return await call(); }
|
|
172
|
-
catch (error) { return unavailable(objectFailure(error), first); }
|
|
234
|
+
catch (error) { return unavailable(objectFailure(error), first, route); }
|
|
173
235
|
},
|
|
174
236
|
};
|
|
175
237
|
|
|
@@ -180,9 +242,9 @@ const PUBLIC_NAMES = new Set(['Error', 'TypeError', 'RangeError', 'ReferenceErro
|
|
|
180
242
|
/** Failures before the HTTP handler (object initialization, migrations, storage, a reset) as our API contract, not a
|
|
181
243
|
* platform HTML page. The answer goes to callers not yet authenticated, so it names only the error's kind and the
|
|
182
244
|
* runtime's flags; its message (storage or SQL text, internal paths) is in the log line its request id leads to. */
|
|
183
|
-
function unavailable(failure, retried) {
|
|
245
|
+
function unavailable(failure, retried, route) {
|
|
184
246
|
const requestId = crypto.randomUUID();
|
|
185
|
-
console.error(JSON.stringify({ event: 'teams_unavailable', request_id: requestId, failure, ...(retried ? { first: retried } : {}) }));
|
|
247
|
+
console.error(JSON.stringify({ event: 'teams_unavailable', request_id: requestId, ...route, failure, ...(retried ? { first: retried } : {}) }));
|
|
186
248
|
const cause = `${PUBLIC_NAMES.has(failure.name) ? failure.name : 'Error'}${failure.reset ? ', object reset' : ''}${failure.overloaded ? ', overloaded' : ''}${retried ? ', after one retry' : ''}`;
|
|
187
|
-
return Response.json({ version: 1, error: { code: 'unavailable', message: `Teams server is temporarily unavailable (${cause})`, request_id: requestId, retryable: true, cause: { name: PUBLIC_NAMES.has(failure.name) ? failure.name : 'Error', retryable: failure.retryable, overloaded: failure.overloaded, remote: failure.remote, reset: failure.reset } } }, { status: 503, headers: { 'Retry-After': failure.overloaded ? '15' : '5', 'X-Request-ID': requestId } });
|
|
249
|
+
return Response.json({ version: 1, error: { code: 'unavailable', message: `Teams server is temporarily unavailable (${cause})`, request_id: requestId, ...route, retryable: true, cause: { name: PUBLIC_NAMES.has(failure.name) ? failure.name : 'Error', retryable: failure.retryable, overloaded: failure.overloaded, remote: failure.remote, reset: failure.reset } } }, { status: 503, headers: { 'Retry-After': failure.overloaded ? '15' : '5', 'X-Request-ID': requestId } });
|
|
188
250
|
}
|