@volter/supercode-teams 0.3.141 → 0.3.143
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/bin/workspace/agent-session.mjs +38 -0
- package/bin/workspace.mjs +11 -17
- package/fleet.mjs +8 -3
- package/machine.mjs +29 -10
- package/package.json +2 -2
- package/server/event-routes.mjs +1 -1
- package/server/harness-routes.mjs +5 -0
- package/server/work-meter.mjs +84 -0
- package/worker/index.mjs +43 -16
package/README.md
CHANGED
|
@@ -188,3 +188,7 @@ Only the session custodian can grant this binding, in person or through an app t
|
|
|
188
188
|
no session-backed channel is created. The agent DM uses the exact resource `${ROOM}:dm:${PRINCIPAL}`.
|
|
189
189
|
`unbind` accepts the same flags. `--conversation` remains available for existing grants; it is mutually exclusive
|
|
190
190
|
with `--agent`. Opening a DM reads the existing session; sending a turn is a separate action.
|
|
191
|
+
|
|
192
|
+
## Read server work diagnostics
|
|
193
|
+
|
|
194
|
+
An administrator can run `supercode teams storage work --json` to read recent handler records and retained slow work. Records name normalized HTTP routes, catalog refreshes, transitions and maintenance steps, with SQL call and cursor scan/write counts. The Worker retains up to 32 slow records across a restart. Elapsed milliseconds are not CPU time; hosted per-query timing is unavailable. [Recorded private Worker proof](../../docs/evidence/teams-handler-work-2026-10-07.md).
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
// Native session hints choose the nearest actual harness when terminal servers inherited both variables.
|
|
2
2
|
import { spawnSync } from 'node:child_process';
|
|
3
3
|
import { basename } from 'node:path';
|
|
4
|
+
import { openLocalMachine, nodeRpc } from '../../client.mjs';
|
|
5
|
+
import { teamsHome, paths } from '@volter/teams/home';
|
|
6
|
+
import { ownClaim } from '../../live-sessions.mjs';
|
|
4
7
|
export function agentSessionHint(env=process.env){
|
|
5
8
|
if(env.SUPERCODE_AGENT_SESSION)return env.SUPERCODE_AGENT_SESSION;
|
|
6
9
|
const claude=env.CLAUDE_CODE_SESSION_ID,codex=env.CODEX_THREAD_ID;
|
|
@@ -18,3 +21,38 @@ export function agentSessionHint(env=process.env){
|
|
|
18
21
|
}
|
|
19
22
|
throw new Error('native session hints are ambiguous; name --session SESSION');
|
|
20
23
|
}
|
|
24
|
+
|
|
25
|
+
/** The daemon's retained binding is authoritative even when a launch has an old or empty context. */
|
|
26
|
+
export async function agentSessionEnvironment(env = process.env, session = agentSessionHint(env)) {
|
|
27
|
+
if (!session) return env;
|
|
28
|
+
let channel;
|
|
29
|
+
try {
|
|
30
|
+
channel = await openLocalMachine(paths(teamsHome(env)).socket);
|
|
31
|
+
const rpc = await nodeRpc(channel);
|
|
32
|
+
const identity = await rpc.call('harness.v1.agents.session', { session, claim: ownClaim(env) });
|
|
33
|
+
if (!identity) return env;
|
|
34
|
+
if (env.SUPERCODE_AGENT_ID && env.SUPERCODE_AGENT_ID !== identity.agent)
|
|
35
|
+
throw Object.assign(new Error('session ownership does not match its launch identity'), { code: 'forbidden' });
|
|
36
|
+
if (env.SUPERCODE_AGENT_ORG && env.SUPERCODE_AGENT_ORG !== identity.org)
|
|
37
|
+
throw Object.assign(new Error('session Org does not match its launch identity'), { code: 'forbidden' });
|
|
38
|
+
return { ...env, SUPERCODE_AGENT_CONTEXT: identity.context ?? '', SUPERCODE_AGENT_ID: identity.agent,
|
|
39
|
+
SUPERCODE_AGENT_ORG: identity.org, SUPERCODE_AGENT_ASSIGNMENTS: JSON.stringify(identity.assignments) };
|
|
40
|
+
} catch (error) {
|
|
41
|
+
// An offline/older daemon does not authorize a fallback to a person's context.
|
|
42
|
+
// An existing launch reader can still be checked by the server itself.
|
|
43
|
+
if (!['ENOENT', 'ECONNREFUSED', 'machine_offline', -32601].includes(error.code)) throw error;
|
|
44
|
+
return env;
|
|
45
|
+
} finally { channel?.close(); }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Choosing a context never changes an agent into the connected person. */
|
|
49
|
+
export function agentContextName(contexts, env = process.env, explicit = null) {
|
|
50
|
+
if (!env.SUPERCODE_AGENT_ID) return explicit ?? env.SUPERCODE_CONTEXT ?? contexts.read().current;
|
|
51
|
+
const name = explicit ?? (env.SUPERCODE_AGENT_CONTEXT || null);
|
|
52
|
+
if (!name) throw Object.assign(new Error('agent delegation is unavailable; no custodian context fallback is permitted'), { code: 'authentication_required' });
|
|
53
|
+
const selected = contexts.read().contexts[name];
|
|
54
|
+
if (!selected || selected.principal_id !== env.SUPERCODE_AGENT_ID
|
|
55
|
+
|| (env.SUPERCODE_AGENT_ORG && selected.team_id !== env.SUPERCODE_AGENT_ORG))
|
|
56
|
+
throw Object.assign(new Error('selected context does not belong to this agent and Org'), { code: 'forbidden' });
|
|
57
|
+
return name;
|
|
58
|
+
}
|
package/bin/workspace.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { StringDecoder } from 'node:string_decoder';
|
|
3
3
|
import { hostname } from 'node:os';
|
|
4
|
-
import { agentSessionHint } from './workspace/agent-session.mjs';
|
|
4
|
+
import { agentSessionHint, agentSessionEnvironment, agentContextName } from './workspace/agent-session.mjs';
|
|
5
5
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
6
6
|
import { existsSync, readFileSync, realpathSync, rmSync } from 'node:fs';
|
|
7
7
|
import { basename, dirname, join, resolve } from 'node:path';
|
|
@@ -60,20 +60,8 @@ export async function workspaceMain(argv = process.argv.slice(2), env = process.
|
|
|
60
60
|
if (verb === 'login') return loginCommand({ flags, origin: sub, rest, home, output });
|
|
61
61
|
if (verb === 'auth') return authCommand({ flags, sub, rest, home, contexts, output, env, dependencies });
|
|
62
62
|
if (verb === 'connect' && flags.enrollment_stdin) adoptEnrollment(contexts, home, flags);
|
|
63
|
-
if(['agent','sessions'].includes(verb))
|
|
64
|
-
|
|
65
|
-
if(session){
|
|
66
|
-
let channel;
|
|
67
|
-
try{
|
|
68
|
-
channel=await openLocalMachine(paths(teamsHome()).socket);const rpc=await nodeRpc(channel);
|
|
69
|
-
const identity=await rpc.call('harness.v1.agents.session',{session});
|
|
70
|
-
if(identity&&!flags.context&&!env.SUPERCODE_AGENT_CONTEXT)env={...env,SUPERCODE_AGENT_CONTEXT:identity.context,SUPERCODE_AGENT_ID:identity.agent,SUPERCODE_AGENT_ASSIGNMENTS:JSON.stringify(identity.assignments)};
|
|
71
|
-
}catch(error){
|
|
72
|
-
// Only absence of an older/offline daemon permits the ordinary plain-session context.
|
|
73
|
-
if(!['ENOENT','ECONNREFUSED','machine_offline',-32601].includes(error.code))throw error;
|
|
74
|
-
}finally{channel?.close();}
|
|
75
|
-
}
|
|
76
|
-
}
|
|
63
|
+
if(env.SUPERCODE_AGENT_ID||['agent','sessions'].includes(verb))
|
|
64
|
+
env=await agentSessionEnvironment(env,flags.session??agentSessionHint(env));
|
|
77
65
|
// Every door called from an agent starts under its delegated context. A sibling product verb
|
|
78
66
|
// must not silently recover the custodian's ordinary login when an agent's reader is bounded.
|
|
79
67
|
const agentContext = env.SUPERCODE_AGENT_CONTEXT || null;
|
|
@@ -87,7 +75,7 @@ export async function workspaceMain(argv = process.argv.slice(2), env = process.
|
|
|
87
75
|
throw new Error('agent delegation is unavailable; no custodian context fallback is permitted');
|
|
88
76
|
}
|
|
89
77
|
const explicitContext = flags.context ?? agentContext ?? env.SUPERCODE_CONTEXT;
|
|
90
|
-
let selected =
|
|
78
|
+
let selected = agentContextName(contexts,env,flags.context??null);
|
|
91
79
|
const urlReference = verb === 'sessions' ? rest.find(value => /^https?:\/\//.test(value)) : null;
|
|
92
80
|
if (!explicitContext && urlReference) selected = contextForUrl(contexts.read(), urlReference) ?? selected;
|
|
93
81
|
if (!selected) usage('no workspace context selected');
|
|
@@ -107,6 +95,11 @@ export async function workspaceMain(argv = process.argv.slice(2), env = process.
|
|
|
107
95
|
if (verb === 'sessions') return sessionsCommand(sub, command);
|
|
108
96
|
if (verb === 'calendar') return calendarCommand(sub, { ...command, teamPath });
|
|
109
97
|
if (verb === 'settings') return settingsCommand(sub, command);
|
|
98
|
+
if (verb === 'storage') {
|
|
99
|
+
only(flags, ['json', 'help', 'context']);requireArgs(rest, 0, 0, 'storage work');
|
|
100
|
+
if(sub!=='work')usage('storage work');
|
|
101
|
+
return output(await client.request('GET',teamPath(context,'/storage/work')));
|
|
102
|
+
}
|
|
110
103
|
if (verb === 'members') return membersCommand(sub, command);
|
|
111
104
|
if (verb === 'groups') return groupsCommand(sub, command);
|
|
112
105
|
if (verb === 'sources') return sourcesCommand(sub, command);
|
|
@@ -871,7 +864,7 @@ async function machinesCommand(sub, { flags, rest, output, client, context, team
|
|
|
871
864
|
}
|
|
872
865
|
if (action === 'grant') {
|
|
873
866
|
for (const key of ['to', 'caps', 'ttl']) if (!flags[key]) usage(`--${key} is required`);
|
|
874
|
-
const subject = /^(user|service|group):(.+)$/.exec(flags.to)?.[2]; if (!subject) usage('--to must be user:ID, service:ID or group:ID');
|
|
867
|
+
const subject = /^(user|service|agent|group):(.+)$/.exec(flags.to)?.[2]; if (!subject) usage('--to must be user:ID, service:ID, agent:ID or group:ID');
|
|
875
868
|
const ttl = duration(flags.ttl);
|
|
876
869
|
if (flags.session || flags.binding) {
|
|
877
870
|
// One session binding's native live actions (§5).
|
|
@@ -1446,6 +1439,7 @@ function printHelp() {
|
|
|
1446
1439
|
identity link
|
|
1447
1440
|
identities observations|sightings|resolve|observe|declare-person|link|infer|revoke|revoke-inference --body-file FILE
|
|
1448
1441
|
server start [--data-dir DIR] [--listen HOST:PORT] [--public-origin ORIGIN] [--init-team NAME] | server token
|
|
1442
|
+
storage work [--context NAME] [--json] administrators: recent handler work and retained slow records
|
|
1449
1443
|
publish pane|root TARGET --room ROOM --workplace ORIGIN --organization ORG [--scope view|control] [--name] [--key] [--machine] [--token-stdin] [--record-only]
|
|
1450
1444
|
unpublish SURFACE --room ROOM --workplace ORIGIN --organization ORG [--machine] [--token-stdin] [--record-only|--workplace-only]
|
|
1451
1445
|
connect|disconnect|status|fold (fold: the connector's state back into connector.json for an older daemon, with it stopped)
|
package/fleet.mjs
CHANGED
|
@@ -8,9 +8,14 @@ import { harnessTransport, nodeRpc, openRemoteMachine } from './client.mjs';
|
|
|
8
8
|
import { openMachine } from './machine-door.mjs';
|
|
9
9
|
import { WorkspaceContexts, fromContext } from './context.mjs';
|
|
10
10
|
import { teamsHome } from '@volter/teams/home';
|
|
11
|
+
import { agentSessionEnvironment, agentContextName } from './bin/workspace/agent-session.mjs';
|
|
11
12
|
|
|
12
13
|
/** Open the channel to the machine `name` names in `context` (or this box's daemon when name is null). */
|
|
13
|
-
export function openOn(name = null, { context = null, env = process.env } = {}) {
|
|
14
|
+
export async function openOn(name = null, { context = null, env = process.env } = {}) {
|
|
15
|
+
if (name && env.SUPERCODE_AGENT_ID) {
|
|
16
|
+
env = await agentSessionEnvironment(env);
|
|
17
|
+
context = agentContextName(new WorkspaceContexts(env.SUPERCODE_HOME ?? resolve(teamsHome(env), '..')), env, context);
|
|
18
|
+
}
|
|
14
19
|
return openMachine({ machine: name, context, env });
|
|
15
20
|
}
|
|
16
21
|
|
|
@@ -44,6 +49,7 @@ export async function nodeRequest(channel, method, params = {}) {
|
|
|
44
49
|
* that refused or is offline carries its reason. Nothing is stored.
|
|
45
50
|
*/
|
|
46
51
|
export async function fleetDiscover({ context = null, machines = null, params = {}, env = process.env } = {}) {
|
|
52
|
+
if (env.SUPERCODE_AGENT_ID) env = await agentSessionEnvironment(env);
|
|
47
53
|
// One page from each machine, never its whole catalog (yuerans' was 5,620 sessions, 20 MB, 13.5 s): the caller's
|
|
48
54
|
// limit, or FLEET_PAGE; each machine answers it from its daemon's held index.
|
|
49
55
|
// `--limit 0` is every session, as it always was
|
|
@@ -51,7 +57,7 @@ export async function fleetDiscover({ context = null, machines = null, params =
|
|
|
51
57
|
const home = teamsHome(env);
|
|
52
58
|
const supercodeHome = env.SUPERCODE_HOME ?? resolve(home, '..');
|
|
53
59
|
const contexts = new WorkspaceContexts(supercodeHome);
|
|
54
|
-
const name =
|
|
60
|
+
const name = agentContextName(contexts, env, context);
|
|
55
61
|
if (!name || name === 'local') throw new Error('--fleet discovers the machines of a Teams context: `supercode teams login …`, then `supercode context use NAME` or --context NAME');
|
|
56
62
|
const { client, context: selected } = fromContext({ supercodeHome, name, fetch: globalThis.fetch });
|
|
57
63
|
const token = contexts.credential(selected.credential_ref).token;
|
|
@@ -105,4 +111,3 @@ export async function fleetDiscover({ context = null, machines = null, params =
|
|
|
105
111
|
|
|
106
112
|
/** Sessions `discover --fleet` asks each machine for when the caller names no limit. */
|
|
107
113
|
const FLEET_PAGE = 50;
|
|
108
|
-
|
package/machine.mjs
CHANGED
|
@@ -682,13 +682,15 @@ export class TeamsMachine extends MachineCore {
|
|
|
682
682
|
// one follow at a time: a change while one is in flight is followed again when it ends, never beside it (two in flight
|
|
683
683
|
// would leave the earlier subscription unsubscribed)
|
|
684
684
|
let pending = null, followed = '', backoffMs = 0, running = false, again = false;
|
|
685
|
+
this.stopLiveFollow = () => { clearTimeout(pending); pending = null; again = false; this.live.stop(); };
|
|
685
686
|
const follow = async () => {
|
|
686
687
|
pending = null;
|
|
688
|
+
if (this.stopping) return;
|
|
687
689
|
if (running) { again = true; return; }
|
|
688
690
|
running = true;
|
|
689
691
|
try { await followNow(); } finally {
|
|
690
692
|
running = false;
|
|
691
|
-
if (again) { again = false; if (!pending) { pending = setTimeout(follow, 0); pending.unref?.(); } }
|
|
693
|
+
if (again && !this.stopping) { again = false; if (!pending) { pending = setTimeout(follow, 0); pending.unref?.(); } }
|
|
692
694
|
}
|
|
693
695
|
};
|
|
694
696
|
const followNow = async () => {
|
|
@@ -699,6 +701,7 @@ export class TeamsMachine extends MachineCore {
|
|
|
699
701
|
try {
|
|
700
702
|
if (locators.length) {
|
|
701
703
|
const opened = await this.#serveRequest('harness.v1.sessions.activity.subscribe', { locators }, 30_000);
|
|
704
|
+
if (this.stopping) return;
|
|
702
705
|
this.liveActivity = opened?.subscription ?? null;
|
|
703
706
|
this.live.codexActivity(opened?.initial ?? [], { replace: true });
|
|
704
707
|
} else { this.liveActivity = null; this.live.codexActivity([], { replace: true }); }
|
|
@@ -706,6 +709,7 @@ export class TeamsMachine extends MachineCore {
|
|
|
706
709
|
this.live.codexFollowedNow();
|
|
707
710
|
if (previous) void this.#serveRequest('harness.v1.sessions.activity.unsubscribe', { subscription: previous }, 30_000).catch(() => {});
|
|
708
711
|
} catch (error) {
|
|
712
|
+
if (this.stopping) return;
|
|
709
713
|
backoffMs = Math.min(300_000, Math.max(2000, (backoffMs || 1000) * 2));
|
|
710
714
|
this.emit('records-log', `live sessions: the Codex sessions' activity could not be followed (${error.message}); tried again in ${Math.round(backoffMs / 1000)} s`);
|
|
711
715
|
pending = setTimeout(follow, backoffMs); pending.unref?.();
|
|
@@ -713,7 +717,7 @@ export class TeamsMachine extends MachineCore {
|
|
|
713
717
|
};
|
|
714
718
|
// a change is followed after 2 s (changes come in bursts); the first follow after the index's first answer at once,
|
|
715
719
|
// as sends wait on it (the Codex sessions are not known until it is done)
|
|
716
|
-
this.live.on('codex-followed', () => { if (!pending) { pending = setTimeout(follow, this.live.codexKnown ? 2000 : 0); pending.unref?.(); } });
|
|
720
|
+
this.live.on('codex-followed', () => { if (!this.stopping && !pending) { pending = setTimeout(follow, this.live.codexKnown ? 2000 : 0); pending.unref?.(); } });
|
|
717
721
|
this.on('internal-notification', (message) => {
|
|
718
722
|
const params = message.params ?? {};
|
|
719
723
|
if (params.subscription && params.subscription === this.liveActivity) this.live.codexActivity(params.activities);
|
|
@@ -1158,6 +1162,8 @@ export class TeamsMachine extends MachineCore {
|
|
|
1158
1162
|
this.jobs?.stopAll();
|
|
1159
1163
|
void this.relayEndpoint?.then((endpoint) => endpoint?.close()).catch(() => {});
|
|
1160
1164
|
this.carrier?.stop();
|
|
1165
|
+
this.stopLiveFollow?.();
|
|
1166
|
+
this.mailStore?.stop();
|
|
1161
1167
|
clearInterval(this.hookSpoolTimer);
|
|
1162
1168
|
this.idleNotices?.stop();
|
|
1163
1169
|
this.agentUpkeep?.stop();
|
|
@@ -1353,17 +1359,30 @@ export class TeamsMachine extends MachineCore {
|
|
|
1353
1359
|
accept();
|
|
1354
1360
|
this.log.append({ principal, cap: 'harness', target: '*', outcome: 'opened' });
|
|
1355
1361
|
const lines = lineSplitter();
|
|
1362
|
+
let forwarding = Promise.resolve();
|
|
1356
1363
|
stream.on('data', (chunk) => {
|
|
1357
1364
|
for (const line of lines(chunk)) {
|
|
1358
1365
|
if (!line) continue;
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1366
|
+
forwarding = forwarding.then(async () => {
|
|
1367
|
+
if (stream.closed) return;
|
|
1368
|
+
if (entry.verify) {
|
|
1369
|
+
const fresh = await entry.verify();
|
|
1370
|
+
if (fresh?.class !== 'operator') {
|
|
1371
|
+
this.log.append({ principal, cap: 'harness', target: '*', outcome: 'refused', detail: 'no live operator grant' });
|
|
1372
|
+
return stream.close('no live operator grant');
|
|
1373
|
+
}
|
|
1374
|
+
entry.admitted = fresh;
|
|
1375
|
+
}
|
|
1376
|
+
if (stream.closed) return;
|
|
1377
|
+
let message;
|
|
1378
|
+
try { message = JSON.parse(line); } catch { return; }
|
|
1379
|
+
const asked = message.id !== undefined && message.id !== null ? message.id : null;
|
|
1380
|
+
if (asked !== null) message.id = joinId(index, message.id);
|
|
1381
|
+
// a serve that is not there to take it answers the request at once, on this stream only
|
|
1382
|
+
if (!this.#toServe(JSON.stringify(message) + '\n') && asked !== null && !stream.closed) {
|
|
1383
|
+
try { stream.write(`${JSON.stringify({ jsonrpc: '2.0', id: asked, error: { code: -32000, message: 'harness serve is not running; it is being started again' } })}\n`); } catch { /* the stream went */ }
|
|
1384
|
+
}
|
|
1385
|
+
}).catch(() => { if (!stream.closed) stream.close('machine authority could not be verified'); });
|
|
1367
1386
|
}
|
|
1368
1387
|
});
|
|
1369
1388
|
stream.on('close', () => { entry.harness = null; });
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-teams",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.143",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A machine daemon serving this machine's doors, and a local or remote Teams server for shared session discovery, OpenTelemetry ingestion and access to enrolled machines.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -70,7 +70,7 @@
|
|
|
70
70
|
"@volter/supercode-health": "^0.1.0",
|
|
71
71
|
"@volter/supercode-terminal": "^0.2.54",
|
|
72
72
|
"@volter/supercode-trace": "^0.1.0",
|
|
73
|
-
"@volter/teams": "0.2.
|
|
73
|
+
"@volter/teams": "0.2.38",
|
|
74
74
|
"@xterm/addon-serialize": "^0.14.0",
|
|
75
75
|
"@xterm/headless": "^6.0.0",
|
|
76
76
|
"protobufjs": "8.8.0",
|
package/server/event-routes.mjs
CHANGED
|
@@ -221,7 +221,7 @@ export async function handleCatalogEvents({ api, req, res, auth, url, requestId,
|
|
|
221
221
|
const stream = await openStream(api, req, res, auth, requestId);
|
|
222
222
|
let lastCursor = null;
|
|
223
223
|
const enqueue = streamQueue(stream, async () => {
|
|
224
|
-
const page = await catalogWindow(api, auth, query);
|
|
224
|
+
const page = await (api.work ? api.work('catalogWindow', () => catalogWindow(api, auth, query)) : catalogWindow(api, auth, query));
|
|
225
225
|
// Authorization and expiry can change a window without creating a session revision.
|
|
226
226
|
// A binding's observed_at moves with every heartbeat; only a change of what the window says is a reset.
|
|
227
227
|
const digest = createHash('sha256').update(JSON.stringify(page.data, (key, value) => key === 'bindings' && Array.isArray(value) ? value.map(({ observed_at: _seen, ...binding }) => binding) : value)).digest('hex');
|
|
@@ -88,6 +88,11 @@ async function beforeIdempotency({ api, req, res, auth, url, path, sub, requestI
|
|
|
88
88
|
|
|
89
89
|
async function team({ api, req, res, auth, url, path, sub, requestId }) {
|
|
90
90
|
let m;
|
|
91
|
+
if(req.method==='GET'&&sub==='/storage/work'){
|
|
92
|
+
if(!isAdmin(auth))fail(403,'forbidden','Teams work diagnostics require an administrator');
|
|
93
|
+
if(!api.workReport)fail(501,'unsupported','this host does not record Teams work diagnostics');
|
|
94
|
+
send(res,200,result(api,auth,api.workReport()),requestId);return true;
|
|
95
|
+
}
|
|
91
96
|
if(req.method==='GET'&&sub==='/storage/uploads'){await storageReport({api,res,auth,url,requestId});return true;}
|
|
92
97
|
if(req.method==='GET'&&sub==='/sessions'){await listSessions(api,res,auth,url,requestId,false);return true;}
|
|
93
98
|
if(req.method==='GET'&&sub==='/sessions/search'){await listSessions(api,res,auth,url,requestId,true);return true;}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
|
|
4
|
+
// Only contract path words survive. Never record a caller's query, identity, body or SQL parameters.
|
|
5
|
+
const WORDS = new Set('api v1 me health teams sessions messages search events catalogue machines heartbeat configuration roots sources captures parts commit uploads chunks members groups credentials permissions bindings agents publications attachments operations continuations sync-rules view-bindings exports audit storage work'.split(' '));
|
|
6
|
+
export function workRoute(input) {
|
|
7
|
+
try { return new URL(input, 'http://local').pathname.split('/').map(part => !part || WORDS.has(part) ? part : ':id').join('/'); }
|
|
8
|
+
catch { return '/:invalid'; }
|
|
9
|
+
}
|
|
10
|
+
const rounded = value => Math.round(value * 1000) / 1000;
|
|
11
|
+
|
|
12
|
+
/** Attribute synchronous SQLite work to its handler, rather than the invocation a timer happened to inherit. */
|
|
13
|
+
export class WorkMeter {
|
|
14
|
+
constructor({ emit = record => console.log(JSON.stringify(record)), now = () => performance.now(), retainSlow = null, advanceClock = null, sqlTiming = true } = {}) {
|
|
15
|
+
this.now = now; this.advanceClock = advanceClock; this.sqlTiming = sqlTiming;
|
|
16
|
+
this.context = new AsyncLocalStorage(); this.recent = []; this.slow = [];
|
|
17
|
+
this.emit = record => {
|
|
18
|
+
emit(record);
|
|
19
|
+
if (!['teams_work', 'teams_sql_work'].includes(record.event)) return;
|
|
20
|
+
this.recent.push(record); if (this.recent.length > 64) this.recent.shift();
|
|
21
|
+
if ((record.elapsed_ms ?? 0) < 1000 && (record.rows_read ?? 0) + (record.rows_written ?? 0) < 5000) return;
|
|
22
|
+
this.slow.push(record); if (this.slow.length > 32) this.slow.shift();
|
|
23
|
+
try { retainSlow?.(this.slow); }
|
|
24
|
+
catch { emit({ event: 'teams_work_retention_failed', work_id: record.work_id ?? null }); }
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
restoreSlow(records) { if (Array.isArray(records)) this.slow = records.slice(-32); }
|
|
28
|
+
report() { return { clock: this.advanceClock ? 'elapsed milliseconds through an end-of-work timer boundary; SQL timing unavailable on the hosted Worker' : 'elapsed milliseconds; sql_ms measures synchronous SQL, not CPU', recent: this.recent.slice(), slow: this.slow.slice() }; }
|
|
29
|
+
async run(work, fn, detail = {}) {
|
|
30
|
+
const parent = this.context.getStore();
|
|
31
|
+
const record = { work, ...detail, work_id: crypto.randomUUID(), parent_work_id: parent?.work_id ?? null, started_at: new Date().toISOString(), started: this.now(), sql_ms: 0, sql_calls: 0, rows_read: 0, rows_written: 0, rows_known: true, returned_rows: 0, slowest_sql: null, most_rows_sql: null, finished: false };
|
|
32
|
+
this.emit({ event: 'teams_work_start', work, ...detail, work_id: record.work_id, parent_work_id: record.parent_work_id, started_at: record.started_at });
|
|
33
|
+
return this.context.run(record, async () => {
|
|
34
|
+
let outcome = 'ok';
|
|
35
|
+
try { return await fn(); }
|
|
36
|
+
catch (error) { outcome = 'error'; throw error; }
|
|
37
|
+
finally {
|
|
38
|
+
record.finished = true;
|
|
39
|
+
// Hosted Workers freeze clocks during CPU work. Yield once after the work, not once per SQL call.
|
|
40
|
+
// This reports elapsed time, including the timer's wait; it does not manufacture a CPU reading.
|
|
41
|
+
if (this.advanceClock) await this.advanceClock();
|
|
42
|
+
const { started, finished: _finished, rows_known, ...data } = record;
|
|
43
|
+
this.emit({ event: 'teams_work', ...data, slowest_sql: this.sqlTiming ? record.slowest_sql : null, rows_read: rows_known ? record.rows_read : null, rows_written: rows_known ? record.rows_written : null, elapsed_ms: rounded(this.now() - started), sql_ms: this.sqlTiming ? rounded(record.sql_ms) : null, outcome });
|
|
44
|
+
// Nested steps are separately named, and their SQL remains counted in the enclosing handler.
|
|
45
|
+
if (parent && !parent.finished) {
|
|
46
|
+
for (const field of ['sql_ms', 'sql_calls', 'rows_read', 'rows_written', 'returned_rows']) parent[field] += record[field];
|
|
47
|
+
parent.rows_known &&= record.rows_known;
|
|
48
|
+
if (record.slowest_sql && (!parent.slowest_sql || record.slowest_sql.elapsed_ms > parent.slowest_sql.elapsed_ms)) parent.slowest_sql = record.slowest_sql;
|
|
49
|
+
if (record.most_rows_sql && (!parent.most_rows_sql || record.most_rows_sql.rows_read + record.most_rows_sql.rows_written > parent.most_rows_sql.rows_read + parent.most_rows_sql.rows_written)) parent.most_rows_sql = record.most_rows_sql;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
detail(values) {
|
|
55
|
+
const record = this.context.getStore();
|
|
56
|
+
if (record && !record.finished) Object.assign(record, values);
|
|
57
|
+
}
|
|
58
|
+
sql(query, fn) {
|
|
59
|
+
const record = this.context.getStore();
|
|
60
|
+
if (!record || record.finished) {
|
|
61
|
+
const start = this.now(); let result;
|
|
62
|
+
try { result = fn(); return result; }
|
|
63
|
+
finally {
|
|
64
|
+
const elapsed = this.now() - start;
|
|
65
|
+
if (elapsed >= 100 || (result?.rowsRead ?? 0) >= 5000) this.emit({ event: 'teams_sql_work', work: 'unscoped', origin_work: record?.work ?? null, fingerprint: createHash('sha256').update(query).digest('hex').slice(0, 16), elapsed_ms: this.sqlTiming ? rounded(elapsed) : null, rows_read: result?.rowsRead ?? null, rows_written: result?.rowsWritten ?? null, returned_rows: result?.returnedRows ?? null });
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
const start = this.now(); let result;
|
|
69
|
+
try { result = fn(); return result; }
|
|
70
|
+
finally {
|
|
71
|
+
const elapsed = this.now() - start;
|
|
72
|
+
record.sql_calls++; record.sql_ms += elapsed;
|
|
73
|
+
record.rows_known &&= Number.isFinite(result?.rowsRead) && Number.isFinite(result?.rowsWritten);
|
|
74
|
+
record.rows_read += result?.rowsRead ?? 0; record.rows_written += result?.rowsWritten ?? 0;
|
|
75
|
+
record.returned_rows += result?.returnedRows ?? 0;
|
|
76
|
+
if (!record.slowest_sql || elapsed > record.slowest_sql.elapsed_ms) {
|
|
77
|
+
record.slowest_sql = { fingerprint: createHash('sha256').update(query).digest('hex').slice(0, 16), operation: /^\s*(\w+)/.exec(query)?.[1]?.toUpperCase() ?? 'UNKNOWN', elapsed_ms: rounded(elapsed), rows_read: result?.rowsRead ?? null, rows_written: result?.rowsWritten ?? null, returned_rows: result?.returnedRows ?? null };
|
|
78
|
+
}
|
|
79
|
+
if (Number.isFinite(result?.rowsRead) && Number.isFinite(result?.rowsWritten) && (!record.most_rows_sql || result.rowsRead + result.rowsWritten > record.most_rows_sql.rows_read + record.most_rows_sql.rows_written)) {
|
|
80
|
+
record.most_rows_sql = { fingerprint: createHash('sha256').update(query).digest('hex').slice(0, 16), operation: /^\s*(\w+)/.exec(query)?.[1]?.toUpperCase() ?? 'UNKNOWN', rows_read: result.rowsRead, rows_written: result.rowsWritten, returned_rows: result?.returnedRows ?? null };
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
package/worker/index.mjs
CHANGED
|
@@ -18,6 +18,7 @@ import { setOtlpSchemaSources } from '../server/otlp/transport.mjs';
|
|
|
18
18
|
import { setViewerSources } from '@volter/teams/server/viewer';
|
|
19
19
|
import { serveFetch } from './fetch-adapter.mjs';
|
|
20
20
|
import { webSocketDuplex } from './websocket-duplex.mjs';
|
|
21
|
+
import { WorkMeter, workRoute } from '../server/work-meter.mjs';
|
|
21
22
|
// The hosted viewer's files, bundled as text (wrangler.toml rules) and served at /view.
|
|
22
23
|
import viewerPage from '../../volter-teams/viewer/index.html';
|
|
23
24
|
import viewerScript from '../../volter-teams/viewer/viewer.mjs';
|
|
@@ -56,31 +57,47 @@ const MAINTAIN_MS = 60_000;
|
|
|
56
57
|
const BOOTSTRAP_TOKEN = /^sct_(cred_[0-9a-f]{32})\.[A-Za-z0-9_-]{16,}$/;
|
|
57
58
|
|
|
58
59
|
/** SQLite primitives over the object's storage, in store-core's shape. */
|
|
59
|
-
function durableSql(storage) {
|
|
60
|
+
function durableSql(storage, meter) {
|
|
60
61
|
const sql = storage.sql;
|
|
61
62
|
const param = (value) => (value instanceof Uint8Array ? value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength) : value);
|
|
62
63
|
const row = (value) => { for (const key of Object.keys(value)) if (value[key] instanceof ArrayBuffer) value[key] = Buffer.from(value[key]); return value; };
|
|
63
|
-
const
|
|
64
|
+
const measured = (query, fn) => meter.sql(query, fn);
|
|
65
|
+
const all = (query, params = []) => measured(query, () => {
|
|
66
|
+
const cursor = sql.exec(query, ...params.map(param));
|
|
67
|
+
const values = cursor.toArray().map(row);
|
|
68
|
+
return { values, rowsRead: cursor.rowsRead, rowsWritten: cursor.rowsWritten, returnedRows: values.length };
|
|
69
|
+
}).values;
|
|
64
70
|
const get = (query, params = []) => all(query, params)[0] ?? null;
|
|
65
|
-
const run = (query, params = []) => {
|
|
71
|
+
const run = (query, params = []) => {
|
|
72
|
+
const counts = measured(query, () => {
|
|
73
|
+
const cursor = sql.exec(query, ...params.map(param));
|
|
74
|
+
return { rowsRead: cursor.rowsRead, rowsWritten: cursor.rowsWritten, returnedRows: 0 };
|
|
75
|
+
});
|
|
76
|
+
return { ...counts, changes: get('SELECT changes() AS c').c };
|
|
77
|
+
};
|
|
66
78
|
const tx = (fn) => storage.transactionSync(fn);
|
|
67
|
-
return { all, get, run, tx, exec: (query) =>
|
|
79
|
+
return { all, get, run, tx, exec: (query) => { all(query); } };
|
|
68
80
|
}
|
|
69
81
|
|
|
70
82
|
export class TeamsServer extends DurableObject {
|
|
71
83
|
constructor(ctx, env) {
|
|
72
84
|
super(ctx, env);
|
|
73
|
-
this.
|
|
85
|
+
this.work = new WorkMeter({ retainSlow: records => ctx.storage.kv.put('teams_work_slow_v1', JSON.stringify(records)), advanceClock: () => scheduler.wait(0), sqlTiming: false });
|
|
86
|
+
this.ready = ctx.blockConcurrencyWhile(() => this.work.run('activation', async () => {
|
|
74
87
|
// Read before migrations/maintenance: allocation failures must not hide
|
|
75
88
|
// the table responsible, including hosts with a lower storage limit.
|
|
76
89
|
console.log(`storage: database_bytes=${ctx.storage.sql.databaseSize}`);
|
|
77
|
-
const db = durableSql(ctx.storage);
|
|
90
|
+
const db = durableSql(ctx.storage, this.work);
|
|
78
91
|
migrate({ exec: db.exec, tx: db.tx, version: () => ctx.storage.kv.get('schema_version') ?? 0, setVersion: (n) => ctx.storage.kv.put('schema_version', n) });
|
|
92
|
+
const retained = ctx.storage.kv.get('teams_work_slow_v1');
|
|
93
|
+
if (retained) this.work.restoreSlow(JSON.parse(retained));
|
|
79
94
|
this.api = createTeamsApi({ publicOrigin: normalizePublicOrigin(env.PUBLIC_ORIGIN), appRootSecret: env.SUPERCODE_TEAMS_BOOTSTRAP_TOKEN || null });
|
|
95
|
+
this.api.work = (name, fn, detail) => this.work.run(name, fn, detail);
|
|
96
|
+
this.api.workReport = () => this.work.report();
|
|
80
97
|
this.api.storageUsage = () => {
|
|
81
98
|
const sql = ctx.storage.sql;
|
|
82
99
|
let freeBytes = null;
|
|
83
|
-
try { freeBytes =
|
|
100
|
+
try { freeBytes = db.get('PRAGMA freelist_count').freelist_count * db.get('PRAGMA page_size').page_size; } catch { /* preserve an explicit unknown if the host omits these pragmas */ }
|
|
84
101
|
return { allocated: sql.databaseSize, free_bytes: freeBytes, used: Math.max(0, sql.databaseSize - (freeBytes ?? 0)), used_is_upper_bound: freeBytes === null, limit: 10 * 1024 ** 3 };
|
|
85
102
|
};
|
|
86
103
|
this.api.attach({ store: new DirectStore(createStoreHandlers(db)), harnessRoutes });
|
|
@@ -93,7 +110,16 @@ export class TeamsServer extends DurableObject {
|
|
|
93
110
|
this.api.attach({ serverId: await bootstrapServer(this.api, { initTeam: env.INIT_TEAM, bootstrap: { server_id: ids.next('server'), team_id: ids.next('team'), principal_id: ids.next('user'), credential_id: match[1], token: env.SUPERCODE_TEAMS_BOOTSTRAP_TOKEN } }) });
|
|
94
111
|
}
|
|
95
112
|
}
|
|
96
|
-
|
|
113
|
+
const handler = createHttpHandler(this.api);
|
|
114
|
+
this.handler = (req, res) => {
|
|
115
|
+
const pending = this.work.run('http', async () => {
|
|
116
|
+
const writeHead = res.writeHead;
|
|
117
|
+
res.writeHead = (...args) => { this.work.detail({ status: args[0], request_id: args[1]?.['X-Request-ID'] ?? null }); return writeHead.apply(res, args); };
|
|
118
|
+
return handler(req, res);
|
|
119
|
+
}, { method: req.method, route: workRoute(req.url) });
|
|
120
|
+
ctx.waitUntil(pending);
|
|
121
|
+
return pending;
|
|
122
|
+
};
|
|
97
123
|
// Retention never runs here: an activation serves its first request as soon as the store is open, and a reset is
|
|
98
124
|
// never followed by a pass that holds every request back (t_6c01c966). A capacity failure can exhaust an old
|
|
99
125
|
// alarm's retries, and an exhausted alarm still reads its old time, already past, while it never fires again: so
|
|
@@ -101,13 +127,13 @@ export class TeamsServer extends DurableObject {
|
|
|
101
127
|
const armed = await ctx.storage.getAlarm().catch(() => null);
|
|
102
128
|
const now = Date.now(), soon = now + 5_000;
|
|
103
129
|
if (armed == null || armed > soon || armed < now) await ctx.storage.setAlarm(soon).catch(error => console.error(`maintain alarm scheduling failed: ${error.message}`));
|
|
104
|
-
});
|
|
130
|
+
}));
|
|
105
131
|
}
|
|
106
132
|
|
|
107
133
|
async fetch(request) {
|
|
108
134
|
await this.ready;
|
|
109
135
|
if (!this.api.serverId) return Response.json({ version: 1, error: { code: 'unavailable', message: 'set the SUPERCODE_TEAMS_BOOTSTRAP_TOKEN secret (supercode teams server token) before first use' } }, { status: 503 });
|
|
110
|
-
if (request.headers.get('upgrade')?.toLowerCase() === 'websocket') return this.upgrade(request);
|
|
136
|
+
if (request.headers.get('upgrade')?.toLowerCase() === 'websocket') return this.work.run('websocket.upgrade', () => this.upgrade(request), { method: request.method, route: workRoute(request.url) });
|
|
111
137
|
return serveFetch(this.handler, request, { remoteAddress: request.headers.get('cf-connecting-ip') });
|
|
112
138
|
}
|
|
113
139
|
|
|
@@ -128,7 +154,7 @@ export class TeamsServer extends DurableObject {
|
|
|
128
154
|
|
|
129
155
|
async alarm() {
|
|
130
156
|
await this.ready;
|
|
131
|
-
const done = this.api.serverId ? await maintain(this.api).catch(error => { console.error(`maintain failed: ${error.message}`); return null; }) : null;
|
|
157
|
+
const done = this.api.serverId ? await this.work.run('alarm', () => maintain(this.api)).catch(error => { console.error(`maintain failed: ${error.message}`); return null; }) : null;
|
|
132
158
|
const usage = this.api.storageUsage();
|
|
133
159
|
if (usage.used >= usage.limit * 0.8) console.warn(`Teams storage pressure: ${JSON.stringify(usage)}; native upload admission stops at 90%`);
|
|
134
160
|
// The database's size on every full pass, where `wrangler tail` reads it.
|
|
@@ -156,6 +182,7 @@ const objectFailure = (error) => ({
|
|
|
156
182
|
export default {
|
|
157
183
|
async fetch(request, env) {
|
|
158
184
|
const repeatable = request.method === 'GET' || request.method === 'HEAD';
|
|
185
|
+
const route = { method: request.method, route: workRoute(request.url) };
|
|
159
186
|
const call = () => env.TEAMS_SERVER.get(env.TEAMS_SERVER.idFromName('server')).fetch(repeatable ? request.clone() : request);
|
|
160
187
|
// A request safe to repeat (a read, or a link's WebSocket upgrade: every machine's connection) is tried once more
|
|
161
188
|
// when the runtime says the failure is retryable and the object is not overloaded: a reset object (its code
|
|
@@ -166,10 +193,10 @@ export default {
|
|
|
166
193
|
try { return await call(); }
|
|
167
194
|
catch (error) {
|
|
168
195
|
first = objectFailure(error);
|
|
169
|
-
if (!(repeatable && first.retryable && !first.overloaded)) return unavailable(first, null);
|
|
196
|
+
if (!(repeatable && first.retryable && !first.overloaded)) return unavailable(first, null, route);
|
|
170
197
|
}
|
|
171
198
|
try { return await call(); }
|
|
172
|
-
catch (error) { return unavailable(objectFailure(error), first); }
|
|
199
|
+
catch (error) { return unavailable(objectFailure(error), first, route); }
|
|
173
200
|
},
|
|
174
201
|
};
|
|
175
202
|
|
|
@@ -180,9 +207,9 @@ const PUBLIC_NAMES = new Set(['Error', 'TypeError', 'RangeError', 'ReferenceErro
|
|
|
180
207
|
/** Failures before the HTTP handler (object initialization, migrations, storage, a reset) as our API contract, not a
|
|
181
208
|
* platform HTML page. The answer goes to callers not yet authenticated, so it names only the error's kind and the
|
|
182
209
|
* runtime's flags; its message (storage or SQL text, internal paths) is in the log line its request id leads to. */
|
|
183
|
-
function unavailable(failure, retried) {
|
|
210
|
+
function unavailable(failure, retried, route) {
|
|
184
211
|
const requestId = crypto.randomUUID();
|
|
185
|
-
console.error(JSON.stringify({ event: 'teams_unavailable', request_id: requestId, failure, ...(retried ? { first: retried } : {}) }));
|
|
212
|
+
console.error(JSON.stringify({ event: 'teams_unavailable', request_id: requestId, ...route, failure, ...(retried ? { first: retried } : {}) }));
|
|
186
213
|
const cause = `${PUBLIC_NAMES.has(failure.name) ? failure.name : 'Error'}${failure.reset ? ', object reset' : ''}${failure.overloaded ? ', overloaded' : ''}${retried ? ', after one retry' : ''}`;
|
|
187
|
-
return Response.json({ version: 1, error: { code: 'unavailable', message: `Teams server is temporarily unavailable (${cause})`, request_id: requestId, retryable: true, cause: { name: PUBLIC_NAMES.has(failure.name) ? failure.name : 'Error', retryable: failure.retryable, overloaded: failure.overloaded, remote: failure.remote, reset: failure.reset } } }, { status: 503, headers: { 'Retry-After': failure.overloaded ? '15' : '5', 'X-Request-ID': requestId } });
|
|
214
|
+
return Response.json({ version: 1, error: { code: 'unavailable', message: `Teams server is temporarily unavailable (${cause})`, request_id: requestId, ...route, retryable: true, cause: { name: PUBLIC_NAMES.has(failure.name) ? failure.name : 'Error', retryable: failure.retryable, overloaded: failure.overloaded, remote: failure.remote, reset: failure.reset } } }, { status: 503, headers: { 'Retry-After': failure.overloaded ? '15' : '5', 'X-Request-ID': requestId } });
|
|
188
215
|
}
|