@worca/ui 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app/main.bundle.js +6180 -4706
- package/app/main.bundle.js.map +4 -4
- package/app/protocol.js +2 -1
- package/app/styles.css +869 -15
- package/app/utils/state-actions.js +13 -2
- package/package.json +1 -1
- package/server/app.js +68 -1
- package/server/fleet-routes.js +1147 -0
- package/server/integrations/commands/fleet.js +266 -0
- package/server/integrations/commands/global.js +9 -0
- package/server/integrations/commands/parser.js +4 -1
- package/server/integrations/index.js +3 -0
- package/server/integrations/renderers.js +98 -0
- package/server/integrations/rest_client.js +7 -0
- package/server/worktrees-routes.js +34 -0
- package/server/ws-fleet-manifest-watcher.js +130 -0
- package/server/ws-message-router.js +20 -0
- package/server/ws-modular.js +10 -1
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fleet-scoped chat commands.
|
|
3
|
+
*
|
|
4
|
+
* /fleets list active fleets (running + paused)
|
|
5
|
+
* /fleet [id|last] show one fleet's status
|
|
6
|
+
* /fleet-children <id|last> per-child status table
|
|
7
|
+
* /fleet-halt <id> graceful halt (in-flight finish naturally)
|
|
8
|
+
* /fleet-stop <id> [--force] hard stop (SIGTERM + control file), with confirmation
|
|
9
|
+
* /fleet-pause <id> pause every in-flight child
|
|
10
|
+
* /fleet-resume <id> resume paused/interrupted children, re-dispatch failed
|
|
11
|
+
*
|
|
12
|
+
* Authz: every handler runs *after* the inbound allowlist gate in
|
|
13
|
+
* index.js — same gate that already guards /pause, /resume, /stop.
|
|
14
|
+
*
|
|
15
|
+
* Destructive actions (/fleet-stop) require a confirmation token written
|
|
16
|
+
* into chat_context with a 60s expiry. The token mechanism is local to
|
|
17
|
+
* this module so we don't bloat chat_context.js's public API for what is
|
|
18
|
+
* really a per-command UX concern.
|
|
19
|
+
*
|
|
20
|
+
* @module commands/fleet
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { statusEmoji } from './global.js';
|
|
24
|
+
|
|
25
|
+
const CONFIRM_TTL_MS = 60_000;
|
|
26
|
+
|
|
27
|
+
/** Pick the most recent fleet from a list (created_at desc). */
|
|
28
|
+
function pickLatest(fleets) {
|
|
29
|
+
if (!fleets || fleets.length === 0) return null;
|
|
30
|
+
return [...fleets].sort((a, b) => {
|
|
31
|
+
const at = a.created_at || '';
|
|
32
|
+
const bt = b.created_at || '';
|
|
33
|
+
return bt.localeCompare(at);
|
|
34
|
+
})[0];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async function fetchFleets(restClient) {
|
|
38
|
+
const resp = await restClient.get('/api/fleet-runs');
|
|
39
|
+
const data = resp.data;
|
|
40
|
+
if (!data || data.ok === false) return [];
|
|
41
|
+
return Array.isArray(data.fleets) ? data.fleets : [];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async function fetchFleetById(restClient, id) {
|
|
45
|
+
const resp = await restClient.get(
|
|
46
|
+
`/api/fleet-runs/${encodeURIComponent(id)}`,
|
|
47
|
+
);
|
|
48
|
+
const data = resp.data;
|
|
49
|
+
if (!data || data.ok === false) return null;
|
|
50
|
+
return data.fleet ?? null;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Resolve "last" / short-suffix / full-id into a fleet manifest object.
|
|
55
|
+
* Returns { fleet, disambig } — exactly one of them populated.
|
|
56
|
+
*/
|
|
57
|
+
async function resolveFleet(restClient, idArg, command) {
|
|
58
|
+
if (!idArg || idArg === 'last') {
|
|
59
|
+
const all = await fetchFleets(restClient);
|
|
60
|
+
const latest = pickLatest(all);
|
|
61
|
+
if (!latest) return { disambig: 'No fleets found.' };
|
|
62
|
+
return { fleet: latest };
|
|
63
|
+
}
|
|
64
|
+
if (idArg.startsWith('f_')) {
|
|
65
|
+
const fleet = await fetchFleetById(restClient, idArg);
|
|
66
|
+
if (!fleet) return { disambig: `Fleet \`${idArg}\` not found.` };
|
|
67
|
+
return { fleet };
|
|
68
|
+
}
|
|
69
|
+
// Short suffix match — `4318dbf9` matches `f_..._4318dbf9`.
|
|
70
|
+
const all = await fetchFleets(restClient);
|
|
71
|
+
const matches = all.filter((f) => f.fleet_id?.endsWith(idArg));
|
|
72
|
+
if (matches.length === 0) {
|
|
73
|
+
return { disambig: `No fleet matches \`${idArg}\`.` };
|
|
74
|
+
}
|
|
75
|
+
if (matches.length > 1) {
|
|
76
|
+
const lines = matches.map((f) => ` • \`${f.fleet_id}\` — ${f.status}`);
|
|
77
|
+
return {
|
|
78
|
+
disambig:
|
|
79
|
+
`Multiple fleets match \`${idArg}\`:\n${lines.join('\n')}\n\n` +
|
|
80
|
+
`Usage: /${command} <fleet_id>`,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
return { fleet: matches[0] };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function fmtFleetSummary(fleet) {
|
|
87
|
+
const id = fleet.fleet_id;
|
|
88
|
+
const title = fleet.work_request?.title || '(no title)';
|
|
89
|
+
const status = fleet.status || 'unknown';
|
|
90
|
+
const reason = fleet.halt_reason ? ` (${fleet.halt_reason})` : '';
|
|
91
|
+
const childCount = fleet.children_count ?? fleet.children?.length ?? 0;
|
|
92
|
+
const completed = (fleet.children || []).filter(
|
|
93
|
+
(c) => c.status === 'completed',
|
|
94
|
+
).length;
|
|
95
|
+
const failed = (fleet.children || []).filter(
|
|
96
|
+
(c) => c.status === 'failed' || c.status === 'setup_failed',
|
|
97
|
+
).length;
|
|
98
|
+
const parts = [`${statusEmoji(status)} **Fleet:** \`${id}\``];
|
|
99
|
+
parts.push(` **Title:** ${title}`);
|
|
100
|
+
parts.push(` **Status:** ${status}${reason}`);
|
|
101
|
+
parts.push(
|
|
102
|
+
` **Children:** ${completed}/${childCount} completed${failed ? `, ${failed} failed` : ''}`,
|
|
103
|
+
);
|
|
104
|
+
if (fleet.cost_usd != null) {
|
|
105
|
+
parts.push(` **Cost:** $${Number(fleet.cost_usd).toFixed(2)}`);
|
|
106
|
+
}
|
|
107
|
+
return parts.join('\n');
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function fmtChildrenTable(fleet) {
|
|
111
|
+
const children = fleet.children || [];
|
|
112
|
+
if (children.length === 0) return ' (no children dispatched yet)';
|
|
113
|
+
return children
|
|
114
|
+
.map((c) => {
|
|
115
|
+
const project =
|
|
116
|
+
(c.project_path || '').split('/').filter(Boolean).pop() || '(?)';
|
|
117
|
+
const st = c.status || 'unknown';
|
|
118
|
+
const rid = c.run_id ? ` \`${c.run_id}\`` : '';
|
|
119
|
+
return ` ${statusEmoji(st)} **${project}** — ${st}${rid}`;
|
|
120
|
+
})
|
|
121
|
+
.join('\n');
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Creates handlers for the /fleet… commands.
|
|
126
|
+
*
|
|
127
|
+
* The chatContext is used for /fleet-stop confirmation tokens — we store
|
|
128
|
+
* a `pending_fleet_stop = { fleet_id, expires_at }` shape on the chat key
|
|
129
|
+
* and clear it after the confirming message comes in.
|
|
130
|
+
*
|
|
131
|
+
* @param {{ chatContext, restClient }} deps
|
|
132
|
+
*/
|
|
133
|
+
export function createFleetHandlers({ chatContext, restClient }) {
|
|
134
|
+
async function fleets() {
|
|
135
|
+
const all = await fetchFleets(restClient);
|
|
136
|
+
const active = all.filter((f) => {
|
|
137
|
+
const s = f.status;
|
|
138
|
+
return s === 'running' || s === 'paused' || s === 'resuming';
|
|
139
|
+
});
|
|
140
|
+
if (active.length === 0) return 'No active fleets.';
|
|
141
|
+
const lines = active.map((f) => {
|
|
142
|
+
const childCount = f.children_count ?? f.children?.length ?? 0;
|
|
143
|
+
const title = f.work_request?.title || '(no title)';
|
|
144
|
+
const reason = f.halt_reason ? ` (${f.halt_reason})` : '';
|
|
145
|
+
return [
|
|
146
|
+
`${statusEmoji(f.status)} **Fleet:** \`${f.fleet_id}\``,
|
|
147
|
+
` **Title:** ${title}`,
|
|
148
|
+
` **Status:** ${f.status}${reason} | **Children:** ${childCount}`,
|
|
149
|
+
].join('\n');
|
|
150
|
+
});
|
|
151
|
+
return `Active fleets:\n\n${lines.join('\n')}`;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
async function fleet(_chatKey, args) {
|
|
155
|
+
const resolved = await resolveFleet(restClient, args[0], 'fleet');
|
|
156
|
+
if (resolved.disambig) return resolved.disambig;
|
|
157
|
+
return fmtFleetSummary(resolved.fleet);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
async function fleetChildren(_chatKey, args) {
|
|
161
|
+
const resolved = await resolveFleet(restClient, args[0], 'fleet-children');
|
|
162
|
+
if (resolved.disambig) return resolved.disambig;
|
|
163
|
+
const f = resolved.fleet;
|
|
164
|
+
return `Children of \`${f.fleet_id}\`:\n\n` + fmtChildrenTable(f);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
async function fleetHalt(_chatKey, args) {
|
|
168
|
+
const resolved = await resolveFleet(restClient, args[0], 'fleet-halt');
|
|
169
|
+
if (resolved.disambig) return resolved.disambig;
|
|
170
|
+
const id = resolved.fleet.fleet_id;
|
|
171
|
+
const resp = await restClient.delete(
|
|
172
|
+
`/api/fleet-runs/${encodeURIComponent(id)}`,
|
|
173
|
+
);
|
|
174
|
+
if (!resp.data || resp.data.ok === false) {
|
|
175
|
+
return `Failed to halt fleet \`${id}\` (${resp.status}).`;
|
|
176
|
+
}
|
|
177
|
+
return `\u{1F7E1} Halted fleet \`${id}\`.\nIn-flight children will finish naturally.`;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async function fleetStop(chatKey, args) {
|
|
181
|
+
const isForce = args.includes('--force') || args.includes('YES');
|
|
182
|
+
const cleanArgs = args.filter((a) => a !== '--force' && a !== 'YES');
|
|
183
|
+
const resolved = await resolveFleet(restClient, cleanArgs[0], 'fleet-stop');
|
|
184
|
+
if (resolved.disambig) return resolved.disambig;
|
|
185
|
+
const id = resolved.fleet.fleet_id;
|
|
186
|
+
|
|
187
|
+
const ctx = chatContext.get(chatKey) || {};
|
|
188
|
+
const pending = ctx.pending_fleet_stop;
|
|
189
|
+
|
|
190
|
+
if (!isForce) {
|
|
191
|
+
// Issue / refresh a confirmation token.
|
|
192
|
+
chatContext.set(chatKey, {
|
|
193
|
+
pending_fleet_stop: {
|
|
194
|
+
fleet_id: id,
|
|
195
|
+
expires_at: new Date(Date.now() + CONFIRM_TTL_MS).toISOString(),
|
|
196
|
+
},
|
|
197
|
+
});
|
|
198
|
+
const child_count =
|
|
199
|
+
resolved.fleet.children_count ?? (resolved.fleet.children || []).length;
|
|
200
|
+
return (
|
|
201
|
+
`⚠ \`/fleet-stop\` will SIGTERM every in-flight child of fleet \`${id}\` (${child_count} children).\n` +
|
|
202
|
+
`Confirm with \`/fleet-stop ${id} YES\` within 60s, or pass \`--force\`.`
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// --force or YES path. Either matches a fresh confirmation token, or
|
|
207
|
+
// the caller is bypassing the gate entirely with --force.
|
|
208
|
+
if (
|
|
209
|
+
args.includes('YES') &&
|
|
210
|
+
(!pending ||
|
|
211
|
+
pending.fleet_id !== id ||
|
|
212
|
+
new Date(pending.expires_at).getTime() < Date.now())
|
|
213
|
+
) {
|
|
214
|
+
return `Confirmation expired or never issued for fleet \`${id}\`. Re-run \`/fleet-stop ${id}\` to get a fresh token.`;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
chatContext.set(chatKey, { pending_fleet_stop: null });
|
|
218
|
+
const resp = await restClient.post(
|
|
219
|
+
`/api/fleet-runs/${encodeURIComponent(id)}/stop`,
|
|
220
|
+
);
|
|
221
|
+
if (!resp.data || resp.data.ok === false) {
|
|
222
|
+
return `Failed to stop fleet \`${id}\` (${resp.status}).`;
|
|
223
|
+
}
|
|
224
|
+
const count = resp.data.stopped_count ?? '?';
|
|
225
|
+
return `\u{1F534} Stopped fleet \`${id}\`. SIGTERM sent to ${count} child(ren).`;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
async function fleetPause(_chatKey, args) {
|
|
229
|
+
const resolved = await resolveFleet(restClient, args[0], 'fleet-pause');
|
|
230
|
+
if (resolved.disambig) return resolved.disambig;
|
|
231
|
+
const id = resolved.fleet.fleet_id;
|
|
232
|
+
const resp = await restClient.post(
|
|
233
|
+
`/api/fleet-runs/${encodeURIComponent(id)}/pause`,
|
|
234
|
+
);
|
|
235
|
+
if (!resp.data || resp.data.ok === false) {
|
|
236
|
+
return `Failed to pause fleet \`${id}\` (${resp.status}).`;
|
|
237
|
+
}
|
|
238
|
+
const count = resp.data.paused_count ?? '?';
|
|
239
|
+
return `\u{1F7E1} Paused fleet \`${id}\`. ${count} child(ren) will pause at their next iteration boundary.`;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
async function fleetResume(_chatKey, args) {
|
|
243
|
+
const resolved = await resolveFleet(restClient, args[0], 'fleet-resume');
|
|
244
|
+
if (resolved.disambig) return resolved.disambig;
|
|
245
|
+
const id = resolved.fleet.fleet_id;
|
|
246
|
+
const resp = await restClient.post(
|
|
247
|
+
`/api/fleet-runs/${encodeURIComponent(id)}/resume`,
|
|
248
|
+
);
|
|
249
|
+
if (!resp.data || resp.data.ok === false) {
|
|
250
|
+
return `Failed to resume fleet \`${id}\` (${resp.status}).`;
|
|
251
|
+
}
|
|
252
|
+
const continued = resp.data.continued_count ?? 0;
|
|
253
|
+
const redispatched = resp.data.redispatched_count ?? 0;
|
|
254
|
+
return `\u{1F7E2} Resumed fleet \`${id}\`. ${continued} continued in-place, ${redispatched} re-dispatched.`;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
return {
|
|
258
|
+
fleets,
|
|
259
|
+
fleet,
|
|
260
|
+
'fleet-children': fleetChildren,
|
|
261
|
+
'fleet-halt': fleetHalt,
|
|
262
|
+
'fleet-stop': fleetStop,
|
|
263
|
+
'fleet-pause': fleetPause,
|
|
264
|
+
'fleet-resume': fleetResume,
|
|
265
|
+
};
|
|
266
|
+
}
|
|
@@ -115,6 +115,15 @@ const HELP_TEXT = `/start \u2014 show your chat ID
|
|
|
115
115
|
/resume [run_id] \u2014 resume paused run
|
|
116
116
|
/stop [run_id] \u2014 stop run
|
|
117
117
|
|
|
118
|
+
Fleet commands (cross-project):
|
|
119
|
+
/fleets \u2014 list active fleets
|
|
120
|
+
/fleet [id|last] \u2014 fleet status
|
|
121
|
+
/fleet-children <id|last> \u2014 per-child status
|
|
122
|
+
/fleet-halt <id> \u2014 graceful halt (in-flight finish naturally)
|
|
123
|
+
/fleet-stop <id> [--force] \u2014 SIGTERM every in-flight child (confirms first)
|
|
124
|
+
/fleet-pause <id> \u2014 pause every in-flight child
|
|
125
|
+
/fleet-resume <id> \u2014 resume paused/interrupted, re-dispatch failed
|
|
126
|
+
|
|
118
127
|
Commands with [run_id] auto-resolve to the active run if omitted.
|
|
119
128
|
Use \`*suffix\` to match by ending, e.g. /status \`*2db5\`
|
|
120
129
|
Project commands require /use first.`;
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
const MENTION_RE = /^@\S+$/i;
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
// Allow `-` so namespaced commands like /fleet-halt and /fleet-resume parse.
|
|
4
|
+
// Hyphens must appear inside the name, not lead or trail. Backwards-compatible
|
|
5
|
+
// — every existing underscore-only command still matches.
|
|
6
|
+
const COMMAND_RE = /^\/([a-z_][a-z0-9_-]*)(?:@\S+)?$/i;
|
|
4
7
|
|
|
5
8
|
/**
|
|
6
9
|
* Parses a chat message into a command name and argument list.
|
|
@@ -12,6 +12,7 @@ import { createWebhookOutAdapter } from './adapters/webhook_out.js';
|
|
|
12
12
|
import { createAllowlistGuard } from './allowlist.js';
|
|
13
13
|
import { createChatContext } from './chat_context.js';
|
|
14
14
|
import { createControlHandlers } from './commands/control.js';
|
|
15
|
+
import { createFleetHandlers } from './commands/fleet.js';
|
|
15
16
|
import { createGlobalHandlers } from './commands/global.js';
|
|
16
17
|
import { parseCommand } from './commands/parser.js';
|
|
17
18
|
import { createProjectHandlers } from './commands/project.js';
|
|
@@ -65,10 +66,12 @@ export function createIntegrations({
|
|
|
65
66
|
});
|
|
66
67
|
const projectHandlers = createProjectHandlers({ chatContext, restClient });
|
|
67
68
|
const controlHandlers = createControlHandlers({ chatContext, restClient });
|
|
69
|
+
const fleetHandlers = createFleetHandlers({ chatContext, restClient });
|
|
68
70
|
const allHandlers = {
|
|
69
71
|
...globalHandlers,
|
|
70
72
|
...projectHandlers,
|
|
71
73
|
...controlHandlers,
|
|
74
|
+
...fleetHandlers,
|
|
72
75
|
};
|
|
73
76
|
|
|
74
77
|
// Mutable adapter registry — keyed by adapter name
|
|
@@ -154,6 +154,90 @@ function renderCostBudgetWarning(envelope) {
|
|
|
154
154
|
return mdMsg(parts.join('\n'), 'warning');
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
+
// ---------------------------------------------------------------------------
|
|
158
|
+
// Fleet event renderers
|
|
159
|
+
// ---------------------------------------------------------------------------
|
|
160
|
+
// Mirror the run-event renderers' shape: short title line + indented meta
|
|
161
|
+
// rows. fleet_id replaces run_id as the primary key; envelopes are
|
|
162
|
+
// fleet-shaped (top-level fleet_id, no `pipeline` wrapper). See
|
|
163
|
+
// src/worca/events/fleet_emitter.py for the envelope schema.
|
|
164
|
+
|
|
165
|
+
function fleetId(envelope) {
|
|
166
|
+
return envelope.fleet_id ?? 'fleet';
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function projectBasename(p) {
|
|
170
|
+
if (!p) return '';
|
|
171
|
+
const parts = p.split('/').filter(Boolean);
|
|
172
|
+
return parts[parts.length - 1] || p;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function renderFleetLaunched(envelope) {
|
|
176
|
+
const p = envelope.payload ?? {};
|
|
177
|
+
const projects = Array.isArray(p.projects) ? p.projects : [];
|
|
178
|
+
const projectsLabel = projects.length
|
|
179
|
+
? projects.slice(0, 5).map(projectBasename).join(', ') +
|
|
180
|
+
(projects.length > 5 ? `, +${projects.length - 5} more` : '')
|
|
181
|
+
: '(none)';
|
|
182
|
+
const parts = [`\u{1F680} **Fleet launched:** \`${fleetId(envelope)}\``];
|
|
183
|
+
parts.push(` **Projects:** ${projects.length} — ${projectsLabel}`);
|
|
184
|
+
if (p.plan_mode && p.plan_mode !== 'none') {
|
|
185
|
+
parts.push(` **Plan mode:** ${p.plan_mode}`);
|
|
186
|
+
}
|
|
187
|
+
if (p.guide_attached) parts.push(' **Guide:** attached');
|
|
188
|
+
if (p.base_branch) parts.push(` **Base:** ${p.base_branch}`);
|
|
189
|
+
return mdMsg(parts.join('\n'), 'info');
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function renderFleetHalted(envelope) {
|
|
193
|
+
const p = envelope.payload ?? {};
|
|
194
|
+
const reason = p.halt_reason || 'unknown';
|
|
195
|
+
// Severity matches the reason: circuit_breaker is an error, user/stopped is
|
|
196
|
+
// a warning. Keeps Slack/Discord colour coding consistent with the per-run
|
|
197
|
+
// pipeline.run.interrupted vs pipeline.circuit_breaker.tripped split.
|
|
198
|
+
const sev = reason === 'circuit_breaker' ? 'error' : 'warning';
|
|
199
|
+
const parts = [`\u{1F6D1} **Fleet halted:** \`${fleetId(envelope)}\``];
|
|
200
|
+
parts.push(` **Reason:** ${reason}`);
|
|
201
|
+
if (p.in_flight_count != null) {
|
|
202
|
+
parts.push(` **In-flight at halt:** ${p.in_flight_count}`);
|
|
203
|
+
}
|
|
204
|
+
if (p.pending_count != null && p.pending_count > 0) {
|
|
205
|
+
parts.push(` **Pending (not dispatched):** ${p.pending_count}`);
|
|
206
|
+
}
|
|
207
|
+
return mdMsg(parts.join('\n'), sev);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function renderFleetCompleted(envelope) {
|
|
211
|
+
const p = envelope.payload ?? {};
|
|
212
|
+
const parts = [`✅ **Fleet completed:** \`${fleetId(envelope)}\``];
|
|
213
|
+
if (p.child_count != null) {
|
|
214
|
+
parts.push(
|
|
215
|
+
` **Children:** ${p.completed_count ?? p.child_count}/${p.child_count} completed`,
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
if (p.duration_ms != null) {
|
|
219
|
+
parts.push(` **Duration:** ${fmtMs(p.duration_ms)}`);
|
|
220
|
+
}
|
|
221
|
+
return mdMsg(parts.join('\n'), 'success');
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function renderFleetFailed(envelope) {
|
|
225
|
+
const p = envelope.payload ?? {};
|
|
226
|
+
const parts = [`❌ **Fleet failed:** \`${fleetId(envelope)}\``];
|
|
227
|
+
if (p.child_count != null) {
|
|
228
|
+
const failed = p.failed_count ?? 0;
|
|
229
|
+
const interrupted = p.interrupted_count ?? 0;
|
|
230
|
+
const completed = p.completed_count ?? 0;
|
|
231
|
+
parts.push(
|
|
232
|
+
` **Children:** ${completed}/${p.child_count} completed, ${failed} failed, ${interrupted} interrupted`,
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
if (p.duration_ms != null) {
|
|
236
|
+
parts.push(` **Duration:** ${fmtMs(p.duration_ms)}`);
|
|
237
|
+
}
|
|
238
|
+
return mdMsg(parts.join('\n'), 'error');
|
|
239
|
+
}
|
|
240
|
+
|
|
157
241
|
// ---------------------------------------------------------------------------
|
|
158
242
|
// Registry
|
|
159
243
|
// ---------------------------------------------------------------------------
|
|
@@ -173,6 +257,20 @@ const EVENT_RENDERERS = {
|
|
|
173
257
|
'pipeline.git.pr_merged': renderGitPrMerged,
|
|
174
258
|
'pipeline.circuit_breaker.tripped': renderCbTripped,
|
|
175
259
|
'pipeline.cost.budget_warning': renderCostBudgetWarning,
|
|
260
|
+
// fleet.launched is intentionally NOT in this map by default — projects
|
|
261
|
+
// that launch many fleets per day would find it noisy. Opt-in callers
|
|
262
|
+
// can register it themselves via renderEvent's renderer override (or
|
|
263
|
+
// by extending TIER1_EVENTS in a future per-project config).
|
|
264
|
+
'fleet.halted': renderFleetHalted,
|
|
265
|
+
'fleet.completed': renderFleetCompleted,
|
|
266
|
+
'fleet.failed': renderFleetFailed,
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
// fleet.launched ships as an opt-in renderer rather than a Tier-1 default —
|
|
270
|
+
// see comment above. Callers that want it can pull it from this export and
|
|
271
|
+
// register it in their own pipeline.
|
|
272
|
+
export const OPT_IN_RENDERERS = {
|
|
273
|
+
'fleet.launched': renderFleetLaunched,
|
|
176
274
|
};
|
|
177
275
|
|
|
178
276
|
export const TIER1_EVENTS = Object.keys(EVENT_RENDERERS);
|
|
@@ -13,5 +13,12 @@ export function createRestClient({ host, port }) {
|
|
|
13
13
|
});
|
|
14
14
|
return { status: r.status, data: r.ok ? await r.json() : null };
|
|
15
15
|
},
|
|
16
|
+
// DELETE used by /fleet-halt (DELETE /api/fleet-runs/:id) — added when
|
|
17
|
+
// chat fleet commands landed. Mirrors the get/post pair: no body, returns
|
|
18
|
+
// parsed JSON on 2xx, null on error status.
|
|
19
|
+
async delete(path) {
|
|
20
|
+
const r = await fetch(`${base}${path}`, { method: 'DELETE' });
|
|
21
|
+
return { status: r.status, data: r.ok ? await r.json() : null };
|
|
22
|
+
},
|
|
16
23
|
};
|
|
17
24
|
}
|
|
@@ -262,6 +262,17 @@ function _patchRegistry(worcaDir, runId, patch) {
|
|
|
262
262
|
}
|
|
263
263
|
}
|
|
264
264
|
|
|
265
|
+
function _isPidAlive(pid) {
|
|
266
|
+
if (!pid || typeof pid !== 'number') return false;
|
|
267
|
+
try {
|
|
268
|
+
process.kill(pid, 0);
|
|
269
|
+
return true;
|
|
270
|
+
} catch (err) {
|
|
271
|
+
if (err.code === 'EPERM') return true;
|
|
272
|
+
return false;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
265
276
|
async function _listWorktrees(worcaDir) {
|
|
266
277
|
const pipelinesDir = join(worcaDir, 'multi', 'pipelines.d');
|
|
267
278
|
if (!existsSync(pipelinesDir)) return [];
|
|
@@ -290,6 +301,29 @@ async function _listWorktrees(worcaDir) {
|
|
|
290
301
|
if (actual) status = actual;
|
|
291
302
|
}
|
|
292
303
|
|
|
304
|
+
// Stale-registry reconciliation: a child can die before ever writing
|
|
305
|
+
// status.json (e.g. fleet halt right after dispatch, preflight crash,
|
|
306
|
+
// SIGKILL). In that case the worktree exists but .worca/runs/ doesn't,
|
|
307
|
+
// _readWorktreeStatus returns null, and we'd fall back to reg.status
|
|
308
|
+
// which may still say "running" with a dead pid. Treat that as
|
|
309
|
+
// "interrupted" and patch the registry so this only happens once.
|
|
310
|
+
//
|
|
311
|
+
// Only reconcile when reg.pid is present — a missing pid means the
|
|
312
|
+
// entry is either from a non-standard registration path (e.g. test
|
|
313
|
+
// fixtures) or pre-dates the pid-on-registration contract, so we
|
|
314
|
+
// can't make liveness claims about it.
|
|
315
|
+
if (
|
|
316
|
+
status === 'running' &&
|
|
317
|
+
typeof reg.pid === 'number' &&
|
|
318
|
+
!_isPidAlive(reg.pid)
|
|
319
|
+
) {
|
|
320
|
+
status = 'interrupted';
|
|
321
|
+
_patchRegistry(worcaDir, reg.run_id, {
|
|
322
|
+
status: 'interrupted',
|
|
323
|
+
interrupted_reason: 'stale_pid',
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
|
|
293
327
|
let ageSeconds = 0;
|
|
294
328
|
if (reg.started_at) {
|
|
295
329
|
const started = new Date(reg.started_at).getTime();
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fleet manifest watcher — monitors ~/.worca/fleet-runs/<fleet_id>.json for changes.
|
|
3
|
+
* Emits fleet-update WS events when a fleet manifest is written (§13.5).
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { existsSync, readFileSync, watch } from 'node:fs';
|
|
7
|
+
import { homedir } from 'node:os';
|
|
8
|
+
import { join } from 'node:path';
|
|
9
|
+
import { effectiveFleetStatus } from './fleet-routes.js';
|
|
10
|
+
|
|
11
|
+
const FLEET_DEBOUNCE_MS = 200;
|
|
12
|
+
const DEFAULT_FLEET_RUNS_DIR = join(homedir(), '.worca', 'fleet-runs');
|
|
13
|
+
|
|
14
|
+
const FAILURE_STATES = new Set(['failed', 'setup_failed', 'unrecoverable']);
|
|
15
|
+
|
|
16
|
+
function readJson(path) {
|
|
17
|
+
try {
|
|
18
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
19
|
+
} catch {
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function resolveChildStatus(child) {
|
|
25
|
+
const { project_path, run_id } = child;
|
|
26
|
+
if (!project_path || !run_id) return 'running';
|
|
27
|
+
const registryPath = join(
|
|
28
|
+
project_path,
|
|
29
|
+
'.worca',
|
|
30
|
+
'multi',
|
|
31
|
+
'pipelines.d',
|
|
32
|
+
`${run_id}.json`,
|
|
33
|
+
);
|
|
34
|
+
const entry = readJson(registryPath);
|
|
35
|
+
return entry?.status ?? 'running';
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* @param {{ broadcaster: { broadcast: Function }, fleetRunsDir?: string }} deps
|
|
40
|
+
*/
|
|
41
|
+
export function createFleetManifestWatcher({
|
|
42
|
+
broadcaster,
|
|
43
|
+
fleetRunsDir = DEFAULT_FLEET_RUNS_DIR,
|
|
44
|
+
}) {
|
|
45
|
+
let fsWatcher = null;
|
|
46
|
+
/** @type {Map<string, ReturnType<typeof setTimeout>>} */
|
|
47
|
+
const debounceTimers = new Map();
|
|
48
|
+
|
|
49
|
+
function broadcastFleetUpdate(fleetId, manifestPath) {
|
|
50
|
+
const manifest = readJson(manifestPath);
|
|
51
|
+
if (!manifest) return;
|
|
52
|
+
|
|
53
|
+
const rawChildren = Array.isArray(manifest.children)
|
|
54
|
+
? manifest.children
|
|
55
|
+
: [];
|
|
56
|
+
const children = rawChildren.map((child) => ({
|
|
57
|
+
run_id: child.run_id,
|
|
58
|
+
project_path: child.project_path,
|
|
59
|
+
status: resolveChildStatus(child),
|
|
60
|
+
}));
|
|
61
|
+
|
|
62
|
+
const completed_children = children.filter(
|
|
63
|
+
(c) => c.status === 'completed',
|
|
64
|
+
).length;
|
|
65
|
+
const failed_children = children.filter((c) =>
|
|
66
|
+
FAILURE_STATES.has(c.status),
|
|
67
|
+
).length;
|
|
68
|
+
|
|
69
|
+
// Derive the effective status (same rules as REST) instead of broadcasting
|
|
70
|
+
// raw manifest.status — otherwise cards stay "running" forever, because
|
|
71
|
+
// run_fleet.py never writes a terminal status after it exits. Pure
|
|
72
|
+
// function: persists nothing, so we don't trigger a watch→write→watch loop.
|
|
73
|
+
const { status, halt_reason } = effectiveFleetStatus(
|
|
74
|
+
manifest,
|
|
75
|
+
children.map((c) => c.status),
|
|
76
|
+
);
|
|
77
|
+
|
|
78
|
+
broadcaster.broadcast('fleet-update', {
|
|
79
|
+
fleet_id: fleetId,
|
|
80
|
+
status,
|
|
81
|
+
halt_reason,
|
|
82
|
+
completed_children,
|
|
83
|
+
failed_children,
|
|
84
|
+
children,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function scheduleUpdate(fleetId, manifestPath) {
|
|
89
|
+
const existing = debounceTimers.get(fleetId);
|
|
90
|
+
if (existing) clearTimeout(existing);
|
|
91
|
+
debounceTimers.set(
|
|
92
|
+
fleetId,
|
|
93
|
+
setTimeout(() => {
|
|
94
|
+
debounceTimers.delete(fleetId);
|
|
95
|
+
broadcastFleetUpdate(fleetId, manifestPath);
|
|
96
|
+
}, FLEET_DEBOUNCE_MS),
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
try {
|
|
101
|
+
if (existsSync(fleetRunsDir)) {
|
|
102
|
+
fsWatcher = watch(
|
|
103
|
+
fleetRunsDir,
|
|
104
|
+
{ persistent: false },
|
|
105
|
+
(_event, filename) => {
|
|
106
|
+
if (!filename?.endsWith('.json')) return;
|
|
107
|
+
const fleetId = filename.slice(0, -5);
|
|
108
|
+
scheduleUpdate(fleetId, join(fleetRunsDir, filename));
|
|
109
|
+
},
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
} catch {
|
|
113
|
+
// fs.watch unsupported or dir unavailable — skip silently
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function destroy() {
|
|
117
|
+
if (fsWatcher) {
|
|
118
|
+
try {
|
|
119
|
+
fsWatcher.close();
|
|
120
|
+
} catch {
|
|
121
|
+
/* ignore */
|
|
122
|
+
}
|
|
123
|
+
fsWatcher = null;
|
|
124
|
+
}
|
|
125
|
+
for (const timer of debounceTimers.values()) clearTimeout(timer);
|
|
126
|
+
debounceTimers.clear();
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return { destroy };
|
|
130
|
+
}
|
|
@@ -80,6 +80,22 @@ export function createMessageRouter({
|
|
|
80
80
|
};
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
+
// When a run-scoped subscribe arrives with a `payload.projectId` that
|
|
84
|
+
// differs from the client's currently-bound subs.projectId, re-bind the
|
|
85
|
+
// WS to that project. Without this, the targeted project's WatcherSet
|
|
86
|
+
// stays in POLLING tier (no logWatcher / no live updates), and the
|
|
87
|
+
// backfill / live stream silently never arrive — symptoms reported by
|
|
88
|
+
// the user as "no logs / no agent prompts" on the run-detail page in
|
|
89
|
+
// global mode. We keep the previous projectId reference so demotion
|
|
90
|
+
// still happens via the normal client-count mechanism.
|
|
91
|
+
function _adoptProjectFromPayload(ws, payload) {
|
|
92
|
+
const requested = payload?.projectId;
|
|
93
|
+
if (!requested || !watcherSets.has(requested)) return;
|
|
94
|
+
const subs = clientManager.getSubs(ws);
|
|
95
|
+
if (subs?.projectId === requested) return;
|
|
96
|
+
clientManager.setProtocol(ws, subs?.protocolVersion ?? 1, requested);
|
|
97
|
+
}
|
|
98
|
+
|
|
83
99
|
async function handleMessage(ws, data) {
|
|
84
100
|
let json;
|
|
85
101
|
try {
|
|
@@ -147,6 +163,7 @@ export function createMessageRouter({
|
|
|
147
163
|
);
|
|
148
164
|
return;
|
|
149
165
|
}
|
|
166
|
+
_adoptProjectFromPayload(ws, req.payload);
|
|
150
167
|
const proj = resolveProject(ws, req.payload);
|
|
151
168
|
const runs = discoverRuns(proj.worcaDir);
|
|
152
169
|
const run = runs.find((r) => r.id === runId);
|
|
@@ -294,6 +311,7 @@ export function createMessageRouter({
|
|
|
294
311
|
);
|
|
295
312
|
return;
|
|
296
313
|
}
|
|
314
|
+
_adoptProjectFromPayload(ws, req.payload);
|
|
297
315
|
const proj = resolveProject(ws, req.payload);
|
|
298
316
|
if (!proj) {
|
|
299
317
|
ws.send(
|
|
@@ -336,6 +354,7 @@ export function createMessageRouter({
|
|
|
336
354
|
// subscribe-log
|
|
337
355
|
if (req.type === 'subscribe-log') {
|
|
338
356
|
const { stage, runId, iteration } = req.payload || {};
|
|
357
|
+
_adoptProjectFromPayload(ws, req.payload);
|
|
339
358
|
const proj = resolveProject(ws, req.payload);
|
|
340
359
|
const s = clientManager.ensureSubs(ws);
|
|
341
360
|
s.logStage = stage || '*';
|
|
@@ -652,6 +671,7 @@ export function createMessageRouter({
|
|
|
652
671
|
);
|
|
653
672
|
return;
|
|
654
673
|
}
|
|
674
|
+
_adoptProjectFromPayload(ws, req.payload);
|
|
655
675
|
const proj = resolveProject(ws, req.payload);
|
|
656
676
|
if (!proj.wset.beadsWatcher) {
|
|
657
677
|
ws.send(JSON.stringify(makeOk(req, { issues: [], runId })));
|