@bakers-build/conductor 0.0.0-stage → 0.1.0-next.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +4 -2
- package/dist/src/bakers-project-context.d.ts +12 -0
- package/dist/src/bakers-project-context.js +59 -0
- package/dist/src/cli/config-plan.d.ts +35 -0
- package/dist/src/cli/config-plan.js +1264 -0
- package/dist/src/cli/config-schema.d.ts +137 -0
- package/dist/src/cli/config-schema.js +239 -0
- package/dist/src/cli/config-source-edit.d.ts +17 -0
- package/dist/src/cli/config-source-edit.js +95 -0
- package/dist/src/cli/config-templates.d.ts +43 -0
- package/dist/src/cli/config-templates.js +130 -0
- package/dist/src/cli/config-validation.d.ts +12 -0
- package/dist/src/cli/config-validation.js +353 -0
- package/dist/src/cli/migrate-addon-names.d.ts +82 -0
- package/dist/src/cli/migrate-addon-names.js +334 -0
- package/dist/src/cli/project-context.d.ts +21 -0
- package/dist/src/cli/project-context.js +105 -0
- package/dist/src/commands/conductor.d.ts +91 -0
- package/dist/src/commands/conductor.js +843 -0
- package/dist/src/config.d.ts +109 -0
- package/dist/src/config.js +712 -0
- package/dist/src/contract.d.ts +13 -0
- package/dist/src/contract.js +19 -0
- package/dist/src/core/action-queue.d.ts +31 -0
- package/dist/src/core/action-queue.js +69 -0
- package/dist/src/core/attempt.d.ts +25 -0
- package/dist/src/core/attempt.js +41 -0
- package/dist/src/core/conductor.d.ts +236 -0
- package/dist/src/core/conductor.js +1293 -0
- package/dist/src/core/host-key.d.ts +19 -0
- package/dist/src/core/host-key.js +27 -0
- package/dist/src/core/inbox-runtime.d.ts +48 -0
- package/dist/src/core/inbox-runtime.js +180 -0
- package/dist/src/core/mr-changes.d.ts +18 -0
- package/dist/src/core/mr-changes.js +47 -0
- package/dist/src/index.d.ts +8 -0
- package/dist/src/index.js +27 -0
- package/dist/src/plugins/agent-args.d.ts +43 -0
- package/dist/src/plugins/agent-args.js +221 -0
- package/dist/src/plugins/agent.d.ts +121 -0
- package/dist/src/plugins/agent.js +12 -0
- package/dist/src/plugins/executor.d.ts +182 -0
- package/dist/src/plugins/executor.js +1 -0
- package/dist/src/plugins/message.d.ts +61 -0
- package/dist/src/plugins/message.js +47 -0
- package/dist/src/plugins/plugins.d.ts +96 -0
- package/dist/src/plugins/plugins.js +120 -0
- package/dist/src/plugins/preset.d.ts +61 -0
- package/dist/src/plugins/preset.js +23 -0
- package/dist/src/plugins/process-host.d.ts +7 -0
- package/dist/src/plugins/process-host.js +1 -0
- package/dist/src/plugins/remote.d.ts +420 -0
- package/dist/src/plugins/remote.js +1 -0
- package/dist/src/plugins/workspace.d.ts +69 -0
- package/dist/src/plugins/workspace.js +1 -0
- package/dist/src/preset.d.ts +2 -0
- package/dist/src/preset.js +8 -0
- package/dist/src/types.d.ts +132 -0
- package/dist/src/types.js +6 -0
- package/package.json +38 -4
|
@@ -0,0 +1,1293 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { basename, dirname, join } from 'node:path';
|
|
3
|
+
import { stemForConfigFile } from '@bakers-build/core';
|
|
4
|
+
import { resolveAgent } from '../plugins/plugins.js';
|
|
5
|
+
import { ActionQueue } from './action-queue.js';
|
|
6
|
+
import { attemptFamily } from './attempt.js';
|
|
7
|
+
import { hostKeyOf } from './host-key.js';
|
|
8
|
+
import { InboxRuntime, runInbox } from './inbox-runtime.js';
|
|
9
|
+
import { measureMrChanges, mrChangesRows } from './mr-changes.js';
|
|
10
|
+
function sleep(ms, signal) {
|
|
11
|
+
return new Promise((resolve) => {
|
|
12
|
+
if (signal?.aborted) {
|
|
13
|
+
resolve();
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
const timer = setTimeout(() => {
|
|
17
|
+
signal?.removeEventListener('abort', onAbort);
|
|
18
|
+
resolve();
|
|
19
|
+
}, ms);
|
|
20
|
+
const onAbort = () => {
|
|
21
|
+
clearTimeout(timer);
|
|
22
|
+
resolve();
|
|
23
|
+
};
|
|
24
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* A run-env key is reserved when it names a conductor-provided core variable:
|
|
29
|
+
* any `BB_*`, `WORKSPACE_ROOT`, or `TICKET_URL`. User `env_vars` may not
|
|
30
|
+
* clobber these — the core vars always win (BB-99).
|
|
31
|
+
*/
|
|
32
|
+
function isReservedEnvKey(key) {
|
|
33
|
+
return (key.startsWith('BB_') || key === 'WORKSPACE_ROOT' || key === 'TICKET_URL');
|
|
34
|
+
}
|
|
35
|
+
/** A portable environment variable name. */
|
|
36
|
+
const ENV_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
37
|
+
/**
|
|
38
|
+
* What a failed listener is logged as: the error's class name, never its
|
|
39
|
+
* message — listener text may carry an env value (BB-599).
|
|
40
|
+
*/
|
|
41
|
+
function errorKind(err) {
|
|
42
|
+
return err instanceof Error ? err.name : typeof err;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Parse the server-canonicalised effective env_vars (`KEY=value` lines, one per
|
|
46
|
+
* line) into a map. The server already stripped comments/blank/malformed lines,
|
|
47
|
+
* so this is deliberately minimal: split on the first `=`, keep a valid key.
|
|
48
|
+
* Defensive against stray blank lines only — never throws.
|
|
49
|
+
*/
|
|
50
|
+
function parseEnvLines(text) {
|
|
51
|
+
const out = {};
|
|
52
|
+
for (const raw of text.split('\n')) {
|
|
53
|
+
const line = raw.trim();
|
|
54
|
+
if (line === '') {
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
const eq = line.indexOf('=');
|
|
58
|
+
if (eq <= 0) {
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
const key = line.slice(0, eq);
|
|
62
|
+
if (!ENV_KEY.test(key)) {
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
out[key] = line.slice(eq + 1);
|
|
66
|
+
}
|
|
67
|
+
return out;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Resolve the environment a run executes in (BB-99): parse the ticket's
|
|
71
|
+
* effective env_vars, drop any reserved key (loud warn — key NAME only, never
|
|
72
|
+
* the value), then overlay the conductor's core vars so they always win.
|
|
73
|
+
*
|
|
74
|
+
* Values are potentially secret (e.g. a DB DSN): this function logs key names
|
|
75
|
+
* only, never a value. The returned map is injected into BOTH the agent run env
|
|
76
|
+
* and every workspace hook.
|
|
77
|
+
*/
|
|
78
|
+
export function resolveRunEnv(effectiveEnvVars, core, logger) {
|
|
79
|
+
const user = effectiveEnvVars ? parseEnvLines(effectiveEnvVars) : {};
|
|
80
|
+
const clobbered = [];
|
|
81
|
+
const safe = {};
|
|
82
|
+
for (const [key, value] of Object.entries(user)) {
|
|
83
|
+
if (isReservedEnvKey(key)) {
|
|
84
|
+
clobbered.push(key);
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
safe[key] = value;
|
|
88
|
+
}
|
|
89
|
+
if (clobbered.length > 0) {
|
|
90
|
+
logger.warn({ keys: clobbered }, 'ignored reserved keys in ticket env_vars (core vars win)');
|
|
91
|
+
}
|
|
92
|
+
// Core vars overlay last — they can never be overridden.
|
|
93
|
+
return { ...safe, ...core };
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Where a conductor keeps its own record of which worktrees have a
|
|
97
|
+
* successfully completed `after_create` (BB-663), beside its config file —
|
|
98
|
+
* same pattern as `frozenStateFile` — rather than inside the worktree. An
|
|
99
|
+
* in-tree marker gets picked up by a stray `git add -A`/commit (review F1):
|
|
100
|
+
* once merged, every checkout of that commit then LOOKS bootstrapped and
|
|
101
|
+
* silently skips `after_create` forever. Keyed by worktree path, not by
|
|
102
|
+
* conductor instance, so it survives both a conductor restart and a fresh
|
|
103
|
+
* `Conductor` object dispatching into the same path.
|
|
104
|
+
*/
|
|
105
|
+
function bootstrapStateFileOf(config) {
|
|
106
|
+
const file = config.config_path;
|
|
107
|
+
return join(dirname(file), `${basename(file, '.js')}.bootstrapped.json`);
|
|
108
|
+
}
|
|
109
|
+
function loadBootstrapState(file) {
|
|
110
|
+
if (!existsSync(file))
|
|
111
|
+
return {};
|
|
112
|
+
try {
|
|
113
|
+
return JSON.parse(readFileSync(file, 'utf8'));
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
return {};
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
function hasSucceededAfterCreate(file, worktreePath) {
|
|
120
|
+
return loadBootstrapState(file)[worktreePath] === true;
|
|
121
|
+
}
|
|
122
|
+
/** Best-effort, like `saveFrozenSeen`: an unwritable state file must not wedge dispatch. */
|
|
123
|
+
function recordAfterCreateOutcome(file, worktreePath, ok) {
|
|
124
|
+
try {
|
|
125
|
+
const state = loadBootstrapState(file);
|
|
126
|
+
if (ok)
|
|
127
|
+
state[worktreePath] = true;
|
|
128
|
+
else
|
|
129
|
+
delete state[worktreePath];
|
|
130
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
131
|
+
writeFileSync(file, `${JSON.stringify(state)}\n`);
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
// best-effort: a later dispatch simply re-runs after_create.
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
function renderPrompt(template, values) {
|
|
138
|
+
return template
|
|
139
|
+
.replaceAll('{identifier}', values.identifier)
|
|
140
|
+
.replaceAll('{state}', values.state)
|
|
141
|
+
.replaceAll('{runUuid}', values.runUuid);
|
|
142
|
+
}
|
|
143
|
+
export class Conductor {
|
|
144
|
+
config;
|
|
145
|
+
remote;
|
|
146
|
+
executor;
|
|
147
|
+
workspace;
|
|
148
|
+
agents;
|
|
149
|
+
logger;
|
|
150
|
+
checkoutRoot;
|
|
151
|
+
frozenStateFile;
|
|
152
|
+
uuid = null;
|
|
153
|
+
/**
|
|
154
|
+
* Frozen tickets seen on the last poll, by uuid, with the worktree their
|
|
155
|
+
* hook ran in. Mirrored to `frozenStateFile` when one is given, so an
|
|
156
|
+
* unfreeze while the conductor is down still gets its `on_resume` and a
|
|
157
|
+
* restart does not repeat `on_freeze`; loaded on the first observation.
|
|
158
|
+
*/
|
|
159
|
+
frozenSeen = null;
|
|
160
|
+
/** The addons with contract-2 inbox messages this conductor runs. */
|
|
161
|
+
inboxAddons;
|
|
162
|
+
/** The per-item work each tick enqueues. */
|
|
163
|
+
queue;
|
|
164
|
+
/** Serializes git worktree add/remove: parallel ones fight over git locks. */
|
|
165
|
+
gitLock = Promise.resolve();
|
|
166
|
+
constructor(config, remote, executor, workspace, agents, logger, checkoutRoot = process.cwd(), frozenStateFile) {
|
|
167
|
+
this.config = config;
|
|
168
|
+
this.remote = remote;
|
|
169
|
+
this.executor = executor;
|
|
170
|
+
this.workspace = workspace;
|
|
171
|
+
this.agents = agents;
|
|
172
|
+
this.logger = logger;
|
|
173
|
+
this.checkoutRoot = checkoutRoot;
|
|
174
|
+
this.frozenStateFile = frozenStateFile;
|
|
175
|
+
this.inboxAddons = new InboxRuntime(config.inboxAddons ?? [], remote, logger, (ticket) => this.runEnv(ticket, undefined, ticket.state));
|
|
176
|
+
this.queue = new ActionQueue(logger);
|
|
177
|
+
}
|
|
178
|
+
serialGit(fn) {
|
|
179
|
+
const result = this.gitLock.then(fn);
|
|
180
|
+
this.gitLock = result.catch(() => { });
|
|
181
|
+
return result;
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* `conductor_id` is required and the loader throws without it (BB-353), so
|
|
185
|
+
* there is no directory-derived fallback here (BB-450 D5) — that fallback
|
|
186
|
+
* was the mechanism by which one checkout came to be registered under two
|
|
187
|
+
* different ids.
|
|
188
|
+
*/
|
|
189
|
+
get id() {
|
|
190
|
+
const id = this.config.conductor_id ?? this.config.machine_id;
|
|
191
|
+
if (id === undefined || id.trim() === '') {
|
|
192
|
+
throw new Error('conductor config has no conductor_id');
|
|
193
|
+
}
|
|
194
|
+
return id;
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Host stem for bb_machine.host_key. Prefer config.host_key from
|
|
198
|
+
* MachineContext.machine_id; fall back to a best-effort parse of the composed
|
|
199
|
+
* conductor_id only when the loader could not supply the stem.
|
|
200
|
+
*/
|
|
201
|
+
get hostKey() {
|
|
202
|
+
return hostKeyOf(this.config);
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Whether this instance is the serve-loop process itself (BB-232).
|
|
206
|
+
*
|
|
207
|
+
* Off by default so that only a caller which genuinely owns the process can
|
|
208
|
+
* report a pid — see {@link claimServeProcess}.
|
|
209
|
+
*/
|
|
210
|
+
ownsServeProcess = false;
|
|
211
|
+
/**
|
|
212
|
+
* Declares that this instance runs *as* the conductor process, so its
|
|
213
|
+
* registrations may carry the process identity a cockpit needs to stop it.
|
|
214
|
+
*
|
|
215
|
+
* Only `cmdStartForeground` calls this. `cmdPoll` and `cmdReap` also build a
|
|
216
|
+
* Conductor and call start() + tick(), then exit; if they reported a pid,
|
|
217
|
+
* every `bakers oven once` would overwrite the running conductor's pid
|
|
218
|
+
* with its own short-lived one and the next stop would signal a dead — or
|
|
219
|
+
* recycled — process. Making the report opt-in means those two commands
|
|
220
|
+
* cannot do so even by accident.
|
|
221
|
+
*/
|
|
222
|
+
claimServeProcess() {
|
|
223
|
+
this.ownsServeProcess = true;
|
|
224
|
+
}
|
|
225
|
+
/** This conductor's registration payload, built from config. */
|
|
226
|
+
registration() {
|
|
227
|
+
const base = {
|
|
228
|
+
id: this.id,
|
|
229
|
+
host_key: this.hostKey,
|
|
230
|
+
project: this.config.project,
|
|
231
|
+
states: this.config.states,
|
|
232
|
+
workspace: this.checkoutRoot,
|
|
233
|
+
label: this.config.label,
|
|
234
|
+
max_parallel: this.config.max_parallel,
|
|
235
|
+
};
|
|
236
|
+
if (!this.ownsServeProcess)
|
|
237
|
+
return base;
|
|
238
|
+
return {
|
|
239
|
+
...base,
|
|
240
|
+
last_pid: process.pid,
|
|
241
|
+
...(this.conductorName ? { conductor_name: this.conductorName } : {}),
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* The stem of the config file this conductor was loaded from.
|
|
246
|
+
*
|
|
247
|
+
* Derived through core's own mapping rather than by slicing the filename
|
|
248
|
+
* here, so the CLI and the conductor cannot disagree about which files count
|
|
249
|
+
* as a conductor config.
|
|
250
|
+
*/
|
|
251
|
+
get conductorName() {
|
|
252
|
+
const path = this.config.config_path;
|
|
253
|
+
return path ? stemForConfigFile(basename(path)) : undefined;
|
|
254
|
+
}
|
|
255
|
+
async start() {
|
|
256
|
+
this.uuid = await this.remote.registerConductor(this.registration());
|
|
257
|
+
this.logger.info({ conductorId: this.id, uuid: this.uuid, project: this.config.project }, 'conductor started');
|
|
258
|
+
try {
|
|
259
|
+
await this.inboxAddons.register(this.id);
|
|
260
|
+
}
|
|
261
|
+
catch (err) {
|
|
262
|
+
// Without addons nothing depends on it (and an older server has no
|
|
263
|
+
// registration resource); with addons the tick retries it.
|
|
264
|
+
const log = this.inboxAddons.empty ? this.logger.warn : this.logger.error;
|
|
265
|
+
log.call(this.logger, { conductorId: this.id, err: String(err) }, 'addon registration failed');
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
/** One tick that also waits for its actions: `bakers oven once`, tests. */
|
|
269
|
+
async tick() {
|
|
270
|
+
await this.produce();
|
|
271
|
+
await this.queue.drain();
|
|
272
|
+
}
|
|
273
|
+
/** A failing list query must not stop the other steps or the claims. */
|
|
274
|
+
async step(name, fn) {
|
|
275
|
+
try {
|
|
276
|
+
await fn();
|
|
277
|
+
}
|
|
278
|
+
catch (err) {
|
|
279
|
+
this.logger.warn({ conductorId: this.id, err: String(err) }, `${name} query failed`);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* The tick body: heartbeat, the list queries and the claim loop. Every
|
|
284
|
+
* per-item job becomes an action on the queue, so no slow item can delay
|
|
285
|
+
* the heartbeat or the claims.
|
|
286
|
+
*/
|
|
287
|
+
async produce() {
|
|
288
|
+
if (!this.uuid)
|
|
289
|
+
throw new Error('not started');
|
|
290
|
+
let count = await this.remote.activeRunCount(this.id);
|
|
291
|
+
// A heartbeat is the server-side self-heal: it upserts by conductor_id (so
|
|
292
|
+
// it never 404s and recreates a vanished registration) and refreshes the
|
|
293
|
+
// lease — no client-side re-register, no separate status read. It also
|
|
294
|
+
// recomputes bb_machine.current_load (peers + this process's load).
|
|
295
|
+
await this.remote.heartbeat(this.registration(), count, this.config.lease_seconds);
|
|
296
|
+
// BB-353 D5: gate claims on the shared host budget, not only this
|
|
297
|
+
// process's active runs. After the heartbeat above, machine.current_load
|
|
298
|
+
// includes peers + self; each claim below increments the local view so
|
|
299
|
+
// pending dispatches count without a second round-trip.
|
|
300
|
+
const capacity = await this.remote.machineCapacity(this.hostKey);
|
|
301
|
+
let machineLoad = capacity?.currentLoad ?? count;
|
|
302
|
+
// BB-464 D3: gate on the MACHINE's own budget, not this conductor's
|
|
303
|
+
// config — the machine row is the one authority a shared host agrees on;
|
|
304
|
+
// falling back to config.max_parallel only when the row is unreadable.
|
|
305
|
+
const maxParallel = capacity?.maxParallel ?? this.config.max_parallel;
|
|
306
|
+
this.logger.info({
|
|
307
|
+
conductorId: this.id,
|
|
308
|
+
active: count,
|
|
309
|
+
machineLoad,
|
|
310
|
+
capacity: maxParallel,
|
|
311
|
+
}, 'conductor tick');
|
|
312
|
+
if (machineLoad > maxParallel) {
|
|
313
|
+
// BB-464 D4: a breach is its own signal, not merely the reason no
|
|
314
|
+
// further claim was made — this server-side ceiling (TicketClaimService)
|
|
315
|
+
// is the actual guard; this is visibility into how it was reached.
|
|
316
|
+
this.logger.warn({ conductorId: this.id, machineLoad, capacity: maxParallel }, 'machine over its shared host budget');
|
|
317
|
+
}
|
|
318
|
+
// A hung action blocks only its ticket's lane; make it visible.
|
|
319
|
+
this.queue.warnOverdue(this.config.lease_seconds * 1000);
|
|
320
|
+
await this.step('finalize', () => this.enqueueFinalize());
|
|
321
|
+
await this.step('reap', () => this.enqueueReap());
|
|
322
|
+
await this.step('freeze', () => this.observeFreezes());
|
|
323
|
+
await runInbox(this.remote, this.inboxAddons, this.id, this.logger, this.queue);
|
|
324
|
+
let claimed = 0;
|
|
325
|
+
let withheld = false;
|
|
326
|
+
while (machineLoad < maxParallel) {
|
|
327
|
+
const run = await this.remote.claimNext({
|
|
328
|
+
leaseSeconds: this.config.lease_seconds,
|
|
329
|
+
conductorId: this.id,
|
|
330
|
+
});
|
|
331
|
+
if (!run)
|
|
332
|
+
break;
|
|
333
|
+
if ('reason' in run) {
|
|
334
|
+
withheld = true;
|
|
335
|
+
this.logger.info({ conductorId: this.id, ...run, reportedLoad: machineLoad }, 'claim withheld: machine at capacity');
|
|
336
|
+
break;
|
|
337
|
+
}
|
|
338
|
+
this.logger.info({ conductorId: this.id, run: run.runUuid, ticket: run.ticketUuid }, 'claimed run');
|
|
339
|
+
this.queue.enqueue({
|
|
340
|
+
key: `dispatch:${run.runUuid}`,
|
|
341
|
+
lane: run.ticketUuid,
|
|
342
|
+
run: () => this.dispatchOrFail(run),
|
|
343
|
+
});
|
|
344
|
+
count += 1;
|
|
345
|
+
machineLoad += 1;
|
|
346
|
+
claimed += 1;
|
|
347
|
+
}
|
|
348
|
+
if (claimed > 0) {
|
|
349
|
+
// BB-464 AC 2: push this tick's claims into the machine row's load
|
|
350
|
+
// immediately, so a peer conductor ticking next reads the up-to-date
|
|
351
|
+
// figure instead of the pre-claim snapshot this tick started from. The
|
|
352
|
+
// staleness window this closes is what let sequential ticks reach the
|
|
353
|
+
// sum of both conductors' budgets rather than the shared ceiling.
|
|
354
|
+
await this.remote.heartbeat(this.registration(), count, this.config.lease_seconds);
|
|
355
|
+
}
|
|
356
|
+
if (claimed === 0 && !withheld) {
|
|
357
|
+
this.logger.info({
|
|
358
|
+
conductorId: this.id,
|
|
359
|
+
active: count,
|
|
360
|
+
machineLoad,
|
|
361
|
+
capacity: maxParallel,
|
|
362
|
+
}, machineLoad >= maxParallel
|
|
363
|
+
? 'at capacity, nothing claimed'
|
|
364
|
+
: 'idle, nothing to claim');
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* One-shot finalisation: for each of this conductor's runs that is state in
|
|
369
|
+
* (done, cancelled) but not yet closed, read the agent transcript from its
|
|
370
|
+
* worktree and write it as the final log, then mark the run closed. Keyed off
|
|
371
|
+
* the durable terminal-state query — no in-memory state, so a restarted
|
|
372
|
+
* conductor still finalises any unclosed run on its next tick.
|
|
373
|
+
*/
|
|
374
|
+
async enqueueFinalize() {
|
|
375
|
+
const runs = await this.remote.fetchFinalizableRuns(this.id);
|
|
376
|
+
for (const r of runs) {
|
|
377
|
+
this.queue.enqueue({
|
|
378
|
+
key: `finalize:${r.runUuid}`,
|
|
379
|
+
lane: r.ticketUuid || r.runUuid,
|
|
380
|
+
run: () => this.finalizeOne(r),
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
/** Finalises one run; a failure is logged and retried next tick. */
|
|
385
|
+
async finalizeOne(r) {
|
|
386
|
+
try {
|
|
387
|
+
// Signal the agent to stop before capturing its log. Best-effort and
|
|
388
|
+
// gated on the persistent capability (only hosted executors have a live
|
|
389
|
+
// agent) — mirrors removeWorktree's gate. For herdr this is a no-op
|
|
390
|
+
// (the tab-close in closeRunTab kills the PTY); a stop failure must
|
|
391
|
+
// never block finalisation.
|
|
392
|
+
const branch = this.executor.capabilities().persistent
|
|
393
|
+
? await this.remote.getRunTicketBranchName(r.runUuid)
|
|
394
|
+
: '';
|
|
395
|
+
if (branch) {
|
|
396
|
+
try {
|
|
397
|
+
await this.executor.stopRun(branch);
|
|
398
|
+
}
|
|
399
|
+
catch (err) {
|
|
400
|
+
this.logger.warn({ run: r.runUuid, err: String(err) }, 'agent stop failed');
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
// BB-599: a failed capture must not skip the worker and sidecar
|
|
404
|
+
// cleanup below, so it is held and re-thrown before the close write —
|
|
405
|
+
// the run then stays unclosed and the next tick retries all of it.
|
|
406
|
+
let captured;
|
|
407
|
+
let captureError;
|
|
408
|
+
try {
|
|
409
|
+
captured = await this.captureRun(r);
|
|
410
|
+
}
|
|
411
|
+
catch (err) {
|
|
412
|
+
captureError = err;
|
|
413
|
+
}
|
|
414
|
+
// The transcript is captured, so killing the PTY is safe (BB-171).
|
|
415
|
+
if (branch)
|
|
416
|
+
await this.closeRunTab(branch, r.runUuid, r.runId);
|
|
417
|
+
// BB-599: `after_run` fires at run end — after the worker is gone,
|
|
418
|
+
// before the durable close. A failed close write repeats it next tick,
|
|
419
|
+
// which is why listeners must be idempotent.
|
|
420
|
+
await this.runFinalizeHook(r);
|
|
421
|
+
if (!captured)
|
|
422
|
+
throw captureError;
|
|
423
|
+
// `?? true` is the default: an unset flag, and every config written
|
|
424
|
+
// before this setting existed, keep uploading exactly as before.
|
|
425
|
+
// `finalizeRun()` already spreads `...(log ? { log } : {})`, so an
|
|
426
|
+
// empty string omits the attribute from the PATCH entirely — no
|
|
427
|
+
// server change is needed and no empty log is written.
|
|
428
|
+
const uploadedLog = (this.config.capture_logs ?? true) ? captured.log : '';
|
|
429
|
+
await this.remote.finalizeRun(r.runUuid, uploadedLog, captured.metrics);
|
|
430
|
+
this.logger.info({ run: r.runUuid, footprint: captured.metrics }, 'run finalised');
|
|
431
|
+
}
|
|
432
|
+
catch (err) {
|
|
433
|
+
this.logger.warn({ run: r.runUuid, err: String(err) }, 'run finalise failed');
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
/** Read the run's transcript and parse its footprint for the close write. */
|
|
437
|
+
async captureRun(r) {
|
|
438
|
+
// Route to the agent that actually ran (BB-144): the persisted
|
|
439
|
+
// run.agent id, falling back to the first candidate for a run written
|
|
440
|
+
// before the field existed (empty id).
|
|
441
|
+
const resolved = this.agents.find((a) => a.id === r.agent) ?? this.agents[0];
|
|
442
|
+
if (!resolved) {
|
|
443
|
+
throw new Error('no agent candidates configured for finalize');
|
|
444
|
+
}
|
|
445
|
+
// Scoped to this run's own identity (BB-473): the run's UUID as a
|
|
446
|
+
// session id, its deterministic attempt family (review round 4,
|
|
447
|
+
// Critical 2) so a re-dispatch resolves by WHICH attempt rather than
|
|
448
|
+
// by a time window a sibling session could still beat, plus its
|
|
449
|
+
// recorded start as a lower-bound tie-breaker for agents with no
|
|
450
|
+
// session mechanism at all — so a foreign transcript left behind by
|
|
451
|
+
// a resumed/re-dispatched/reused worktree can never be finalised
|
|
452
|
+
// here.
|
|
453
|
+
const log = r.worktreePath
|
|
454
|
+
? await resolved.agent.getRunLog(r.worktreePath, {
|
|
455
|
+
sessionId: r.runUuid,
|
|
456
|
+
attemptFamily: attemptFamily(r.runUuid),
|
|
457
|
+
...(r.startedAt ? { startedAt: r.startedAt } : {}),
|
|
458
|
+
})
|
|
459
|
+
: '';
|
|
460
|
+
// Footprint (BB-132): the agent parses its own transcript for the
|
|
461
|
+
// effort metrics AND its wall-clock duration_s (from the transcript's
|
|
462
|
+
// first→last timestamps — BB-151). Deriving duration from the log
|
|
463
|
+
// keeps it a single source with one format, avoiding the earlier
|
|
464
|
+
// `started_at` re-read that mis-parsed the JSON:API ISO string to 0.
|
|
465
|
+
// The conductor owns the run, so this write always lands (the agent
|
|
466
|
+
// session could not).
|
|
467
|
+
const parsed = resolved.agent.parseFootprint(log);
|
|
468
|
+
// Runtime provenance beside the footprint: the harness that drove the
|
|
469
|
+
// session, the model that actually ran, and the exact model id the
|
|
470
|
+
// transcript records. All three ride the SAME close PATCH — no extra
|
|
471
|
+
// write path, so the conductor-owner field access that already covers
|
|
472
|
+
// the metrics covers these too.
|
|
473
|
+
//
|
|
474
|
+
// `model` and `model_version` each prefer an explicit configured pin
|
|
475
|
+
// (`resolved.model` / `resolved.modelVersion`) when one is set — a
|
|
476
|
+
// human's own answer beats a guess. Absent a pin, each falls back to
|
|
477
|
+
// the raw, unmodified model id the transcript names (`parsed.model`)
|
|
478
|
+
// rather than staying empty: that is the real model that ran, not a
|
|
479
|
+
// fabricated placeholder. No per-harness splitting or abbreviation
|
|
480
|
+
// happens on either source, which is what keeps the columns
|
|
481
|
+
// comparable across harnesses. Each key is spread conditionally: an
|
|
482
|
+
// unresolved value sends no key at all.
|
|
483
|
+
const metrics = {
|
|
484
|
+
tokens: parsed.tokens,
|
|
485
|
+
duration_s: parsed.duration_s,
|
|
486
|
+
agent_turns: parsed.agent_turns,
|
|
487
|
+
tool_calls: parsed.tool_calls,
|
|
488
|
+
user_prompts: parsed.user_prompts,
|
|
489
|
+
user_prompt_words: parsed.user_prompt_words,
|
|
490
|
+
active_s: parsed.active_s,
|
|
491
|
+
harness: resolved.id,
|
|
492
|
+
...(resolved.model !== undefined
|
|
493
|
+
? { model: resolved.model }
|
|
494
|
+
: parsed.model !== undefined
|
|
495
|
+
? { model: parsed.model }
|
|
496
|
+
: {}),
|
|
497
|
+
...(resolved.modelVersion !== undefined
|
|
498
|
+
? { model_version: resolved.modelVersion }
|
|
499
|
+
: parsed.model !== undefined
|
|
500
|
+
? { model_version: parsed.model }
|
|
501
|
+
: {}),
|
|
502
|
+
};
|
|
503
|
+
return { log, metrics };
|
|
504
|
+
}
|
|
505
|
+
/**
|
|
506
|
+
* Close one run's tab. Run-scoped (BB-183): only the `#<runId>` tab, never
|
|
507
|
+
* a sibling run's. Best-effort: a close failure never mislabels the run as
|
|
508
|
+
* "finalise failed" or replaces a dispatch error.
|
|
509
|
+
*/
|
|
510
|
+
async closeRunTab(branch, runUuid, runId) {
|
|
511
|
+
try {
|
|
512
|
+
await this.executor.cleanupRun(branch, runId);
|
|
513
|
+
}
|
|
514
|
+
catch (err) {
|
|
515
|
+
this.logger.warn({ run: runUuid, err: String(err) }, 'run tab cleanup failed');
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
/**
|
|
519
|
+
* `after_run` for an ended run (BB-599), rebuilt from durable state: the
|
|
520
|
+
* run record names its ticket and start state, and the env is resolved from
|
|
521
|
+
* the CURRENT ticket — so a restarted conductor stops the same sidecars.
|
|
522
|
+
* Like `after_done`, it needs the worktree directory to exist.
|
|
523
|
+
*/
|
|
524
|
+
async runFinalizeHook(r) {
|
|
525
|
+
if (!r.worktreePath || !existsSync(r.worktreePath))
|
|
526
|
+
return;
|
|
527
|
+
const ctx = await this.listenerContext(r.ticketUuid, r.worktreePath, {
|
|
528
|
+
hook: 'after_run',
|
|
529
|
+
run: { uuid: r.runUuid, id: r.runId },
|
|
530
|
+
...(r.stateAtStart ? { state: r.stateAtStart } : {}),
|
|
531
|
+
});
|
|
532
|
+
await this.runSlot('after_run', r.worktreePath, ctx?.ticket.identifier || r.ticketUuid || r.runUuid, ctx, ctx?.env);
|
|
533
|
+
}
|
|
534
|
+
/**
|
|
535
|
+
* Rebuild a listener context from the ticket's current record, or
|
|
536
|
+
* `undefined` (logged) when it cannot be read — the caller still runs the
|
|
537
|
+
* project shell hook, it only loses the addon listeners. Built even with no
|
|
538
|
+
* listeners: the shell `after_run` takes its env from it.
|
|
539
|
+
*/
|
|
540
|
+
async listenerContext(ticketUuid, worktreePath, opts) {
|
|
541
|
+
try {
|
|
542
|
+
if (!ticketUuid)
|
|
543
|
+
throw new Error('run has no ticket');
|
|
544
|
+
const ticket = await this.remote.getTicket(ticketUuid);
|
|
545
|
+
return {
|
|
546
|
+
worktreePath,
|
|
547
|
+
ticket,
|
|
548
|
+
state: opts.state ?? ticket.state,
|
|
549
|
+
env: await this.runEnv(ticket, opts.run?.uuid, opts.state ?? ticket.state),
|
|
550
|
+
...(opts.run ? { run: opts.run } : {}),
|
|
551
|
+
};
|
|
552
|
+
}
|
|
553
|
+
catch (err) {
|
|
554
|
+
this.logger.warn({
|
|
555
|
+
hook: opts.hook,
|
|
556
|
+
ticket: ticketUuid,
|
|
557
|
+
run: opts.run?.uuid,
|
|
558
|
+
err: String(err),
|
|
559
|
+
}, 'hook listener context unavailable — listeners not run');
|
|
560
|
+
return undefined;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
get listeners() {
|
|
564
|
+
return this.config.hookListeners ?? [];
|
|
565
|
+
}
|
|
566
|
+
/**
|
|
567
|
+
* Run one lifecycle slot (BB-599): every addon listener in accumulator
|
|
568
|
+
* order, awaited, then the project shell hook through the executor. Never
|
|
569
|
+
* throws. A failing listener is logged by slot, listener, ticket and run —
|
|
570
|
+
* never with its message or the env — and reported in the returned result
|
|
571
|
+
* the same way a failing shell command is, so the rest of the slot, the
|
|
572
|
+
* shell hook and dispatch all continue.
|
|
573
|
+
*/
|
|
574
|
+
async runSlot(name, cwd, ticketRef, ctx, shellEnv) {
|
|
575
|
+
const failed = [];
|
|
576
|
+
if (ctx) {
|
|
577
|
+
const env = ctx.env;
|
|
578
|
+
for (const [index, listener] of this.listeners.entries()) {
|
|
579
|
+
const fn = listener[name];
|
|
580
|
+
if (!fn)
|
|
581
|
+
continue;
|
|
582
|
+
const id = listener.id ?? `#${index}`;
|
|
583
|
+
const before = { ...env };
|
|
584
|
+
try {
|
|
585
|
+
await fn(ctx);
|
|
586
|
+
}
|
|
587
|
+
catch (err) {
|
|
588
|
+
failed.push(id);
|
|
589
|
+
this.logger.error({
|
|
590
|
+
hook: name,
|
|
591
|
+
listener: id,
|
|
592
|
+
ticket: ticketRef,
|
|
593
|
+
run: ctx.run?.uuid,
|
|
594
|
+
err: errorKind(err),
|
|
595
|
+
}, 'hook listener failed');
|
|
596
|
+
}
|
|
597
|
+
ctx.env = env;
|
|
598
|
+
this.guardEnv(env, before, {
|
|
599
|
+
hook: name,
|
|
600
|
+
listener: id,
|
|
601
|
+
ticket: ticketRef,
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
const shell = await this.executor.runHook(name, cwd, { ticket: ticketRef }, shellEnv);
|
|
606
|
+
if (failed.length === 0)
|
|
607
|
+
return shell;
|
|
608
|
+
const error = `hook listener failed: ${failed.join(', ')}`;
|
|
609
|
+
return {
|
|
610
|
+
hook: name,
|
|
611
|
+
ran: true,
|
|
612
|
+
ok: false,
|
|
613
|
+
error: shell.ok ? error : `${shell.error}; ${error}`,
|
|
614
|
+
};
|
|
615
|
+
}
|
|
616
|
+
/**
|
|
617
|
+
* Undo a listener's env change the conductor does not accept: a reserved
|
|
618
|
+
* key keeps the conductor's value, and an invalid name or a non-string value
|
|
619
|
+
* never reaches a spawn. Key names only are logged.
|
|
620
|
+
*/
|
|
621
|
+
guardEnv(env, before, log) {
|
|
622
|
+
const rejected = [];
|
|
623
|
+
for (const key of Object.keys(env)) {
|
|
624
|
+
const value = env[key];
|
|
625
|
+
const ok = isReservedEnvKey(key)
|
|
626
|
+
? before[key] === value
|
|
627
|
+
: ENV_KEY.test(key) && typeof value === 'string';
|
|
628
|
+
if (ok)
|
|
629
|
+
continue;
|
|
630
|
+
rejected.push(key);
|
|
631
|
+
if (key in before)
|
|
632
|
+
env[key] = before[key];
|
|
633
|
+
else
|
|
634
|
+
delete env[key];
|
|
635
|
+
}
|
|
636
|
+
for (const key of Object.keys(before)) {
|
|
637
|
+
if (isReservedEnvKey(key) && !(key in env)) {
|
|
638
|
+
env[key] = before[key];
|
|
639
|
+
rejected.push(key);
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
if (rejected.length > 0) {
|
|
643
|
+
this.logger.warn({ ...log, keys: rejected }, 'hook listener env change rejected');
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
/**
|
|
647
|
+
* Run `on_freeze` once for each newly frozen ticket and `on_resume` once
|
|
648
|
+
* for each that left the frozen list, in the ticket's worktree on this
|
|
649
|
+
* machine. A ticket without a worktree here gets no hook. Never throws: an
|
|
650
|
+
* unreadable list keeps the previous observation for the next poll.
|
|
651
|
+
*/
|
|
652
|
+
async observeFreezes() {
|
|
653
|
+
let frozen;
|
|
654
|
+
try {
|
|
655
|
+
frozen = await this.remote.fetchFrozenTickets(this.id);
|
|
656
|
+
}
|
|
657
|
+
catch (err) {
|
|
658
|
+
this.logger.warn({ conductorId: this.id, err: String(err) }, 'frozen tickets unreadable — freeze hooks deferred');
|
|
659
|
+
return;
|
|
660
|
+
}
|
|
661
|
+
const seen = this.loadFrozenSeen();
|
|
662
|
+
const current = new Set(frozen.map((t) => t.ticketUuid));
|
|
663
|
+
let changed = false;
|
|
664
|
+
for (const [uuid, was] of [...seen]) {
|
|
665
|
+
if (current.has(uuid))
|
|
666
|
+
continue;
|
|
667
|
+
seen.delete(uuid);
|
|
668
|
+
changed = true;
|
|
669
|
+
this.queue.enqueue({
|
|
670
|
+
key: `freeze:${uuid}:on_resume`,
|
|
671
|
+
lane: uuid,
|
|
672
|
+
run: () => this.runFreezeSlot('on_resume', uuid, was),
|
|
673
|
+
});
|
|
674
|
+
}
|
|
675
|
+
for (const t of frozen) {
|
|
676
|
+
if (seen.has(t.ticketUuid))
|
|
677
|
+
continue;
|
|
678
|
+
seen.set(t.ticketUuid, {
|
|
679
|
+
identifier: t.identifier,
|
|
680
|
+
worktreePath: t.worktreePath,
|
|
681
|
+
});
|
|
682
|
+
changed = true;
|
|
683
|
+
this.queue.enqueue({
|
|
684
|
+
key: `freeze:${t.ticketUuid}:on_freeze`,
|
|
685
|
+
lane: t.ticketUuid,
|
|
686
|
+
run: () => this.runFreezeSlot('on_freeze', t.ticketUuid, t),
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
if (changed)
|
|
690
|
+
this.saveFrozenSeen(seen);
|
|
691
|
+
}
|
|
692
|
+
loadFrozenSeen() {
|
|
693
|
+
if (this.frozenSeen)
|
|
694
|
+
return this.frozenSeen;
|
|
695
|
+
this.frozenSeen = new Map();
|
|
696
|
+
const file = this.frozenStateFile;
|
|
697
|
+
if (file && existsSync(file)) {
|
|
698
|
+
try {
|
|
699
|
+
this.frozenSeen = new Map(Object.entries(JSON.parse(readFileSync(file, 'utf8'))));
|
|
700
|
+
}
|
|
701
|
+
catch (err) {
|
|
702
|
+
this.logger.warn({ file, err: String(err) }, 'frozen ticket state unreadable — starting empty');
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
return this.frozenSeen;
|
|
706
|
+
}
|
|
707
|
+
saveFrozenSeen(seen) {
|
|
708
|
+
const file = this.frozenStateFile;
|
|
709
|
+
if (!file)
|
|
710
|
+
return;
|
|
711
|
+
try {
|
|
712
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
713
|
+
writeFileSync(file, `${JSON.stringify(Object.fromEntries(seen))}\n`);
|
|
714
|
+
}
|
|
715
|
+
catch (err) {
|
|
716
|
+
this.logger.warn({ file, err: String(err) }, 'frozen ticket state not saved');
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
async runFreezeSlot(name, ticketUuid, t) {
|
|
720
|
+
if (!t.worktreePath || !existsSync(t.worktreePath))
|
|
721
|
+
return;
|
|
722
|
+
const ctx = await this.listenerContext(ticketUuid, t.worktreePath, {
|
|
723
|
+
hook: name,
|
|
724
|
+
});
|
|
725
|
+
await this.runSlot(name, t.worktreePath, t.identifier, ctx, ctx?.env);
|
|
726
|
+
}
|
|
727
|
+
/**
|
|
728
|
+
* Reconcile closed tickets against their herdr worktrees and tear down any
|
|
729
|
+
* whose workspace still lingers (BB-89). Driven by the durable `cleaned_up`
|
|
730
|
+
* flag via {@link BbRemote.fetchUncleanedTickets}: every ticket the server
|
|
731
|
+
* has already closed but not yet cleaned. The SAME reconciliation runs on
|
|
732
|
+
* every tick AND standalone as `bakers oven clean`, and the query is
|
|
733
|
+
* scoped to THIS conductor (BB-121) — so it catches the orphans a
|
|
734
|
+
* live-tick-only path structurally cannot (the conductor was down when the
|
|
735
|
+
* ticket closed, or crashed mid-cleanup) for its OWN tickets. A foreign
|
|
736
|
+
* (non-bakers) worktree has no bakers ticket, so it is never on the work list; a
|
|
737
|
+
* ticket assigned to another conductor is filtered out at the query — both are
|
|
738
|
+
* left untouched with no host enumeration.
|
|
739
|
+
*
|
|
740
|
+
* The conductor is structurally UNABLE to close a ticket (BB-451): closing
|
|
741
|
+
* is a server-side projection of terminality written by
|
|
742
|
+
* `BbTicket::preSave()` on any save of a terminal, unclosed ticket, so it
|
|
743
|
+
* holds for every write path and cannot be withheld by a teardown hiccup —
|
|
744
|
+
* the reap reasons about `cleaned_up` alone. `cleaned_up` is withheld unless
|
|
745
|
+
* the teardown is ACCOUNTED FOR (BB-141 RC-2), which `removeWorktree`
|
|
746
|
+
* reports as one of three outcomes (BB-293):
|
|
747
|
+
* - `removed` — gone because this host tore it down → flag cleaned;
|
|
748
|
+
* - `already_absent` — nothing was there to tear down. A FINISHED teardown,
|
|
749
|
+
* not a failure: it is recorded in this log AND on the run record, then
|
|
750
|
+
* flagged cleaned. Recording it is the point — swallowing the absence and
|
|
751
|
+
* leaving `cleaned_up=0` is what made every later reap repeat the same
|
|
752
|
+
* failing teardown;
|
|
753
|
+
* - `unverified` — it could not be resolved and may still be on disk (e.g.
|
|
754
|
+
* its `worktree_path` was never persisted). Like a THROW, it logs loudly
|
|
755
|
+
* and leaves `cleaned_up=0` so the next reconciliation retries.
|
|
756
|
+
* Flagging cleaned on an unverified teardown was the root cause of the ~28
|
|
757
|
+
* never-touched leftovers, because the ticket then dropped off the
|
|
758
|
+
* (cleaned_up=0) work list forever.
|
|
759
|
+
* Re-running on an already-cleaned ticket is a no-op — it is off the list
|
|
760
|
+
* (idempotent). Each ticket is isolated in a try/catch so one failure never
|
|
761
|
+
* aborts the rest.
|
|
762
|
+
*
|
|
763
|
+
* The `after_done` hook runs through the executor (BB-84), which owns the
|
|
764
|
+
* best-effort policy — it logs+swallows a hook failure and never throws, so
|
|
765
|
+
* the core needs no per-call-site try/catch around it.
|
|
766
|
+
*/
|
|
767
|
+
async reap() {
|
|
768
|
+
await this.enqueueReap();
|
|
769
|
+
await this.queue.drain();
|
|
770
|
+
}
|
|
771
|
+
async enqueueReap() {
|
|
772
|
+
const tickets = await this.remote.fetchUncleanedTickets(this.id);
|
|
773
|
+
for (const t of tickets) {
|
|
774
|
+
this.queue.enqueue({
|
|
775
|
+
key: `reap:${t.ticketUuid}`,
|
|
776
|
+
lane: t.ticketUuid,
|
|
777
|
+
run: () => this.reapOne(t),
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
/**
|
|
782
|
+
* Record the merge request's size on a done ticket while its worktree still
|
|
783
|
+
* exists. The server keeps `metrics` writable until `cleaned_up`; a failure
|
|
784
|
+
* is logged and never holds up the teardown. A teardown that keeps failing
|
|
785
|
+
* re-measures on every reap; the rows are replaced, so they stay correct.
|
|
786
|
+
*/
|
|
787
|
+
async recordMrChanges(t) {
|
|
788
|
+
if (t.state !== 'done' || !t.hasMr)
|
|
789
|
+
return;
|
|
790
|
+
try {
|
|
791
|
+
if (!t.baseBranch)
|
|
792
|
+
throw new Error('ticket has no base branch');
|
|
793
|
+
// Under the git lock: workspace.ensure fetches the same ref into this repo.
|
|
794
|
+
const values = await this.serialGit(() => measureMrChanges(t.worktreePath, t.baseBranch));
|
|
795
|
+
await this.remote.replaceTicketMetrics(t.ticketUuid, 'mr_changes.', mrChangesRows(values, Math.floor(Date.now() / 1000)));
|
|
796
|
+
}
|
|
797
|
+
catch (err) {
|
|
798
|
+
this.logger.warn({ ticket: t.ticketUuid, err: String(err) }, 'mr_changes not recorded');
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
/** Reaps one ticket; isolated, so one failure never aborts the rest. */
|
|
802
|
+
async reapOne(t) {
|
|
803
|
+
try {
|
|
804
|
+
// Infra teardown. The `after_done` hook runs in the worktree regardless
|
|
805
|
+
// of executor kind (a non-persistent gitWorkspace still has an on-disk
|
|
806
|
+
// worktree); only the workspace removal is gated on the persistent
|
|
807
|
+
// capability (mirrors the executor-agnostic release path). Pass the
|
|
808
|
+
// stable worktreePath so the executor resolves the worktree by path,
|
|
809
|
+
// not by the mutable checked-out branch (the coding agent may rename
|
|
810
|
+
// it). Teardown is idempotent: a worktree herdr no longer tracks is a
|
|
811
|
+
// no-op / on-disk reclaim, not a failure.
|
|
812
|
+
//
|
|
813
|
+
// The hook needs the DIRECTORY, not just a recorded path (BB-293): a
|
|
814
|
+
// hook is a shell spawned with the worktree as its `cwd`, so against an
|
|
815
|
+
// absent one the spawn fails with ENOENT and the shared exec helper logs
|
|
816
|
+
// `ERROR: exec failed` — for the very condition this ticket declares
|
|
817
|
+
// normal. There is no useful work such a hook could do either: a command
|
|
818
|
+
// that must run inside the worktree cannot run anywhere else.
|
|
819
|
+
//
|
|
820
|
+
// The absence is logged rather than passed over in silence, because an
|
|
821
|
+
// `after_done` can reach beyond the directory it runs in — `ddev delete
|
|
822
|
+
// -Oy` also drops a DDEV project registration and its database — and
|
|
823
|
+
// that leftover is then an operator's to clear by hand.
|
|
824
|
+
//
|
|
825
|
+
// But the line states the CONDITION, not a decision about a hook: hook
|
|
826
|
+
// commands are executor-owned config that the core cannot see, so it
|
|
827
|
+
// does not know whether an `after_done` is configured at all. Saying a
|
|
828
|
+
// hook was "skipped" would be false wherever none exists (the generated
|
|
829
|
+
// playground config is exactly such a case). What the core does know is
|
|
830
|
+
// that the directory is gone and that no `after_done` ran — both true in
|
|
831
|
+
// either configuration.
|
|
832
|
+
//
|
|
833
|
+
// `existsSync` here is a plain filesystem question, not worktree
|
|
834
|
+
// knowledge: the core stays executor-agnostic and asks nothing about
|
|
835
|
+
// git. The check is deliberately reap-local — at dispatch a worktree the
|
|
836
|
+
// workspace just created MUST exist, and silencing a hook there would
|
|
837
|
+
// hide a real provisioning failure.
|
|
838
|
+
if (t.worktreePath) {
|
|
839
|
+
if (existsSync(t.worktreePath)) {
|
|
840
|
+
await this.recordMrChanges(t);
|
|
841
|
+
// BB-599: listeners get the ticket's current context; the shell
|
|
842
|
+
// hook keeps its inherited env, exactly as before.
|
|
843
|
+
const ctx = this.listeners.length === 0
|
|
844
|
+
? undefined
|
|
845
|
+
: await this.listenerContext(t.ticketUuid, t.worktreePath, {
|
|
846
|
+
hook: 'after_done',
|
|
847
|
+
...(t.runUuid ? { run: { uuid: t.runUuid } } : {}),
|
|
848
|
+
});
|
|
849
|
+
await this.runSlot('after_done', t.worktreePath, t.ticketUuid, ctx, undefined);
|
|
850
|
+
// BB-663 review F2: a torn-down environment invalidates any
|
|
851
|
+
// recorded `after_create` success — otherwise a worktree that
|
|
852
|
+
// survives teardown (non-persistent executor, unverified reap)
|
|
853
|
+
// looks bootstrapped forever and a reopened ticket dispatches
|
|
854
|
+
// straight into a worktree with no provisioned environment.
|
|
855
|
+
recordAfterCreateOutcome(bootstrapStateFileOf(this.config), t.worktreePath, false);
|
|
856
|
+
}
|
|
857
|
+
else {
|
|
858
|
+
this.logger.info({
|
|
859
|
+
ticket: t.ticketUuid,
|
|
860
|
+
hook: 'after_done',
|
|
861
|
+
worktreePath: t.worktreePath,
|
|
862
|
+
}, 'worktree directory absent — after_done not run');
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
if (this.executor.capabilities().persistent) {
|
|
866
|
+
let outcome;
|
|
867
|
+
try {
|
|
868
|
+
// removeWorktree reports what the teardown DID (BB-293):
|
|
869
|
+
// `removed` = the worktree is gone because this host removed it;
|
|
870
|
+
// `already_absent` = there was nothing to tear down; `unverified` =
|
|
871
|
+
// it could NOT be resolved and may still be on disk (BB-141 RC-2
|
|
872
|
+
// — e.g. worktree_path was never persisted and herdr lost track of
|
|
873
|
+
// the branch). A THROW is a hard failure.
|
|
874
|
+
outcome = await this.serialGit(() => this.executor.removeWorktree(t.branchName, t.worktreePath || undefined));
|
|
875
|
+
}
|
|
876
|
+
catch (err) {
|
|
877
|
+
this.logger.warn({
|
|
878
|
+
ticket: t.ticketUuid,
|
|
879
|
+
branch: t.branchName,
|
|
880
|
+
worktreePath: t.worktreePath,
|
|
881
|
+
err: String(err),
|
|
882
|
+
}, 'worktree teardown failed');
|
|
883
|
+
return; // leave cleaned_up=0 → the next reconciliation retries.
|
|
884
|
+
}
|
|
885
|
+
if (outcome === 'unverified') {
|
|
886
|
+
// BB-468 — an unverified teardown with NO run and NO recorded
|
|
887
|
+
// path has nothing left for a later reap to resolve: no run will
|
|
888
|
+
// appear on a terminal ticket, and no branch will appear for work
|
|
889
|
+
// that never started (`reap()` passes `t.worktreePath ||
|
|
890
|
+
// undefined`, so a never-ran ticket and a lost-path ticket reach
|
|
891
|
+
// the executor byte-identically — only the reaper, which alone
|
|
892
|
+
// knows `runUuid === ''`, can tell them apart). That retry is not
|
|
893
|
+
// deferred, it is unreachable, so it is logged ONCE at info and
|
|
894
|
+
// accounted for below rather than retried forever. BB-141 RC-2
|
|
895
|
+
// is unchanged for every ticket that DOES carry a run or a
|
|
896
|
+
// recorded path: those keep retrying below, unaffected.
|
|
897
|
+
if (!t.runUuid && !t.worktreePath) {
|
|
898
|
+
this.logger.info({ ticket: t.ticketUuid, branch: t.branchName }, 'no run and no recorded path — nothing to tear down');
|
|
899
|
+
}
|
|
900
|
+
else {
|
|
901
|
+
// Unverified teardown (BB-141 RC-2): do NOT flag cleaned_up, or
|
|
902
|
+
// the ticket drops off fetchUncleanedTickets (which filters
|
|
903
|
+
// cleaned_up=0) and the orphan is never retried. Leave it on the
|
|
904
|
+
// work list so a later reap — with a persisted path, or once the
|
|
905
|
+
// executor resolves it via the parent-repo git worktree list —
|
|
906
|
+
// can finish the job.
|
|
907
|
+
this.logger.warn({
|
|
908
|
+
ticket: t.ticketUuid,
|
|
909
|
+
branch: t.branchName,
|
|
910
|
+
worktreePath: t.worktreePath,
|
|
911
|
+
}, 'worktree teardown unverified — retrying next reap');
|
|
912
|
+
return; // leave cleaned_up=0 → the next reconciliation retries.
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
if (outcome === 'already_absent') {
|
|
916
|
+
// BB-293: the worktree was gone before we got here. That is a
|
|
917
|
+
// FINISHED teardown, so it is recorded rather than swallowed and
|
|
918
|
+
// the ticket is flagged cleaned below — the next reap will not
|
|
919
|
+
// retry it.
|
|
920
|
+
this.logger.info({
|
|
921
|
+
ticket: t.ticketUuid,
|
|
922
|
+
branch: t.branchName,
|
|
923
|
+
worktreePath: t.worktreePath,
|
|
924
|
+
}, 'worktree already absent — teardown complete');
|
|
925
|
+
// A ticket with no run has no record to write to — the conductor
|
|
926
|
+
// log above is then the whole account.
|
|
927
|
+
if (t.runUuid) {
|
|
928
|
+
try {
|
|
929
|
+
await this.remote.appendRunNote(t.runUuid, `worktree already absent at teardown: ${t.worktreePath || '(path unknown)'}`);
|
|
930
|
+
}
|
|
931
|
+
catch (err) {
|
|
932
|
+
// Best-effort: an unreachable control plane must not keep a
|
|
933
|
+
// finished teardown from being flagged.
|
|
934
|
+
this.logger.warn({ ticket: t.ticketUuid, run: t.runUuid, err: String(err) }, 'run note write failed');
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
// Teardown verified locally, the worktree was already absent, or there
|
|
940
|
+
// is nothing hosted to tear down: flag it cleaned so it drops off the
|
|
941
|
+
// work list.
|
|
942
|
+
await this.remote.markTicketCleanedUp(t.ticketUuid);
|
|
943
|
+
this.logger.info({ ticket: t.ticketUuid }, 'ticket cleaned up');
|
|
944
|
+
}
|
|
945
|
+
catch (err) {
|
|
946
|
+
this.logger.warn({ ticket: t.ticketUuid, err: String(err) }, 'ticket cleanup failed');
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
/**
|
|
950
|
+
* The env a run executes in: the ticket's effective env_vars under the core
|
|
951
|
+
* vars. Used at dispatch and, rebuilt from the current ticket, for the
|
|
952
|
+
* `after_run` / `after_done` listener contexts (BB-599).
|
|
953
|
+
*/
|
|
954
|
+
async runEnv(t, runUuid, state) {
|
|
955
|
+
// BB-495: the agent names the workspace on its own calls now (the
|
|
956
|
+
// dropsh transport stamps none), so it has to know which one it is
|
|
957
|
+
// standing in and the full radius it may reach. Reserved core keys
|
|
958
|
+
// always win over per-ticket env_vars (below), so no precedence code is
|
|
959
|
+
// needed here. A key whose value could not be resolved is OMITTED, never
|
|
960
|
+
// exported empty — an empty variable reads as "set but blank" to a skill,
|
|
961
|
+
// which is worse than absent.
|
|
962
|
+
let workspaceIds;
|
|
963
|
+
try {
|
|
964
|
+
const ids = await this.remote.getReachableWorkspaces();
|
|
965
|
+
if (ids.length)
|
|
966
|
+
workspaceIds = ids.join(',');
|
|
967
|
+
}
|
|
968
|
+
catch (err) {
|
|
969
|
+
this.logger.warn({ ticket: t.identifier, err: String(err) }, 'could not resolve reachable workspaces for BB_WORKSPACE_IDS');
|
|
970
|
+
}
|
|
971
|
+
const core = {
|
|
972
|
+
BB_URL: this.config.site.base_url,
|
|
973
|
+
BB_ID: t.uuid,
|
|
974
|
+
// BB-569: lets a launch wrapper key off the ticket and the run's
|
|
975
|
+
// start state without parsing the prompt (addon-claude `measure`).
|
|
976
|
+
...(t.identifier ? { BB_TICKET_IDENTIFIER: t.identifier } : {}),
|
|
977
|
+
...(state ? { BB_STATE: state } : {}),
|
|
978
|
+
...(runUuid ? { BB_RUN_UUID: runUuid } : {}),
|
|
979
|
+
BB_TUNNEL: this.config.tunnel === true ? '1' : '0',
|
|
980
|
+
WORKSPACE_ROOT: this.checkoutRoot,
|
|
981
|
+
...(t.issueUrl ? { TICKET_URL: t.issueUrl } : {}),
|
|
982
|
+
...(t.projectName ? { BB_PROJECT: t.projectName } : {}),
|
|
983
|
+
...(t.projectId ? { BB_PROJECT_ID: t.projectId } : {}),
|
|
984
|
+
...(t.workspaceName ? { BB_WORKSPACE: t.workspaceName } : {}),
|
|
985
|
+
...(t.workspaceId ? { BB_WORKSPACE_ID: t.workspaceId } : {}),
|
|
986
|
+
...(workspaceIds ? { BB_WORKSPACE_IDS: workspaceIds } : {}),
|
|
987
|
+
};
|
|
988
|
+
return resolveRunEnv(t.effectiveEnvVars, core, this.logger);
|
|
989
|
+
}
|
|
990
|
+
/** Dispatches one claimed run; a failure terminalises it (BB-149). */
|
|
991
|
+
async dispatchOrFail(run) {
|
|
992
|
+
try {
|
|
993
|
+
await this.dispatch(run);
|
|
994
|
+
}
|
|
995
|
+
catch (err) {
|
|
996
|
+
// BB-149: terminalise the run to `failed` with the error captured,
|
|
997
|
+
// instead of leaving it `claimed` to expire after the ~300s lease (the
|
|
998
|
+
// old silent stall). `failed` is a terminal sibling of `expired`
|
|
999
|
+
// (dispatch/setup error vs lease lapse); it does NOT reopen the ticket
|
|
1000
|
+
// the way the BB-41-era fabricated `done` did, so there is no runaway.
|
|
1001
|
+
// Marking it now frees the ticket immediately and records *why* it
|
|
1002
|
+
// failed; a persistently-failing setup is then parked by the server-side
|
|
1003
|
+
// circuit breaker (TicketClaimService::assertNotCircuitBroken) after the
|
|
1004
|
+
// retry cap, not by a hot re-dispatch loop. Best-effort: a markFailed
|
|
1005
|
+
// that itself errors must not stop the claim loop, so it is swallowed
|
|
1006
|
+
// (the run then falls back to lease-expiry recovery).
|
|
1007
|
+
this.logger.error({ run: run.runUuid, err: String(err) }, 'dispatch failed');
|
|
1008
|
+
try {
|
|
1009
|
+
await this.remote.markFailed(run.runUuid, String(err));
|
|
1010
|
+
}
|
|
1011
|
+
catch (markErr) {
|
|
1012
|
+
this.logger.warn({ run: run.runUuid, err: String(markErr) }, 'markFailed failed');
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
async dispatch(run) {
|
|
1017
|
+
const t = await this.remote.getTicket(run.ticketUuid);
|
|
1018
|
+
// Resolve the ticket alias and machine defaults, or the project selection hook.
|
|
1019
|
+
// Persist the package id below so finalize can choose its transcript parser.
|
|
1020
|
+
const chosen = resolveAgent(this.agents, t, {
|
|
1021
|
+
selectAgent: this.config.selectAgent,
|
|
1022
|
+
routes: this.config.routes,
|
|
1023
|
+
defaultAgent: this.config.default_agent,
|
|
1024
|
+
warn: (message) => this.logger.warn({ ticket: t.identifier }, message),
|
|
1025
|
+
});
|
|
1026
|
+
const baseRef = t.baseBranch ? `origin/${t.baseBranch}` : undefined;
|
|
1027
|
+
// Resolve the run env once (per-ticket env_vars + core BB_* vars) and
|
|
1028
|
+
// inject it into BOTH the executor-owned lifecycle hooks AND the agent run
|
|
1029
|
+
// env (BB-99). Reserved core keys always win; values may be sensitive so
|
|
1030
|
+
// only key names are ever logged.
|
|
1031
|
+
const env = await this.runEnv(t, run.runUuid, run.stateAtStart);
|
|
1032
|
+
const ws = await this.serialGit(() => this.workspace.ensure(t.identifier, t.branchName, baseRef));
|
|
1033
|
+
// A claimable state has no project policy without a WORKFLOW.md. Warn and
|
|
1034
|
+
// proceed so the missing project instructions are observable without
|
|
1035
|
+
// wedging the run. `ws.instructions` is the loaded file (null when absent).
|
|
1036
|
+
if (!ws.instructions) {
|
|
1037
|
+
this.logger.warn({ ticket: t.identifier, workspace: ws.path }, 'WORKFLOW.md missing — no project instructions dispatched');
|
|
1038
|
+
}
|
|
1039
|
+
// BB-599: one listener context per dispatch. Its `env` IS the run env,
|
|
1040
|
+
// so what a listener adds reaches later listeners, the shell hooks and
|
|
1041
|
+
// `startRun`. From the first listener on, a sidecar may exist — so any
|
|
1042
|
+
// failure from here unwinds through `after_run` before it propagates.
|
|
1043
|
+
const ctx = {
|
|
1044
|
+
worktreePath: ws.path,
|
|
1045
|
+
ticket: t,
|
|
1046
|
+
state: run.stateAtStart,
|
|
1047
|
+
env,
|
|
1048
|
+
run: { uuid: run.runUuid, id: run.runId },
|
|
1049
|
+
};
|
|
1050
|
+
const launch = { started: false };
|
|
1051
|
+
try {
|
|
1052
|
+
await this.launch(run, t, chosen, ws, ctx, launch);
|
|
1053
|
+
}
|
|
1054
|
+
catch (err) {
|
|
1055
|
+
await this.unwindDispatch(ctx, launch.started);
|
|
1056
|
+
throw err;
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
/**
|
|
1060
|
+
* The part of a dispatch that can leave a sidecar or a worker behind
|
|
1061
|
+
* (BB-599): its caller unwinds it through `after_run` on any failure.
|
|
1062
|
+
*/
|
|
1063
|
+
async launch(run, t, chosen, ws, ctx, launch) {
|
|
1064
|
+
const env = ctx.env;
|
|
1065
|
+
// Lifecycle hooks are executor-owned + best-effort (BB-84): runHook never
|
|
1066
|
+
// throws, so neither call can abort dispatch or wedge a run in `claimed`.
|
|
1067
|
+
// `runSlot` runs the addon listeners first under the same rule (BB-599).
|
|
1068
|
+
// Both get the resolved run env so a setup/pre hook sees the same vars (e.g.
|
|
1069
|
+
// a DB DSN) the agent does (BB-99).
|
|
1070
|
+
//
|
|
1071
|
+
// BB-237 / BB-663: `after_create` runs when the worktree was just
|
|
1072
|
+
// created OR this worktree has no record of a successful `after_create`.
|
|
1073
|
+
// `created` alone answers "was this worktree born just now?", which is
|
|
1074
|
+
// not the question dispatch needs answered — a worktree whose bootstrap
|
|
1075
|
+
// aborted, an interrupted earlier run, or a hand-made `git worktree add`
|
|
1076
|
+
// all exist WITHOUT being provisioned, and under a creation-only gate
|
|
1077
|
+
// they never got another chance (BB-237). But the inverse bug is just
|
|
1078
|
+
// as real: a worktree whose `after_create` already succeeded got the
|
|
1079
|
+
// hook re-run on EVERY later dispatch, because the conductor tracked no
|
|
1080
|
+
// outcome at all — every workflow step re-ran the project bootstrap
|
|
1081
|
+
// (BB-663). The record lives beside the conductor's own config file,
|
|
1082
|
+
// keyed by worktree path (not inside the worktree — review F1 — and not
|
|
1083
|
+
// in conductor memory, so it survives a conductor restart). `created`
|
|
1084
|
+
// always wins over a leftover record: `reapOne` clears it on `after_done`
|
|
1085
|
+
// (review F2), so a worktree torn down and reopened bootstraps again too.
|
|
1086
|
+
const bootstrapStateFile = bootstrapStateFileOf(this.config);
|
|
1087
|
+
const bootstrapped = !ws.created && hasSucceededAfterCreate(bootstrapStateFile, ws.path);
|
|
1088
|
+
const provisioning = bootstrapped
|
|
1089
|
+
? { ok: true, ran: false, hook: 'after_create' }
|
|
1090
|
+
: await this.runSlot('after_create', ws.path, t.identifier, ctx, env);
|
|
1091
|
+
if (!bootstrapped) {
|
|
1092
|
+
recordAfterCreateOutcome(bootstrapStateFile, ws.path, provisioning.ok);
|
|
1093
|
+
}
|
|
1094
|
+
// AC-6: an agent is never dispatched into a worktree whose bootstrap failed
|
|
1095
|
+
// without that being visible. Dispatch is NOT withheld — runHook's
|
|
1096
|
+
// never-throw contract exists so a hook cannot wedge a run, and withholding
|
|
1097
|
+
// would re-introduce that wedge — so the failure is recorded against the run
|
|
1098
|
+
// instead (ticket + run + hook + workspace + err) and the run proceeds.
|
|
1099
|
+
// Boundedness stays with the existing circuit breaker.
|
|
1100
|
+
if (!provisioning.ok) {
|
|
1101
|
+
this.logger.error({
|
|
1102
|
+
hook: provisioning.hook,
|
|
1103
|
+
ticket: t.identifier,
|
|
1104
|
+
run: run.runUuid,
|
|
1105
|
+
workspace: ws.path,
|
|
1106
|
+
err: provisioning.error,
|
|
1107
|
+
}, 'lifecycle hook failed');
|
|
1108
|
+
}
|
|
1109
|
+
// BB-237: apply the workspace's open layout HERE — after `after_create`,
|
|
1110
|
+
// before the run starts. The layout's startup command names the bakers CLI
|
|
1111
|
+
// that `after_create` builds; applied inside `workspace.ensure()` (where it
|
|
1112
|
+
// used to live) it necessarily fired first, so a self-hosting checkout had
|
|
1113
|
+
// to paper over the ordering with a sentinel wait loop inside the pane
|
|
1114
|
+
// command. Ordering the two correctly removes the need for any handshake.
|
|
1115
|
+
//
|
|
1116
|
+
// Gated on `ws.created` — DELIBERATELY narrower than the `after_create` gate
|
|
1117
|
+
// right above it, which is convergent (a worktree can exist without ever
|
|
1118
|
+
// having been provisioned). Only a fresh create has an initial pane to lay
|
|
1119
|
+
// out; re-applying on reuse would start a duplicate editor on reattach.
|
|
1120
|
+
// Optional on the contract, so a workspace with no layout concept is a
|
|
1121
|
+
// no-op. Best-effort by contract: implementations swallow their own errors.
|
|
1122
|
+
if (ws.created) {
|
|
1123
|
+
await this.workspace.applyOpenLayout?.(ws.path, {
|
|
1124
|
+
identifier: t.identifier,
|
|
1125
|
+
branch: t.branchName,
|
|
1126
|
+
title: t.title,
|
|
1127
|
+
});
|
|
1128
|
+
}
|
|
1129
|
+
await this.runSlot('before_run', ws.path, t.identifier, ctx, env);
|
|
1130
|
+
// Clear a leftover tab for THIS run id in a reused workspace before
|
|
1131
|
+
// starting it (BB-183): run-scoped so a still-open sibling run's tab is
|
|
1132
|
+
// never killed. A fresh dispatch has no matching tab yet → a quiet no-op;
|
|
1133
|
+
// a re-dispatch of the same run drops its stale duplicate before startRun
|
|
1134
|
+
// re-creates it. Sibling prior-state run tabs are left to the worktree reap.
|
|
1135
|
+
await this.executor.cleanupRun(t.branchName, run.runId);
|
|
1136
|
+
const prompt = ws.instructions
|
|
1137
|
+
? renderPrompt(this.config.prompt, {
|
|
1138
|
+
identifier: t.identifier,
|
|
1139
|
+
state: t.state || 'qualification',
|
|
1140
|
+
runUuid: run.runUuid,
|
|
1141
|
+
})
|
|
1142
|
+
: '';
|
|
1143
|
+
// BB-473 review round 2 (Critical 1): pinning the run's own UUID
|
|
1144
|
+
// unconditionally makes a re-dispatch of the same run — an existing,
|
|
1145
|
+
// supported path (see the cleanupRun comment above) — fail to launch,
|
|
1146
|
+
// because `claude`/`grok --session-id` refuse a session id that already
|
|
1147
|
+
// exists.
|
|
1148
|
+
//
|
|
1149
|
+
// BB-473 review round 4 (Critical 2): a `startedAt` time window is not
|
|
1150
|
+
// an identity — it can still select a SIBLING run's transcript that
|
|
1151
|
+
// happens to be newer than the bound, in the very worktree-keyed
|
|
1152
|
+
// directory this ticket exists to stop scanning by recency. Walking this
|
|
1153
|
+
// run's deterministic attempt family (attempt 0 == the run's own UUID)
|
|
1154
|
+
// and pinning the first member whose transcript does not already exist
|
|
1155
|
+
// gives dispatch a collision-free id AND gives finalise something to
|
|
1156
|
+
// select by that nothing outside this run can collide with — no
|
|
1157
|
+
// `sessionExists?` on the agent means every candidate reads as "free", so
|
|
1158
|
+
// an agent without the capability keeps pinning attempt 0 exactly as
|
|
1159
|
+
// before.
|
|
1160
|
+
const family = attemptFamily(run.runUuid);
|
|
1161
|
+
let pinnedSessionId;
|
|
1162
|
+
for (const candidate of family) {
|
|
1163
|
+
const exists = (await chosen.agent.sessionExists?.(ws.path, candidate)) ?? false;
|
|
1164
|
+
if (!exists) {
|
|
1165
|
+
pinnedSessionId = candidate;
|
|
1166
|
+
break;
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
const identity = pinnedSessionId
|
|
1170
|
+
? { sessionId: pinnedSessionId }
|
|
1171
|
+
: undefined;
|
|
1172
|
+
// BB-473 round 4 (review round 3, Critical 1): captured BEFORE
|
|
1173
|
+
// `startRun` so it precedes any transcript the agent is about to write —
|
|
1174
|
+
// re-stamped into `bb_run.started_at` on every dispatch, including a
|
|
1175
|
+
// re-dispatch, as a tie-breaker for agents with no session mechanism at
|
|
1176
|
+
// all (codex, opencode). Sent as whole seconds, matching every other
|
|
1177
|
+
// timestamp this method writes (`heartbeat`, `closed_date`) — a
|
|
1178
|
+
// fractional-second ISO string is not one of the Drupal `timestamp`
|
|
1179
|
+
// field's accepted formats and the PATCH would be rejected (review round
|
|
1180
|
+
// 4, Critical 1).
|
|
1181
|
+
const dispatchedAt = Math.floor(Date.now() / 1000);
|
|
1182
|
+
const warnings = chosen.agent.launchWarnings;
|
|
1183
|
+
if (warnings?.length) {
|
|
1184
|
+
const alias = chosen.alias ?? chosen.id;
|
|
1185
|
+
for (const warning of warnings) {
|
|
1186
|
+
this.logger.info({
|
|
1187
|
+
ticket: t.identifier,
|
|
1188
|
+
run: run.runUuid,
|
|
1189
|
+
harness: chosen.agent.id,
|
|
1190
|
+
alias,
|
|
1191
|
+
}, `${chosen.agent.id}/${alias}: ${warning}`);
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
await this.executor.startRun({
|
|
1195
|
+
ticket: {
|
|
1196
|
+
uuid: t.uuid,
|
|
1197
|
+
identifier: t.identifier,
|
|
1198
|
+
title: t.title,
|
|
1199
|
+
branchName: t.branchName,
|
|
1200
|
+
state: t.state,
|
|
1201
|
+
...(t.issueUrl ? { url: t.issueUrl } : {}),
|
|
1202
|
+
},
|
|
1203
|
+
run: { uuid: run.runUuid, id: run.runId, handler: run.handler },
|
|
1204
|
+
workspacePath: ws.path,
|
|
1205
|
+
instructions: ws.instructions,
|
|
1206
|
+
command: chosen.agent.launchCommand(prompt, identity),
|
|
1207
|
+
env,
|
|
1208
|
+
});
|
|
1209
|
+
launch.started = true;
|
|
1210
|
+
this.logger.info({
|
|
1211
|
+
ticket: t.identifier,
|
|
1212
|
+
run: run.runUuid,
|
|
1213
|
+
state: t.state,
|
|
1214
|
+
workspace: ws.path,
|
|
1215
|
+
// Key names only — env values may be sensitive (BB-99).
|
|
1216
|
+
envKeys: Object.keys(env),
|
|
1217
|
+
}, 'dispatched run');
|
|
1218
|
+
// The conductor — not the agent — knows the per-run git worktree path, so
|
|
1219
|
+
// it persists `worktree_path` to bb_run here alongside the running state.
|
|
1220
|
+
// It also persists the chosen agent id (BB-144) for finalize footprint
|
|
1221
|
+
// routing.
|
|
1222
|
+
await this.remote.markRunning(run.runUuid, {
|
|
1223
|
+
worktree_path: ws.path,
|
|
1224
|
+
agent: chosen.id,
|
|
1225
|
+
startedAt: dispatchedAt,
|
|
1226
|
+
});
|
|
1227
|
+
}
|
|
1228
|
+
/**
|
|
1229
|
+
* Stop what a failed dispatch started (BB-599): the worker if it was
|
|
1230
|
+
* launched, then the `after_run` listeners and shell hook. Best-effort — it
|
|
1231
|
+
* never replaces the dispatch error the caller re-throws.
|
|
1232
|
+
*/
|
|
1233
|
+
async unwindDispatch(ctx, launched) {
|
|
1234
|
+
const runUuid = ctx.run?.uuid ?? '';
|
|
1235
|
+
try {
|
|
1236
|
+
if (launched && this.executor.capabilities().persistent) {
|
|
1237
|
+
try {
|
|
1238
|
+
await this.executor.stopRun(ctx.ticket.branchName);
|
|
1239
|
+
}
|
|
1240
|
+
catch (err) {
|
|
1241
|
+
this.logger.warn({ run: runUuid, err: String(err) }, 'agent stop failed');
|
|
1242
|
+
}
|
|
1243
|
+
await this.closeRunTab(ctx.ticket.branchName, runUuid, ctx.run?.id ?? 0);
|
|
1244
|
+
}
|
|
1245
|
+
await this.runSlot('after_run', ctx.worktreePath, ctx.ticket.identifier, ctx, ctx.env);
|
|
1246
|
+
}
|
|
1247
|
+
catch (err) {
|
|
1248
|
+
this.logger.warn({ run: runUuid, err: String(err) }, 'dispatch cleanup failed');
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
async serve(signal) {
|
|
1252
|
+
await this.pollLoop(signal);
|
|
1253
|
+
}
|
|
1254
|
+
/**
|
|
1255
|
+
* True once this conductor's own checkout has been removed from disk (its
|
|
1256
|
+
* worktree was reaped while the process kept running). Such a conductor is a
|
|
1257
|
+
* pure liability: it still heartbeats and still claims runs, but every
|
|
1258
|
+
* dispatch fails — no git command and no relative path can resolve from a
|
|
1259
|
+
* deleted directory — so each claim burns one attempt and three of them park
|
|
1260
|
+
* the ticket behind the circuit breaker. Observed as 30 orphans claiming and
|
|
1261
|
+
* failing tickets they could never dispatch.
|
|
1262
|
+
*/
|
|
1263
|
+
checkoutGone() {
|
|
1264
|
+
return !existsSync(this.checkoutRoot);
|
|
1265
|
+
}
|
|
1266
|
+
async pollLoop(signal) {
|
|
1267
|
+
while (!signal?.aborted) {
|
|
1268
|
+
if (this.checkoutGone()) {
|
|
1269
|
+
// Stop claiming and let the loop end — the process exits and the cron
|
|
1270
|
+
// reaper flips status to offline on lease expiry, same as any crash.
|
|
1271
|
+
// Deliberately NOT an offline write from here: the loop never owns that
|
|
1272
|
+
// transition (see the note at the end of this method).
|
|
1273
|
+
this.logger.error({ conductorId: this.id, workspace: this.checkoutRoot }, 'checkout is gone — stopping conductor instead of claiming runs it cannot dispatch');
|
|
1274
|
+
await this.queue.drain();
|
|
1275
|
+
return;
|
|
1276
|
+
}
|
|
1277
|
+
// Only produces: the loop never waits for the actions it enqueued.
|
|
1278
|
+
try {
|
|
1279
|
+
await this.produce();
|
|
1280
|
+
}
|
|
1281
|
+
catch (err) {
|
|
1282
|
+
this.logger.error({ conductorId: this.id, err: String(err) }, 'tick failed');
|
|
1283
|
+
}
|
|
1284
|
+
await sleep(this.config.poll_interval_ms, signal);
|
|
1285
|
+
}
|
|
1286
|
+
await this.queue.drain();
|
|
1287
|
+
// NB: the poll loop never writes status=offline itself. A graceful stop
|
|
1288
|
+
// (`bakers oven stop`) writes offline directly server-side; a crash or
|
|
1289
|
+
// hard-kill leaves a stale lease that the Drupal cron reaper flips to
|
|
1290
|
+
// offline (bb_core cron → ConductorReaper). Liveness is thus always
|
|
1291
|
+
// derived from status + last_seen/lease_expires_at, never from the loop.
|
|
1292
|
+
}
|
|
1293
|
+
}
|