@bakers-build/conductor 0.0.0-stage → 0.1.0-next.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +4 -2
  3. package/dist/src/bakers-project-context.d.ts +12 -0
  4. package/dist/src/bakers-project-context.js +59 -0
  5. package/dist/src/cli/config-plan.d.ts +35 -0
  6. package/dist/src/cli/config-plan.js +1264 -0
  7. package/dist/src/cli/config-schema.d.ts +137 -0
  8. package/dist/src/cli/config-schema.js +239 -0
  9. package/dist/src/cli/config-source-edit.d.ts +17 -0
  10. package/dist/src/cli/config-source-edit.js +95 -0
  11. package/dist/src/cli/config-templates.d.ts +43 -0
  12. package/dist/src/cli/config-templates.js +130 -0
  13. package/dist/src/cli/config-validation.d.ts +12 -0
  14. package/dist/src/cli/config-validation.js +353 -0
  15. package/dist/src/cli/migrate-addon-names.d.ts +82 -0
  16. package/dist/src/cli/migrate-addon-names.js +334 -0
  17. package/dist/src/cli/project-context.d.ts +21 -0
  18. package/dist/src/cli/project-context.js +105 -0
  19. package/dist/src/commands/conductor.d.ts +91 -0
  20. package/dist/src/commands/conductor.js +843 -0
  21. package/dist/src/config.d.ts +109 -0
  22. package/dist/src/config.js +712 -0
  23. package/dist/src/contract.d.ts +13 -0
  24. package/dist/src/contract.js +19 -0
  25. package/dist/src/core/action-queue.d.ts +31 -0
  26. package/dist/src/core/action-queue.js +69 -0
  27. package/dist/src/core/attempt.d.ts +25 -0
  28. package/dist/src/core/attempt.js +41 -0
  29. package/dist/src/core/conductor.d.ts +236 -0
  30. package/dist/src/core/conductor.js +1293 -0
  31. package/dist/src/core/host-key.d.ts +19 -0
  32. package/dist/src/core/host-key.js +27 -0
  33. package/dist/src/core/inbox-runtime.d.ts +48 -0
  34. package/dist/src/core/inbox-runtime.js +180 -0
  35. package/dist/src/core/mr-changes.d.ts +18 -0
  36. package/dist/src/core/mr-changes.js +47 -0
  37. package/dist/src/index.d.ts +8 -0
  38. package/dist/src/index.js +27 -0
  39. package/dist/src/plugins/agent-args.d.ts +43 -0
  40. package/dist/src/plugins/agent-args.js +221 -0
  41. package/dist/src/plugins/agent.d.ts +121 -0
  42. package/dist/src/plugins/agent.js +12 -0
  43. package/dist/src/plugins/executor.d.ts +182 -0
  44. package/dist/src/plugins/executor.js +1 -0
  45. package/dist/src/plugins/message.d.ts +61 -0
  46. package/dist/src/plugins/message.js +47 -0
  47. package/dist/src/plugins/plugins.d.ts +96 -0
  48. package/dist/src/plugins/plugins.js +120 -0
  49. package/dist/src/plugins/preset.d.ts +61 -0
  50. package/dist/src/plugins/preset.js +23 -0
  51. package/dist/src/plugins/process-host.d.ts +7 -0
  52. package/dist/src/plugins/process-host.js +1 -0
  53. package/dist/src/plugins/remote.d.ts +420 -0
  54. package/dist/src/plugins/remote.js +1 -0
  55. package/dist/src/plugins/workspace.d.ts +69 -0
  56. package/dist/src/plugins/workspace.js +1 -0
  57. package/dist/src/preset.d.ts +2 -0
  58. package/dist/src/preset.js +8 -0
  59. package/dist/src/types.d.ts +132 -0
  60. package/dist/src/types.js +6 -0
  61. package/package.json +38 -4
@@ -0,0 +1,1293 @@
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
+ import { basename, dirname, join } from 'node:path';
3
+ import { stemForConfigFile } from '@bakers-build/core';
4
+ import { resolveAgent } from '../plugins/plugins.js';
5
+ import { ActionQueue } from './action-queue.js';
6
+ import { attemptFamily } from './attempt.js';
7
+ import { hostKeyOf } from './host-key.js';
8
+ import { InboxRuntime, runInbox } from './inbox-runtime.js';
9
+ import { measureMrChanges, mrChangesRows } from './mr-changes.js';
10
+ function sleep(ms, signal) {
11
+ return new Promise((resolve) => {
12
+ if (signal?.aborted) {
13
+ resolve();
14
+ return;
15
+ }
16
+ const timer = setTimeout(() => {
17
+ signal?.removeEventListener('abort', onAbort);
18
+ resolve();
19
+ }, ms);
20
+ const onAbort = () => {
21
+ clearTimeout(timer);
22
+ resolve();
23
+ };
24
+ signal?.addEventListener('abort', onAbort, { once: true });
25
+ });
26
+ }
27
+ /**
28
+ * A run-env key is reserved when it names a conductor-provided core variable:
29
+ * any `BB_*`, `WORKSPACE_ROOT`, or `TICKET_URL`. User `env_vars` may not
30
+ * clobber these — the core vars always win (BB-99).
31
+ */
32
+ function isReservedEnvKey(key) {
33
+ return (key.startsWith('BB_') || key === 'WORKSPACE_ROOT' || key === 'TICKET_URL');
34
+ }
35
+ /** A portable environment variable name. */
36
+ const ENV_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
37
+ /**
38
+ * What a failed listener is logged as: the error's class name, never its
39
+ * message — listener text may carry an env value (BB-599).
40
+ */
41
+ function errorKind(err) {
42
+ return err instanceof Error ? err.name : typeof err;
43
+ }
44
+ /**
45
+ * Parse the server-canonicalised effective env_vars (`KEY=value` lines, one per
46
+ * line) into a map. The server already stripped comments/blank/malformed lines,
47
+ * so this is deliberately minimal: split on the first `=`, keep a valid key.
48
+ * Defensive against stray blank lines only — never throws.
49
+ */
50
+ function parseEnvLines(text) {
51
+ const out = {};
52
+ for (const raw of text.split('\n')) {
53
+ const line = raw.trim();
54
+ if (line === '') {
55
+ continue;
56
+ }
57
+ const eq = line.indexOf('=');
58
+ if (eq <= 0) {
59
+ continue;
60
+ }
61
+ const key = line.slice(0, eq);
62
+ if (!ENV_KEY.test(key)) {
63
+ continue;
64
+ }
65
+ out[key] = line.slice(eq + 1);
66
+ }
67
+ return out;
68
+ }
69
+ /**
70
+ * Resolve the environment a run executes in (BB-99): parse the ticket's
71
+ * effective env_vars, drop any reserved key (loud warn — key NAME only, never
72
+ * the value), then overlay the conductor's core vars so they always win.
73
+ *
74
+ * Values are potentially secret (e.g. a DB DSN): this function logs key names
75
+ * only, never a value. The returned map is injected into BOTH the agent run env
76
+ * and every workspace hook.
77
+ */
78
+ export function resolveRunEnv(effectiveEnvVars, core, logger) {
79
+ const user = effectiveEnvVars ? parseEnvLines(effectiveEnvVars) : {};
80
+ const clobbered = [];
81
+ const safe = {};
82
+ for (const [key, value] of Object.entries(user)) {
83
+ if (isReservedEnvKey(key)) {
84
+ clobbered.push(key);
85
+ continue;
86
+ }
87
+ safe[key] = value;
88
+ }
89
+ if (clobbered.length > 0) {
90
+ logger.warn({ keys: clobbered }, 'ignored reserved keys in ticket env_vars (core vars win)');
91
+ }
92
+ // Core vars overlay last — they can never be overridden.
93
+ return { ...safe, ...core };
94
+ }
95
+ /**
96
+ * Where a conductor keeps its own record of which worktrees have a
97
+ * successfully completed `after_create` (BB-663), beside its config file —
98
+ * same pattern as `frozenStateFile` — rather than inside the worktree. An
99
+ * in-tree marker gets picked up by a stray `git add -A`/commit (review F1):
100
+ * once merged, every checkout of that commit then LOOKS bootstrapped and
101
+ * silently skips `after_create` forever. Keyed by worktree path, not by
102
+ * conductor instance, so it survives both a conductor restart and a fresh
103
+ * `Conductor` object dispatching into the same path.
104
+ */
105
+ function bootstrapStateFileOf(config) {
106
+ const file = config.config_path;
107
+ return join(dirname(file), `${basename(file, '.js')}.bootstrapped.json`);
108
+ }
109
+ function loadBootstrapState(file) {
110
+ if (!existsSync(file))
111
+ return {};
112
+ try {
113
+ return JSON.parse(readFileSync(file, 'utf8'));
114
+ }
115
+ catch {
116
+ return {};
117
+ }
118
+ }
119
+ function hasSucceededAfterCreate(file, worktreePath) {
120
+ return loadBootstrapState(file)[worktreePath] === true;
121
+ }
122
+ /** Best-effort, like `saveFrozenSeen`: an unwritable state file must not wedge dispatch. */
123
+ function recordAfterCreateOutcome(file, worktreePath, ok) {
124
+ try {
125
+ const state = loadBootstrapState(file);
126
+ if (ok)
127
+ state[worktreePath] = true;
128
+ else
129
+ delete state[worktreePath];
130
+ mkdirSync(dirname(file), { recursive: true });
131
+ writeFileSync(file, `${JSON.stringify(state)}\n`);
132
+ }
133
+ catch {
134
+ // best-effort: a later dispatch simply re-runs after_create.
135
+ }
136
+ }
137
+ function renderPrompt(template, values) {
138
+ return template
139
+ .replaceAll('{identifier}', values.identifier)
140
+ .replaceAll('{state}', values.state)
141
+ .replaceAll('{runUuid}', values.runUuid);
142
+ }
143
+ export class Conductor {
144
+ config;
145
+ remote;
146
+ executor;
147
+ workspace;
148
+ agents;
149
+ logger;
150
+ checkoutRoot;
151
+ frozenStateFile;
152
+ uuid = null;
153
+ /**
154
+ * Frozen tickets seen on the last poll, by uuid, with the worktree their
155
+ * hook ran in. Mirrored to `frozenStateFile` when one is given, so an
156
+ * unfreeze while the conductor is down still gets its `on_resume` and a
157
+ * restart does not repeat `on_freeze`; loaded on the first observation.
158
+ */
159
+ frozenSeen = null;
160
+ /** The addons with contract-2 inbox messages this conductor runs. */
161
+ inboxAddons;
162
+ /** The per-item work each tick enqueues. */
163
+ queue;
164
+ /** Serializes git worktree add/remove: parallel ones fight over git locks. */
165
+ gitLock = Promise.resolve();
166
+ constructor(config, remote, executor, workspace, agents, logger, checkoutRoot = process.cwd(), frozenStateFile) {
167
+ this.config = config;
168
+ this.remote = remote;
169
+ this.executor = executor;
170
+ this.workspace = workspace;
171
+ this.agents = agents;
172
+ this.logger = logger;
173
+ this.checkoutRoot = checkoutRoot;
174
+ this.frozenStateFile = frozenStateFile;
175
+ this.inboxAddons = new InboxRuntime(config.inboxAddons ?? [], remote, logger, (ticket) => this.runEnv(ticket, undefined, ticket.state));
176
+ this.queue = new ActionQueue(logger);
177
+ }
178
+ serialGit(fn) {
179
+ const result = this.gitLock.then(fn);
180
+ this.gitLock = result.catch(() => { });
181
+ return result;
182
+ }
183
+ /**
184
+ * `conductor_id` is required and the loader throws without it (BB-353), so
185
+ * there is no directory-derived fallback here (BB-450 D5) — that fallback
186
+ * was the mechanism by which one checkout came to be registered under two
187
+ * different ids.
188
+ */
189
+ get id() {
190
+ const id = this.config.conductor_id ?? this.config.machine_id;
191
+ if (id === undefined || id.trim() === '') {
192
+ throw new Error('conductor config has no conductor_id');
193
+ }
194
+ return id;
195
+ }
196
+ /**
197
+ * Host stem for bb_machine.host_key. Prefer config.host_key from
198
+ * MachineContext.machine_id; fall back to a best-effort parse of the composed
199
+ * conductor_id only when the loader could not supply the stem.
200
+ */
201
+ get hostKey() {
202
+ return hostKeyOf(this.config);
203
+ }
204
+ /**
205
+ * Whether this instance is the serve-loop process itself (BB-232).
206
+ *
207
+ * Off by default so that only a caller which genuinely owns the process can
208
+ * report a pid — see {@link claimServeProcess}.
209
+ */
210
+ ownsServeProcess = false;
211
+ /**
212
+ * Declares that this instance runs *as* the conductor process, so its
213
+ * registrations may carry the process identity a cockpit needs to stop it.
214
+ *
215
+ * Only `cmdStartForeground` calls this. `cmdPoll` and `cmdReap` also build a
216
+ * Conductor and call start() + tick(), then exit; if they reported a pid,
217
+ * every `bakers oven once` would overwrite the running conductor's pid
218
+ * with its own short-lived one and the next stop would signal a dead — or
219
+ * recycled — process. Making the report opt-in means those two commands
220
+ * cannot do so even by accident.
221
+ */
222
+ claimServeProcess() {
223
+ this.ownsServeProcess = true;
224
+ }
225
+ /** This conductor's registration payload, built from config. */
226
+ registration() {
227
+ const base = {
228
+ id: this.id,
229
+ host_key: this.hostKey,
230
+ project: this.config.project,
231
+ states: this.config.states,
232
+ workspace: this.checkoutRoot,
233
+ label: this.config.label,
234
+ max_parallel: this.config.max_parallel,
235
+ };
236
+ if (!this.ownsServeProcess)
237
+ return base;
238
+ return {
239
+ ...base,
240
+ last_pid: process.pid,
241
+ ...(this.conductorName ? { conductor_name: this.conductorName } : {}),
242
+ };
243
+ }
244
+ /**
245
+ * The stem of the config file this conductor was loaded from.
246
+ *
247
+ * Derived through core's own mapping rather than by slicing the filename
248
+ * here, so the CLI and the conductor cannot disagree about which files count
249
+ * as a conductor config.
250
+ */
251
+ get conductorName() {
252
+ const path = this.config.config_path;
253
+ return path ? stemForConfigFile(basename(path)) : undefined;
254
+ }
255
+ async start() {
256
+ this.uuid = await this.remote.registerConductor(this.registration());
257
+ this.logger.info({ conductorId: this.id, uuid: this.uuid, project: this.config.project }, 'conductor started');
258
+ try {
259
+ await this.inboxAddons.register(this.id);
260
+ }
261
+ catch (err) {
262
+ // Without addons nothing depends on it (and an older server has no
263
+ // registration resource); with addons the tick retries it.
264
+ const log = this.inboxAddons.empty ? this.logger.warn : this.logger.error;
265
+ log.call(this.logger, { conductorId: this.id, err: String(err) }, 'addon registration failed');
266
+ }
267
+ }
268
+ /** One tick that also waits for its actions: `bakers oven once`, tests. */
269
+ async tick() {
270
+ await this.produce();
271
+ await this.queue.drain();
272
+ }
273
+ /** A failing list query must not stop the other steps or the claims. */
274
+ async step(name, fn) {
275
+ try {
276
+ await fn();
277
+ }
278
+ catch (err) {
279
+ this.logger.warn({ conductorId: this.id, err: String(err) }, `${name} query failed`);
280
+ }
281
+ }
282
+ /**
283
+ * The tick body: heartbeat, the list queries and the claim loop. Every
284
+ * per-item job becomes an action on the queue, so no slow item can delay
285
+ * the heartbeat or the claims.
286
+ */
287
+ async produce() {
288
+ if (!this.uuid)
289
+ throw new Error('not started');
290
+ let count = await this.remote.activeRunCount(this.id);
291
+ // A heartbeat is the server-side self-heal: it upserts by conductor_id (so
292
+ // it never 404s and recreates a vanished registration) and refreshes the
293
+ // lease — no client-side re-register, no separate status read. It also
294
+ // recomputes bb_machine.current_load (peers + this process's load).
295
+ await this.remote.heartbeat(this.registration(), count, this.config.lease_seconds);
296
+ // BB-353 D5: gate claims on the shared host budget, not only this
297
+ // process's active runs. After the heartbeat above, machine.current_load
298
+ // includes peers + self; each claim below increments the local view so
299
+ // pending dispatches count without a second round-trip.
300
+ const capacity = await this.remote.machineCapacity(this.hostKey);
301
+ let machineLoad = capacity?.currentLoad ?? count;
302
+ // BB-464 D3: gate on the MACHINE's own budget, not this conductor's
303
+ // config — the machine row is the one authority a shared host agrees on;
304
+ // falling back to config.max_parallel only when the row is unreadable.
305
+ const maxParallel = capacity?.maxParallel ?? this.config.max_parallel;
306
+ this.logger.info({
307
+ conductorId: this.id,
308
+ active: count,
309
+ machineLoad,
310
+ capacity: maxParallel,
311
+ }, 'conductor tick');
312
+ if (machineLoad > maxParallel) {
313
+ // BB-464 D4: a breach is its own signal, not merely the reason no
314
+ // further claim was made — this server-side ceiling (TicketClaimService)
315
+ // is the actual guard; this is visibility into how it was reached.
316
+ this.logger.warn({ conductorId: this.id, machineLoad, capacity: maxParallel }, 'machine over its shared host budget');
317
+ }
318
+ // A hung action blocks only its ticket's lane; make it visible.
319
+ this.queue.warnOverdue(this.config.lease_seconds * 1000);
320
+ await this.step('finalize', () => this.enqueueFinalize());
321
+ await this.step('reap', () => this.enqueueReap());
322
+ await this.step('freeze', () => this.observeFreezes());
323
+ await runInbox(this.remote, this.inboxAddons, this.id, this.logger, this.queue);
324
+ let claimed = 0;
325
+ let withheld = false;
326
+ while (machineLoad < maxParallel) {
327
+ const run = await this.remote.claimNext({
328
+ leaseSeconds: this.config.lease_seconds,
329
+ conductorId: this.id,
330
+ });
331
+ if (!run)
332
+ break;
333
+ if ('reason' in run) {
334
+ withheld = true;
335
+ this.logger.info({ conductorId: this.id, ...run, reportedLoad: machineLoad }, 'claim withheld: machine at capacity');
336
+ break;
337
+ }
338
+ this.logger.info({ conductorId: this.id, run: run.runUuid, ticket: run.ticketUuid }, 'claimed run');
339
+ this.queue.enqueue({
340
+ key: `dispatch:${run.runUuid}`,
341
+ lane: run.ticketUuid,
342
+ run: () => this.dispatchOrFail(run),
343
+ });
344
+ count += 1;
345
+ machineLoad += 1;
346
+ claimed += 1;
347
+ }
348
+ if (claimed > 0) {
349
+ // BB-464 AC 2: push this tick's claims into the machine row's load
350
+ // immediately, so a peer conductor ticking next reads the up-to-date
351
+ // figure instead of the pre-claim snapshot this tick started from. The
352
+ // staleness window this closes is what let sequential ticks reach the
353
+ // sum of both conductors' budgets rather than the shared ceiling.
354
+ await this.remote.heartbeat(this.registration(), count, this.config.lease_seconds);
355
+ }
356
+ if (claimed === 0 && !withheld) {
357
+ this.logger.info({
358
+ conductorId: this.id,
359
+ active: count,
360
+ machineLoad,
361
+ capacity: maxParallel,
362
+ }, machineLoad >= maxParallel
363
+ ? 'at capacity, nothing claimed'
364
+ : 'idle, nothing to claim');
365
+ }
366
+ }
367
+ /**
368
+ * One-shot finalisation: for each of this conductor's runs that is state in
369
+ * (done, cancelled) but not yet closed, read the agent transcript from its
370
+ * worktree and write it as the final log, then mark the run closed. Keyed off
371
+ * the durable terminal-state query — no in-memory state, so a restarted
372
+ * conductor still finalises any unclosed run on its next tick.
373
+ */
374
+ async enqueueFinalize() {
375
+ const runs = await this.remote.fetchFinalizableRuns(this.id);
376
+ for (const r of runs) {
377
+ this.queue.enqueue({
378
+ key: `finalize:${r.runUuid}`,
379
+ lane: r.ticketUuid || r.runUuid,
380
+ run: () => this.finalizeOne(r),
381
+ });
382
+ }
383
+ }
384
+ /** Finalises one run; a failure is logged and retried next tick. */
385
+ async finalizeOne(r) {
386
+ try {
387
+ // Signal the agent to stop before capturing its log. Best-effort and
388
+ // gated on the persistent capability (only hosted executors have a live
389
+ // agent) — mirrors removeWorktree's gate. For herdr this is a no-op
390
+ // (the tab-close in closeRunTab kills the PTY); a stop failure must
391
+ // never block finalisation.
392
+ const branch = this.executor.capabilities().persistent
393
+ ? await this.remote.getRunTicketBranchName(r.runUuid)
394
+ : '';
395
+ if (branch) {
396
+ try {
397
+ await this.executor.stopRun(branch);
398
+ }
399
+ catch (err) {
400
+ this.logger.warn({ run: r.runUuid, err: String(err) }, 'agent stop failed');
401
+ }
402
+ }
403
+ // BB-599: a failed capture must not skip the worker and sidecar
404
+ // cleanup below, so it is held and re-thrown before the close write —
405
+ // the run then stays unclosed and the next tick retries all of it.
406
+ let captured;
407
+ let captureError;
408
+ try {
409
+ captured = await this.captureRun(r);
410
+ }
411
+ catch (err) {
412
+ captureError = err;
413
+ }
414
+ // The transcript is captured, so killing the PTY is safe (BB-171).
415
+ if (branch)
416
+ await this.closeRunTab(branch, r.runUuid, r.runId);
417
+ // BB-599: `after_run` fires at run end — after the worker is gone,
418
+ // before the durable close. A failed close write repeats it next tick,
419
+ // which is why listeners must be idempotent.
420
+ await this.runFinalizeHook(r);
421
+ if (!captured)
422
+ throw captureError;
423
+ // `?? true` is the default: an unset flag, and every config written
424
+ // before this setting existed, keep uploading exactly as before.
425
+ // `finalizeRun()` already spreads `...(log ? { log } : {})`, so an
426
+ // empty string omits the attribute from the PATCH entirely — no
427
+ // server change is needed and no empty log is written.
428
+ const uploadedLog = (this.config.capture_logs ?? true) ? captured.log : '';
429
+ await this.remote.finalizeRun(r.runUuid, uploadedLog, captured.metrics);
430
+ this.logger.info({ run: r.runUuid, footprint: captured.metrics }, 'run finalised');
431
+ }
432
+ catch (err) {
433
+ this.logger.warn({ run: r.runUuid, err: String(err) }, 'run finalise failed');
434
+ }
435
+ }
436
+ /** Read the run's transcript and parse its footprint for the close write. */
437
+ async captureRun(r) {
438
+ // Route to the agent that actually ran (BB-144): the persisted
439
+ // run.agent id, falling back to the first candidate for a run written
440
+ // before the field existed (empty id).
441
+ const resolved = this.agents.find((a) => a.id === r.agent) ?? this.agents[0];
442
+ if (!resolved) {
443
+ throw new Error('no agent candidates configured for finalize');
444
+ }
445
+ // Scoped to this run's own identity (BB-473): the run's UUID as a
446
+ // session id, its deterministic attempt family (review round 4,
447
+ // Critical 2) so a re-dispatch resolves by WHICH attempt rather than
448
+ // by a time window a sibling session could still beat, plus its
449
+ // recorded start as a lower-bound tie-breaker for agents with no
450
+ // session mechanism at all — so a foreign transcript left behind by
451
+ // a resumed/re-dispatched/reused worktree can never be finalised
452
+ // here.
453
+ const log = r.worktreePath
454
+ ? await resolved.agent.getRunLog(r.worktreePath, {
455
+ sessionId: r.runUuid,
456
+ attemptFamily: attemptFamily(r.runUuid),
457
+ ...(r.startedAt ? { startedAt: r.startedAt } : {}),
458
+ })
459
+ : '';
460
+ // Footprint (BB-132): the agent parses its own transcript for the
461
+ // effort metrics AND its wall-clock duration_s (from the transcript's
462
+ // first→last timestamps — BB-151). Deriving duration from the log
463
+ // keeps it a single source with one format, avoiding the earlier
464
+ // `started_at` re-read that mis-parsed the JSON:API ISO string to 0.
465
+ // The conductor owns the run, so this write always lands (the agent
466
+ // session could not).
467
+ const parsed = resolved.agent.parseFootprint(log);
468
+ // Runtime provenance beside the footprint: the harness that drove the
469
+ // session, the model that actually ran, and the exact model id the
470
+ // transcript records. All three ride the SAME close PATCH — no extra
471
+ // write path, so the conductor-owner field access that already covers
472
+ // the metrics covers these too.
473
+ //
474
+ // `model` and `model_version` each prefer an explicit configured pin
475
+ // (`resolved.model` / `resolved.modelVersion`) when one is set — a
476
+ // human's own answer beats a guess. Absent a pin, each falls back to
477
+ // the raw, unmodified model id the transcript names (`parsed.model`)
478
+ // rather than staying empty: that is the real model that ran, not a
479
+ // fabricated placeholder. No per-harness splitting or abbreviation
480
+ // happens on either source, which is what keeps the columns
481
+ // comparable across harnesses. Each key is spread conditionally: an
482
+ // unresolved value sends no key at all.
483
+ const metrics = {
484
+ tokens: parsed.tokens,
485
+ duration_s: parsed.duration_s,
486
+ agent_turns: parsed.agent_turns,
487
+ tool_calls: parsed.tool_calls,
488
+ user_prompts: parsed.user_prompts,
489
+ user_prompt_words: parsed.user_prompt_words,
490
+ active_s: parsed.active_s,
491
+ harness: resolved.id,
492
+ ...(resolved.model !== undefined
493
+ ? { model: resolved.model }
494
+ : parsed.model !== undefined
495
+ ? { model: parsed.model }
496
+ : {}),
497
+ ...(resolved.modelVersion !== undefined
498
+ ? { model_version: resolved.modelVersion }
499
+ : parsed.model !== undefined
500
+ ? { model_version: parsed.model }
501
+ : {}),
502
+ };
503
+ return { log, metrics };
504
+ }
505
+ /**
506
+ * Close one run's tab. Run-scoped (BB-183): only the `#<runId>` tab, never
507
+ * a sibling run's. Best-effort: a close failure never mislabels the run as
508
+ * "finalise failed" or replaces a dispatch error.
509
+ */
510
+ async closeRunTab(branch, runUuid, runId) {
511
+ try {
512
+ await this.executor.cleanupRun(branch, runId);
513
+ }
514
+ catch (err) {
515
+ this.logger.warn({ run: runUuid, err: String(err) }, 'run tab cleanup failed');
516
+ }
517
+ }
518
+ /**
519
+ * `after_run` for an ended run (BB-599), rebuilt from durable state: the
520
+ * run record names its ticket and start state, and the env is resolved from
521
+ * the CURRENT ticket — so a restarted conductor stops the same sidecars.
522
+ * Like `after_done`, it needs the worktree directory to exist.
523
+ */
524
+ async runFinalizeHook(r) {
525
+ if (!r.worktreePath || !existsSync(r.worktreePath))
526
+ return;
527
+ const ctx = await this.listenerContext(r.ticketUuid, r.worktreePath, {
528
+ hook: 'after_run',
529
+ run: { uuid: r.runUuid, id: r.runId },
530
+ ...(r.stateAtStart ? { state: r.stateAtStart } : {}),
531
+ });
532
+ await this.runSlot('after_run', r.worktreePath, ctx?.ticket.identifier || r.ticketUuid || r.runUuid, ctx, ctx?.env);
533
+ }
534
+ /**
535
+ * Rebuild a listener context from the ticket's current record, or
536
+ * `undefined` (logged) when it cannot be read — the caller still runs the
537
+ * project shell hook, it only loses the addon listeners. Built even with no
538
+ * listeners: the shell `after_run` takes its env from it.
539
+ */
540
+ async listenerContext(ticketUuid, worktreePath, opts) {
541
+ try {
542
+ if (!ticketUuid)
543
+ throw new Error('run has no ticket');
544
+ const ticket = await this.remote.getTicket(ticketUuid);
545
+ return {
546
+ worktreePath,
547
+ ticket,
548
+ state: opts.state ?? ticket.state,
549
+ env: await this.runEnv(ticket, opts.run?.uuid, opts.state ?? ticket.state),
550
+ ...(opts.run ? { run: opts.run } : {}),
551
+ };
552
+ }
553
+ catch (err) {
554
+ this.logger.warn({
555
+ hook: opts.hook,
556
+ ticket: ticketUuid,
557
+ run: opts.run?.uuid,
558
+ err: String(err),
559
+ }, 'hook listener context unavailable — listeners not run');
560
+ return undefined;
561
+ }
562
+ }
563
+ get listeners() {
564
+ return this.config.hookListeners ?? [];
565
+ }
566
+ /**
567
+ * Run one lifecycle slot (BB-599): every addon listener in accumulator
568
+ * order, awaited, then the project shell hook through the executor. Never
569
+ * throws. A failing listener is logged by slot, listener, ticket and run —
570
+ * never with its message or the env — and reported in the returned result
571
+ * the same way a failing shell command is, so the rest of the slot, the
572
+ * shell hook and dispatch all continue.
573
+ */
574
+ async runSlot(name, cwd, ticketRef, ctx, shellEnv) {
575
+ const failed = [];
576
+ if (ctx) {
577
+ const env = ctx.env;
578
+ for (const [index, listener] of this.listeners.entries()) {
579
+ const fn = listener[name];
580
+ if (!fn)
581
+ continue;
582
+ const id = listener.id ?? `#${index}`;
583
+ const before = { ...env };
584
+ try {
585
+ await fn(ctx);
586
+ }
587
+ catch (err) {
588
+ failed.push(id);
589
+ this.logger.error({
590
+ hook: name,
591
+ listener: id,
592
+ ticket: ticketRef,
593
+ run: ctx.run?.uuid,
594
+ err: errorKind(err),
595
+ }, 'hook listener failed');
596
+ }
597
+ ctx.env = env;
598
+ this.guardEnv(env, before, {
599
+ hook: name,
600
+ listener: id,
601
+ ticket: ticketRef,
602
+ });
603
+ }
604
+ }
605
+ const shell = await this.executor.runHook(name, cwd, { ticket: ticketRef }, shellEnv);
606
+ if (failed.length === 0)
607
+ return shell;
608
+ const error = `hook listener failed: ${failed.join(', ')}`;
609
+ return {
610
+ hook: name,
611
+ ran: true,
612
+ ok: false,
613
+ error: shell.ok ? error : `${shell.error}; ${error}`,
614
+ };
615
+ }
616
+ /**
617
+ * Undo a listener's env change the conductor does not accept: a reserved
618
+ * key keeps the conductor's value, and an invalid name or a non-string value
619
+ * never reaches a spawn. Key names only are logged.
620
+ */
621
+ guardEnv(env, before, log) {
622
+ const rejected = [];
623
+ for (const key of Object.keys(env)) {
624
+ const value = env[key];
625
+ const ok = isReservedEnvKey(key)
626
+ ? before[key] === value
627
+ : ENV_KEY.test(key) && typeof value === 'string';
628
+ if (ok)
629
+ continue;
630
+ rejected.push(key);
631
+ if (key in before)
632
+ env[key] = before[key];
633
+ else
634
+ delete env[key];
635
+ }
636
+ for (const key of Object.keys(before)) {
637
+ if (isReservedEnvKey(key) && !(key in env)) {
638
+ env[key] = before[key];
639
+ rejected.push(key);
640
+ }
641
+ }
642
+ if (rejected.length > 0) {
643
+ this.logger.warn({ ...log, keys: rejected }, 'hook listener env change rejected');
644
+ }
645
+ }
646
+ /**
647
+ * Run `on_freeze` once for each newly frozen ticket and `on_resume` once
648
+ * for each that left the frozen list, in the ticket's worktree on this
649
+ * machine. A ticket without a worktree here gets no hook. Never throws: an
650
+ * unreadable list keeps the previous observation for the next poll.
651
+ */
652
+ async observeFreezes() {
653
+ let frozen;
654
+ try {
655
+ frozen = await this.remote.fetchFrozenTickets(this.id);
656
+ }
657
+ catch (err) {
658
+ this.logger.warn({ conductorId: this.id, err: String(err) }, 'frozen tickets unreadable — freeze hooks deferred');
659
+ return;
660
+ }
661
+ const seen = this.loadFrozenSeen();
662
+ const current = new Set(frozen.map((t) => t.ticketUuid));
663
+ let changed = false;
664
+ for (const [uuid, was] of [...seen]) {
665
+ if (current.has(uuid))
666
+ continue;
667
+ seen.delete(uuid);
668
+ changed = true;
669
+ this.queue.enqueue({
670
+ key: `freeze:${uuid}:on_resume`,
671
+ lane: uuid,
672
+ run: () => this.runFreezeSlot('on_resume', uuid, was),
673
+ });
674
+ }
675
+ for (const t of frozen) {
676
+ if (seen.has(t.ticketUuid))
677
+ continue;
678
+ seen.set(t.ticketUuid, {
679
+ identifier: t.identifier,
680
+ worktreePath: t.worktreePath,
681
+ });
682
+ changed = true;
683
+ this.queue.enqueue({
684
+ key: `freeze:${t.ticketUuid}:on_freeze`,
685
+ lane: t.ticketUuid,
686
+ run: () => this.runFreezeSlot('on_freeze', t.ticketUuid, t),
687
+ });
688
+ }
689
+ if (changed)
690
+ this.saveFrozenSeen(seen);
691
+ }
692
+ loadFrozenSeen() {
693
+ if (this.frozenSeen)
694
+ return this.frozenSeen;
695
+ this.frozenSeen = new Map();
696
+ const file = this.frozenStateFile;
697
+ if (file && existsSync(file)) {
698
+ try {
699
+ this.frozenSeen = new Map(Object.entries(JSON.parse(readFileSync(file, 'utf8'))));
700
+ }
701
+ catch (err) {
702
+ this.logger.warn({ file, err: String(err) }, 'frozen ticket state unreadable — starting empty');
703
+ }
704
+ }
705
+ return this.frozenSeen;
706
+ }
707
+ saveFrozenSeen(seen) {
708
+ const file = this.frozenStateFile;
709
+ if (!file)
710
+ return;
711
+ try {
712
+ mkdirSync(dirname(file), { recursive: true });
713
+ writeFileSync(file, `${JSON.stringify(Object.fromEntries(seen))}\n`);
714
+ }
715
+ catch (err) {
716
+ this.logger.warn({ file, err: String(err) }, 'frozen ticket state not saved');
717
+ }
718
+ }
719
+ async runFreezeSlot(name, ticketUuid, t) {
720
+ if (!t.worktreePath || !existsSync(t.worktreePath))
721
+ return;
722
+ const ctx = await this.listenerContext(ticketUuid, t.worktreePath, {
723
+ hook: name,
724
+ });
725
+ await this.runSlot(name, t.worktreePath, t.identifier, ctx, ctx?.env);
726
+ }
727
+ /**
728
+ * Reconcile closed tickets against their herdr worktrees and tear down any
729
+ * whose workspace still lingers (BB-89). Driven by the durable `cleaned_up`
730
+ * flag via {@link BbRemote.fetchUncleanedTickets}: every ticket the server
731
+ * has already closed but not yet cleaned. The SAME reconciliation runs on
732
+ * every tick AND standalone as `bakers oven clean`, and the query is
733
+ * scoped to THIS conductor (BB-121) — so it catches the orphans a
734
+ * live-tick-only path structurally cannot (the conductor was down when the
735
+ * ticket closed, or crashed mid-cleanup) for its OWN tickets. A foreign
736
+ * (non-bakers) worktree has no bakers ticket, so it is never on the work list; a
737
+ * ticket assigned to another conductor is filtered out at the query — both are
738
+ * left untouched with no host enumeration.
739
+ *
740
+ * The conductor is structurally UNABLE to close a ticket (BB-451): closing
741
+ * is a server-side projection of terminality written by
742
+ * `BbTicket::preSave()` on any save of a terminal, unclosed ticket, so it
743
+ * holds for every write path and cannot be withheld by a teardown hiccup —
744
+ * the reap reasons about `cleaned_up` alone. `cleaned_up` is withheld unless
745
+ * the teardown is ACCOUNTED FOR (BB-141 RC-2), which `removeWorktree`
746
+ * reports as one of three outcomes (BB-293):
747
+ * - `removed` — gone because this host tore it down → flag cleaned;
748
+ * - `already_absent` — nothing was there to tear down. A FINISHED teardown,
749
+ * not a failure: it is recorded in this log AND on the run record, then
750
+ * flagged cleaned. Recording it is the point — swallowing the absence and
751
+ * leaving `cleaned_up=0` is what made every later reap repeat the same
752
+ * failing teardown;
753
+ * - `unverified` — it could not be resolved and may still be on disk (e.g.
754
+ * its `worktree_path` was never persisted). Like a THROW, it logs loudly
755
+ * and leaves `cleaned_up=0` so the next reconciliation retries.
756
+ * Flagging cleaned on an unverified teardown was the root cause of the ~28
757
+ * never-touched leftovers, because the ticket then dropped off the
758
+ * (cleaned_up=0) work list forever.
759
+ * Re-running on an already-cleaned ticket is a no-op — it is off the list
760
+ * (idempotent). Each ticket is isolated in a try/catch so one failure never
761
+ * aborts the rest.
762
+ *
763
+ * The `after_done` hook runs through the executor (BB-84), which owns the
764
+ * best-effort policy — it logs+swallows a hook failure and never throws, so
765
+ * the core needs no per-call-site try/catch around it.
766
+ */
767
+ async reap() {
768
+ await this.enqueueReap();
769
+ await this.queue.drain();
770
+ }
771
+ async enqueueReap() {
772
+ const tickets = await this.remote.fetchUncleanedTickets(this.id);
773
+ for (const t of tickets) {
774
+ this.queue.enqueue({
775
+ key: `reap:${t.ticketUuid}`,
776
+ lane: t.ticketUuid,
777
+ run: () => this.reapOne(t),
778
+ });
779
+ }
780
+ }
781
+ /**
782
+ * Record the merge request's size on a done ticket while its worktree still
783
+ * exists. The server keeps `metrics` writable until `cleaned_up`; a failure
784
+ * is logged and never holds up the teardown. A teardown that keeps failing
785
+ * re-measures on every reap; the rows are replaced, so they stay correct.
786
+ */
787
+ async recordMrChanges(t) {
788
+ if (t.state !== 'done' || !t.hasMr)
789
+ return;
790
+ try {
791
+ if (!t.baseBranch)
792
+ throw new Error('ticket has no base branch');
793
+ // Under the git lock: workspace.ensure fetches the same ref into this repo.
794
+ const values = await this.serialGit(() => measureMrChanges(t.worktreePath, t.baseBranch));
795
+ await this.remote.replaceTicketMetrics(t.ticketUuid, 'mr_changes.', mrChangesRows(values, Math.floor(Date.now() / 1000)));
796
+ }
797
+ catch (err) {
798
+ this.logger.warn({ ticket: t.ticketUuid, err: String(err) }, 'mr_changes not recorded');
799
+ }
800
+ }
801
+ /** Reaps one ticket; isolated, so one failure never aborts the rest. */
802
+ async reapOne(t) {
803
+ try {
804
+ // Infra teardown. The `after_done` hook runs in the worktree regardless
805
+ // of executor kind (a non-persistent gitWorkspace still has an on-disk
806
+ // worktree); only the workspace removal is gated on the persistent
807
+ // capability (mirrors the executor-agnostic release path). Pass the
808
+ // stable worktreePath so the executor resolves the worktree by path,
809
+ // not by the mutable checked-out branch (the coding agent may rename
810
+ // it). Teardown is idempotent: a worktree herdr no longer tracks is a
811
+ // no-op / on-disk reclaim, not a failure.
812
+ //
813
+ // The hook needs the DIRECTORY, not just a recorded path (BB-293): a
814
+ // hook is a shell spawned with the worktree as its `cwd`, so against an
815
+ // absent one the spawn fails with ENOENT and the shared exec helper logs
816
+ // `ERROR: exec failed` — for the very condition this ticket declares
817
+ // normal. There is no useful work such a hook could do either: a command
818
+ // that must run inside the worktree cannot run anywhere else.
819
+ //
820
+ // The absence is logged rather than passed over in silence, because an
821
+ // `after_done` can reach beyond the directory it runs in — `ddev delete
822
+ // -Oy` also drops a DDEV project registration and its database — and
823
+ // that leftover is then an operator's to clear by hand.
824
+ //
825
+ // But the line states the CONDITION, not a decision about a hook: hook
826
+ // commands are executor-owned config that the core cannot see, so it
827
+ // does not know whether an `after_done` is configured at all. Saying a
828
+ // hook was "skipped" would be false wherever none exists (the generated
829
+ // playground config is exactly such a case). What the core does know is
830
+ // that the directory is gone and that no `after_done` ran — both true in
831
+ // either configuration.
832
+ //
833
+ // `existsSync` here is a plain filesystem question, not worktree
834
+ // knowledge: the core stays executor-agnostic and asks nothing about
835
+ // git. The check is deliberately reap-local — at dispatch a worktree the
836
+ // workspace just created MUST exist, and silencing a hook there would
837
+ // hide a real provisioning failure.
838
+ if (t.worktreePath) {
839
+ if (existsSync(t.worktreePath)) {
840
+ await this.recordMrChanges(t);
841
+ // BB-599: listeners get the ticket's current context; the shell
842
+ // hook keeps its inherited env, exactly as before.
843
+ const ctx = this.listeners.length === 0
844
+ ? undefined
845
+ : await this.listenerContext(t.ticketUuid, t.worktreePath, {
846
+ hook: 'after_done',
847
+ ...(t.runUuid ? { run: { uuid: t.runUuid } } : {}),
848
+ });
849
+ await this.runSlot('after_done', t.worktreePath, t.ticketUuid, ctx, undefined);
850
+ // BB-663 review F2: a torn-down environment invalidates any
851
+ // recorded `after_create` success — otherwise a worktree that
852
+ // survives teardown (non-persistent executor, unverified reap)
853
+ // looks bootstrapped forever and a reopened ticket dispatches
854
+ // straight into a worktree with no provisioned environment.
855
+ recordAfterCreateOutcome(bootstrapStateFileOf(this.config), t.worktreePath, false);
856
+ }
857
+ else {
858
+ this.logger.info({
859
+ ticket: t.ticketUuid,
860
+ hook: 'after_done',
861
+ worktreePath: t.worktreePath,
862
+ }, 'worktree directory absent — after_done not run');
863
+ }
864
+ }
865
+ if (this.executor.capabilities().persistent) {
866
+ let outcome;
867
+ try {
868
+ // removeWorktree reports what the teardown DID (BB-293):
869
+ // `removed` = the worktree is gone because this host removed it;
870
+ // `already_absent` = there was nothing to tear down; `unverified` =
871
+ // it could NOT be resolved and may still be on disk (BB-141 RC-2
872
+ // — e.g. worktree_path was never persisted and herdr lost track of
873
+ // the branch). A THROW is a hard failure.
874
+ outcome = await this.serialGit(() => this.executor.removeWorktree(t.branchName, t.worktreePath || undefined));
875
+ }
876
+ catch (err) {
877
+ this.logger.warn({
878
+ ticket: t.ticketUuid,
879
+ branch: t.branchName,
880
+ worktreePath: t.worktreePath,
881
+ err: String(err),
882
+ }, 'worktree teardown failed');
883
+ return; // leave cleaned_up=0 → the next reconciliation retries.
884
+ }
885
+ if (outcome === 'unverified') {
886
+ // BB-468 — an unverified teardown with NO run and NO recorded
887
+ // path has nothing left for a later reap to resolve: no run will
888
+ // appear on a terminal ticket, and no branch will appear for work
889
+ // that never started (`reap()` passes `t.worktreePath ||
890
+ // undefined`, so a never-ran ticket and a lost-path ticket reach
891
+ // the executor byte-identically — only the reaper, which alone
892
+ // knows `runUuid === ''`, can tell them apart). That retry is not
893
+ // deferred, it is unreachable, so it is logged ONCE at info and
894
+ // accounted for below rather than retried forever. BB-141 RC-2
895
+ // is unchanged for every ticket that DOES carry a run or a
896
+ // recorded path: those keep retrying below, unaffected.
897
+ if (!t.runUuid && !t.worktreePath) {
898
+ this.logger.info({ ticket: t.ticketUuid, branch: t.branchName }, 'no run and no recorded path — nothing to tear down');
899
+ }
900
+ else {
901
+ // Unverified teardown (BB-141 RC-2): do NOT flag cleaned_up, or
902
+ // the ticket drops off fetchUncleanedTickets (which filters
903
+ // cleaned_up=0) and the orphan is never retried. Leave it on the
904
+ // work list so a later reap — with a persisted path, or once the
905
+ // executor resolves it via the parent-repo git worktree list —
906
+ // can finish the job.
907
+ this.logger.warn({
908
+ ticket: t.ticketUuid,
909
+ branch: t.branchName,
910
+ worktreePath: t.worktreePath,
911
+ }, 'worktree teardown unverified — retrying next reap');
912
+ return; // leave cleaned_up=0 → the next reconciliation retries.
913
+ }
914
+ }
915
+ if (outcome === 'already_absent') {
916
+ // BB-293: the worktree was gone before we got here. That is a
917
+ // FINISHED teardown, so it is recorded rather than swallowed and
918
+ // the ticket is flagged cleaned below — the next reap will not
919
+ // retry it.
920
+ this.logger.info({
921
+ ticket: t.ticketUuid,
922
+ branch: t.branchName,
923
+ worktreePath: t.worktreePath,
924
+ }, 'worktree already absent — teardown complete');
925
+ // A ticket with no run has no record to write to — the conductor
926
+ // log above is then the whole account.
927
+ if (t.runUuid) {
928
+ try {
929
+ await this.remote.appendRunNote(t.runUuid, `worktree already absent at teardown: ${t.worktreePath || '(path unknown)'}`);
930
+ }
931
+ catch (err) {
932
+ // Best-effort: an unreachable control plane must not keep a
933
+ // finished teardown from being flagged.
934
+ this.logger.warn({ ticket: t.ticketUuid, run: t.runUuid, err: String(err) }, 'run note write failed');
935
+ }
936
+ }
937
+ }
938
+ }
939
+ // Teardown verified locally, the worktree was already absent, or there
940
+ // is nothing hosted to tear down: flag it cleaned so it drops off the
941
+ // work list.
942
+ await this.remote.markTicketCleanedUp(t.ticketUuid);
943
+ this.logger.info({ ticket: t.ticketUuid }, 'ticket cleaned up');
944
+ }
945
+ catch (err) {
946
+ this.logger.warn({ ticket: t.ticketUuid, err: String(err) }, 'ticket cleanup failed');
947
+ }
948
+ }
949
+ /**
950
+ * The env a run executes in: the ticket's effective env_vars under the core
951
+ * vars. Used at dispatch and, rebuilt from the current ticket, for the
952
+ * `after_run` / `after_done` listener contexts (BB-599).
953
+ */
954
+ async runEnv(t, runUuid, state) {
955
+ // BB-495: the agent names the workspace on its own calls now (the
956
+ // dropsh transport stamps none), so it has to know which one it is
957
+ // standing in and the full radius it may reach. Reserved core keys
958
+ // always win over per-ticket env_vars (below), so no precedence code is
959
+ // needed here. A key whose value could not be resolved is OMITTED, never
960
+ // exported empty — an empty variable reads as "set but blank" to a skill,
961
+ // which is worse than absent.
962
+ let workspaceIds;
963
+ try {
964
+ const ids = await this.remote.getReachableWorkspaces();
965
+ if (ids.length)
966
+ workspaceIds = ids.join(',');
967
+ }
968
+ catch (err) {
969
+ this.logger.warn({ ticket: t.identifier, err: String(err) }, 'could not resolve reachable workspaces for BB_WORKSPACE_IDS');
970
+ }
971
+ const core = {
972
+ BB_URL: this.config.site.base_url,
973
+ BB_ID: t.uuid,
974
+ // BB-569: lets a launch wrapper key off the ticket and the run's
975
+ // start state without parsing the prompt (addon-claude `measure`).
976
+ ...(t.identifier ? { BB_TICKET_IDENTIFIER: t.identifier } : {}),
977
+ ...(state ? { BB_STATE: state } : {}),
978
+ ...(runUuid ? { BB_RUN_UUID: runUuid } : {}),
979
+ BB_TUNNEL: this.config.tunnel === true ? '1' : '0',
980
+ WORKSPACE_ROOT: this.checkoutRoot,
981
+ ...(t.issueUrl ? { TICKET_URL: t.issueUrl } : {}),
982
+ ...(t.projectName ? { BB_PROJECT: t.projectName } : {}),
983
+ ...(t.projectId ? { BB_PROJECT_ID: t.projectId } : {}),
984
+ ...(t.workspaceName ? { BB_WORKSPACE: t.workspaceName } : {}),
985
+ ...(t.workspaceId ? { BB_WORKSPACE_ID: t.workspaceId } : {}),
986
+ ...(workspaceIds ? { BB_WORKSPACE_IDS: workspaceIds } : {}),
987
+ };
988
+ return resolveRunEnv(t.effectiveEnvVars, core, this.logger);
989
+ }
990
+ /** Dispatches one claimed run; a failure terminalises it (BB-149). */
991
+ async dispatchOrFail(run) {
992
+ try {
993
+ await this.dispatch(run);
994
+ }
995
+ catch (err) {
996
+ // BB-149: terminalise the run to `failed` with the error captured,
997
+ // instead of leaving it `claimed` to expire after the ~300s lease (the
998
+ // old silent stall). `failed` is a terminal sibling of `expired`
999
+ // (dispatch/setup error vs lease lapse); it does NOT reopen the ticket
1000
+ // the way the BB-41-era fabricated `done` did, so there is no runaway.
1001
+ // Marking it now frees the ticket immediately and records *why* it
1002
+ // failed; a persistently-failing setup is then parked by the server-side
1003
+ // circuit breaker (TicketClaimService::assertNotCircuitBroken) after the
1004
+ // retry cap, not by a hot re-dispatch loop. Best-effort: a markFailed
1005
+ // that itself errors must not stop the claim loop, so it is swallowed
1006
+ // (the run then falls back to lease-expiry recovery).
1007
+ this.logger.error({ run: run.runUuid, err: String(err) }, 'dispatch failed');
1008
+ try {
1009
+ await this.remote.markFailed(run.runUuid, String(err));
1010
+ }
1011
+ catch (markErr) {
1012
+ this.logger.warn({ run: run.runUuid, err: String(markErr) }, 'markFailed failed');
1013
+ }
1014
+ }
1015
+ }
1016
+ async dispatch(run) {
1017
+ const t = await this.remote.getTicket(run.ticketUuid);
1018
+ // Resolve the ticket alias and machine defaults, or the project selection hook.
1019
+ // Persist the package id below so finalize can choose its transcript parser.
1020
+ const chosen = resolveAgent(this.agents, t, {
1021
+ selectAgent: this.config.selectAgent,
1022
+ routes: this.config.routes,
1023
+ defaultAgent: this.config.default_agent,
1024
+ warn: (message) => this.logger.warn({ ticket: t.identifier }, message),
1025
+ });
1026
+ const baseRef = t.baseBranch ? `origin/${t.baseBranch}` : undefined;
1027
+ // Resolve the run env once (per-ticket env_vars + core BB_* vars) and
1028
+ // inject it into BOTH the executor-owned lifecycle hooks AND the agent run
1029
+ // env (BB-99). Reserved core keys always win; values may be sensitive so
1030
+ // only key names are ever logged.
1031
+ const env = await this.runEnv(t, run.runUuid, run.stateAtStart);
1032
+ const ws = await this.serialGit(() => this.workspace.ensure(t.identifier, t.branchName, baseRef));
1033
+ // A claimable state has no project policy without a WORKFLOW.md. Warn and
1034
+ // proceed so the missing project instructions are observable without
1035
+ // wedging the run. `ws.instructions` is the loaded file (null when absent).
1036
+ if (!ws.instructions) {
1037
+ this.logger.warn({ ticket: t.identifier, workspace: ws.path }, 'WORKFLOW.md missing — no project instructions dispatched');
1038
+ }
1039
+ // BB-599: one listener context per dispatch. Its `env` IS the run env,
1040
+ // so what a listener adds reaches later listeners, the shell hooks and
1041
+ // `startRun`. From the first listener on, a sidecar may exist — so any
1042
+ // failure from here unwinds through `after_run` before it propagates.
1043
+ const ctx = {
1044
+ worktreePath: ws.path,
1045
+ ticket: t,
1046
+ state: run.stateAtStart,
1047
+ env,
1048
+ run: { uuid: run.runUuid, id: run.runId },
1049
+ };
1050
+ const launch = { started: false };
1051
+ try {
1052
+ await this.launch(run, t, chosen, ws, ctx, launch);
1053
+ }
1054
+ catch (err) {
1055
+ await this.unwindDispatch(ctx, launch.started);
1056
+ throw err;
1057
+ }
1058
+ }
1059
+ /**
1060
+ * The part of a dispatch that can leave a sidecar or a worker behind
1061
+ * (BB-599): its caller unwinds it through `after_run` on any failure.
1062
+ */
1063
+ async launch(run, t, chosen, ws, ctx, launch) {
1064
+ const env = ctx.env;
1065
+ // Lifecycle hooks are executor-owned + best-effort (BB-84): runHook never
1066
+ // throws, so neither call can abort dispatch or wedge a run in `claimed`.
1067
+ // `runSlot` runs the addon listeners first under the same rule (BB-599).
1068
+ // Both get the resolved run env so a setup/pre hook sees the same vars (e.g.
1069
+ // a DB DSN) the agent does (BB-99).
1070
+ //
1071
+ // BB-237 / BB-663: `after_create` runs when the worktree was just
1072
+ // created OR this worktree has no record of a successful `after_create`.
1073
+ // `created` alone answers "was this worktree born just now?", which is
1074
+ // not the question dispatch needs answered — a worktree whose bootstrap
1075
+ // aborted, an interrupted earlier run, or a hand-made `git worktree add`
1076
+ // all exist WITHOUT being provisioned, and under a creation-only gate
1077
+ // they never got another chance (BB-237). But the inverse bug is just
1078
+ // as real: a worktree whose `after_create` already succeeded got the
1079
+ // hook re-run on EVERY later dispatch, because the conductor tracked no
1080
+ // outcome at all — every workflow step re-ran the project bootstrap
1081
+ // (BB-663). The record lives beside the conductor's own config file,
1082
+ // keyed by worktree path (not inside the worktree — review F1 — and not
1083
+ // in conductor memory, so it survives a conductor restart). `created`
1084
+ // always wins over a leftover record: `reapOne` clears it on `after_done`
1085
+ // (review F2), so a worktree torn down and reopened bootstraps again too.
1086
+ const bootstrapStateFile = bootstrapStateFileOf(this.config);
1087
+ const bootstrapped = !ws.created && hasSucceededAfterCreate(bootstrapStateFile, ws.path);
1088
+ const provisioning = bootstrapped
1089
+ ? { ok: true, ran: false, hook: 'after_create' }
1090
+ : await this.runSlot('after_create', ws.path, t.identifier, ctx, env);
1091
+ if (!bootstrapped) {
1092
+ recordAfterCreateOutcome(bootstrapStateFile, ws.path, provisioning.ok);
1093
+ }
1094
+ // AC-6: an agent is never dispatched into a worktree whose bootstrap failed
1095
+ // without that being visible. Dispatch is NOT withheld — runHook's
1096
+ // never-throw contract exists so a hook cannot wedge a run, and withholding
1097
+ // would re-introduce that wedge — so the failure is recorded against the run
1098
+ // instead (ticket + run + hook + workspace + err) and the run proceeds.
1099
+ // Boundedness stays with the existing circuit breaker.
1100
+ if (!provisioning.ok) {
1101
+ this.logger.error({
1102
+ hook: provisioning.hook,
1103
+ ticket: t.identifier,
1104
+ run: run.runUuid,
1105
+ workspace: ws.path,
1106
+ err: provisioning.error,
1107
+ }, 'lifecycle hook failed');
1108
+ }
1109
+ // BB-237: apply the workspace's open layout HERE — after `after_create`,
1110
+ // before the run starts. The layout's startup command names the bakers CLI
1111
+ // that `after_create` builds; applied inside `workspace.ensure()` (where it
1112
+ // used to live) it necessarily fired first, so a self-hosting checkout had
1113
+ // to paper over the ordering with a sentinel wait loop inside the pane
1114
+ // command. Ordering the two correctly removes the need for any handshake.
1115
+ //
1116
+ // Gated on `ws.created` — DELIBERATELY narrower than the `after_create` gate
1117
+ // right above it, which is convergent (a worktree can exist without ever
1118
+ // having been provisioned). Only a fresh create has an initial pane to lay
1119
+ // out; re-applying on reuse would start a duplicate editor on reattach.
1120
+ // Optional on the contract, so a workspace with no layout concept is a
1121
+ // no-op. Best-effort by contract: implementations swallow their own errors.
1122
+ if (ws.created) {
1123
+ await this.workspace.applyOpenLayout?.(ws.path, {
1124
+ identifier: t.identifier,
1125
+ branch: t.branchName,
1126
+ title: t.title,
1127
+ });
1128
+ }
1129
+ await this.runSlot('before_run', ws.path, t.identifier, ctx, env);
1130
+ // Clear a leftover tab for THIS run id in a reused workspace before
1131
+ // starting it (BB-183): run-scoped so a still-open sibling run's tab is
1132
+ // never killed. A fresh dispatch has no matching tab yet → a quiet no-op;
1133
+ // a re-dispatch of the same run drops its stale duplicate before startRun
1134
+ // re-creates it. Sibling prior-state run tabs are left to the worktree reap.
1135
+ await this.executor.cleanupRun(t.branchName, run.runId);
1136
+ const prompt = ws.instructions
1137
+ ? renderPrompt(this.config.prompt, {
1138
+ identifier: t.identifier,
1139
+ state: t.state || 'qualification',
1140
+ runUuid: run.runUuid,
1141
+ })
1142
+ : '';
1143
+ // BB-473 review round 2 (Critical 1): pinning the run's own UUID
1144
+ // unconditionally makes a re-dispatch of the same run — an existing,
1145
+ // supported path (see the cleanupRun comment above) — fail to launch,
1146
+ // because `claude`/`grok --session-id` refuse a session id that already
1147
+ // exists.
1148
+ //
1149
+ // BB-473 review round 4 (Critical 2): a `startedAt` time window is not
1150
+ // an identity — it can still select a SIBLING run's transcript that
1151
+ // happens to be newer than the bound, in the very worktree-keyed
1152
+ // directory this ticket exists to stop scanning by recency. Walking this
1153
+ // run's deterministic attempt family (attempt 0 == the run's own UUID)
1154
+ // and pinning the first member whose transcript does not already exist
1155
+ // gives dispatch a collision-free id AND gives finalise something to
1156
+ // select by that nothing outside this run can collide with — no
1157
+ // `sessionExists?` on the agent means every candidate reads as "free", so
1158
+ // an agent without the capability keeps pinning attempt 0 exactly as
1159
+ // before.
1160
+ const family = attemptFamily(run.runUuid);
1161
+ let pinnedSessionId;
1162
+ for (const candidate of family) {
1163
+ const exists = (await chosen.agent.sessionExists?.(ws.path, candidate)) ?? false;
1164
+ if (!exists) {
1165
+ pinnedSessionId = candidate;
1166
+ break;
1167
+ }
1168
+ }
1169
+ const identity = pinnedSessionId
1170
+ ? { sessionId: pinnedSessionId }
1171
+ : undefined;
1172
+ // BB-473 round 4 (review round 3, Critical 1): captured BEFORE
1173
+ // `startRun` so it precedes any transcript the agent is about to write —
1174
+ // re-stamped into `bb_run.started_at` on every dispatch, including a
1175
+ // re-dispatch, as a tie-breaker for agents with no session mechanism at
1176
+ // all (codex, opencode). Sent as whole seconds, matching every other
1177
+ // timestamp this method writes (`heartbeat`, `closed_date`) — a
1178
+ // fractional-second ISO string is not one of the Drupal `timestamp`
1179
+ // field's accepted formats and the PATCH would be rejected (review round
1180
+ // 4, Critical 1).
1181
+ const dispatchedAt = Math.floor(Date.now() / 1000);
1182
+ const warnings = chosen.agent.launchWarnings;
1183
+ if (warnings?.length) {
1184
+ const alias = chosen.alias ?? chosen.id;
1185
+ for (const warning of warnings) {
1186
+ this.logger.info({
1187
+ ticket: t.identifier,
1188
+ run: run.runUuid,
1189
+ harness: chosen.agent.id,
1190
+ alias,
1191
+ }, `${chosen.agent.id}/${alias}: ${warning}`);
1192
+ }
1193
+ }
1194
+ await this.executor.startRun({
1195
+ ticket: {
1196
+ uuid: t.uuid,
1197
+ identifier: t.identifier,
1198
+ title: t.title,
1199
+ branchName: t.branchName,
1200
+ state: t.state,
1201
+ ...(t.issueUrl ? { url: t.issueUrl } : {}),
1202
+ },
1203
+ run: { uuid: run.runUuid, id: run.runId, handler: run.handler },
1204
+ workspacePath: ws.path,
1205
+ instructions: ws.instructions,
1206
+ command: chosen.agent.launchCommand(prompt, identity),
1207
+ env,
1208
+ });
1209
+ launch.started = true;
1210
+ this.logger.info({
1211
+ ticket: t.identifier,
1212
+ run: run.runUuid,
1213
+ state: t.state,
1214
+ workspace: ws.path,
1215
+ // Key names only — env values may be sensitive (BB-99).
1216
+ envKeys: Object.keys(env),
1217
+ }, 'dispatched run');
1218
+ // The conductor — not the agent — knows the per-run git worktree path, so
1219
+ // it persists `worktree_path` to bb_run here alongside the running state.
1220
+ // It also persists the chosen agent id (BB-144) for finalize footprint
1221
+ // routing.
1222
+ await this.remote.markRunning(run.runUuid, {
1223
+ worktree_path: ws.path,
1224
+ agent: chosen.id,
1225
+ startedAt: dispatchedAt,
1226
+ });
1227
+ }
1228
+ /**
1229
+ * Stop what a failed dispatch started (BB-599): the worker if it was
1230
+ * launched, then the `after_run` listeners and shell hook. Best-effort — it
1231
+ * never replaces the dispatch error the caller re-throws.
1232
+ */
1233
+ async unwindDispatch(ctx, launched) {
1234
+ const runUuid = ctx.run?.uuid ?? '';
1235
+ try {
1236
+ if (launched && this.executor.capabilities().persistent) {
1237
+ try {
1238
+ await this.executor.stopRun(ctx.ticket.branchName);
1239
+ }
1240
+ catch (err) {
1241
+ this.logger.warn({ run: runUuid, err: String(err) }, 'agent stop failed');
1242
+ }
1243
+ await this.closeRunTab(ctx.ticket.branchName, runUuid, ctx.run?.id ?? 0);
1244
+ }
1245
+ await this.runSlot('after_run', ctx.worktreePath, ctx.ticket.identifier, ctx, ctx.env);
1246
+ }
1247
+ catch (err) {
1248
+ this.logger.warn({ run: runUuid, err: String(err) }, 'dispatch cleanup failed');
1249
+ }
1250
+ }
1251
+ async serve(signal) {
1252
+ await this.pollLoop(signal);
1253
+ }
1254
+ /**
1255
+ * True once this conductor's own checkout has been removed from disk (its
1256
+ * worktree was reaped while the process kept running). Such a conductor is a
1257
+ * pure liability: it still heartbeats and still claims runs, but every
1258
+ * dispatch fails — no git command and no relative path can resolve from a
1259
+ * deleted directory — so each claim burns one attempt and three of them park
1260
+ * the ticket behind the circuit breaker. Observed as 30 orphans claiming and
1261
+ * failing tickets they could never dispatch.
1262
+ */
1263
+ checkoutGone() {
1264
+ return !existsSync(this.checkoutRoot);
1265
+ }
1266
+ async pollLoop(signal) {
1267
+ while (!signal?.aborted) {
1268
+ if (this.checkoutGone()) {
1269
+ // Stop claiming and let the loop end — the process exits and the cron
1270
+ // reaper flips status to offline on lease expiry, same as any crash.
1271
+ // Deliberately NOT an offline write from here: the loop never owns that
1272
+ // transition (see the note at the end of this method).
1273
+ this.logger.error({ conductorId: this.id, workspace: this.checkoutRoot }, 'checkout is gone — stopping conductor instead of claiming runs it cannot dispatch');
1274
+ await this.queue.drain();
1275
+ return;
1276
+ }
1277
+ // Only produces: the loop never waits for the actions it enqueued.
1278
+ try {
1279
+ await this.produce();
1280
+ }
1281
+ catch (err) {
1282
+ this.logger.error({ conductorId: this.id, err: String(err) }, 'tick failed');
1283
+ }
1284
+ await sleep(this.config.poll_interval_ms, signal);
1285
+ }
1286
+ await this.queue.drain();
1287
+ // NB: the poll loop never writes status=offline itself. A graceful stop
1288
+ // (`bakers oven stop`) writes offline directly server-side; a crash or
1289
+ // hard-kill leaves a stale lease that the Drupal cron reaper flips to
1290
+ // offline (bb_core cron → ConductorReaper). Liveness is thus always
1291
+ // derived from status + last_seen/lease_expires_at, never from the loop.
1292
+ }
1293
+ }