omp-conductor 0.15.3 → 0.15.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -5
- package/package.json +1 -1
- package/src/cli.ts +7 -2
- package/src/daemon.ts +146 -0
- package/src/gitops.ts +98 -16
- package/src/lifecycle.ts +149 -29
- package/src/omp.ts +34 -0
- package/src/types.ts +1 -0
- package/src/upgrade.ts +283 -52
- package/src/worker.ts +41 -15
package/src/upgrade.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import { backupTimestamp, copyToUniqueBackup } from "./backups.ts";
|
|
3
3
|
import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
|
|
4
|
-
import { setPaused, statusSnapshot } from "./daemon.ts";
|
|
4
|
+
import { pauseInstance, setPaused, statusSnapshot } from "./daemon.ts";
|
|
5
5
|
import {
|
|
6
6
|
DEFAULT_HERDR_SESSION,
|
|
7
7
|
fleetLayers,
|
|
@@ -52,7 +52,28 @@ export interface UpgradeDeps {
|
|
|
52
52
|
snapshot(project?: string): { liveWorkers: number };
|
|
53
53
|
layers(project?: string): FleetLayers;
|
|
54
54
|
brief(project?: string): { kind: BriefLayout["kind"]; current: boolean };
|
|
55
|
-
|
|
55
|
+
/**
|
|
56
|
+
* Every configured project name, in config order. One daemon serves all of
|
|
57
|
+
* them (the generated unit's ExecStart carries no `--project`), so a
|
|
58
|
+
* transaction that restarts that daemon has to know the whole set rather
|
|
59
|
+
* than the one project a caller happened to name (#389).
|
|
60
|
+
*/
|
|
61
|
+
projectNames(): readonly string[];
|
|
62
|
+
/**
|
|
63
|
+
* The daemon the transaction targets, when one is running. `generation`
|
|
64
|
+
* identifies the exact instance (pid + start time) so a later restart cannot
|
|
65
|
+
* stop an instance created after the request began (#377).
|
|
66
|
+
*/
|
|
67
|
+
daemonIdentity(): { running: boolean; project?: string; generation?: string };
|
|
68
|
+
/**
|
|
69
|
+
* The durable pause sentinel this fleet currently holds, as an *instance*,
|
|
70
|
+
* read from one selected file (the per-project sentinel first): who set it,
|
|
71
|
+
* why, and when that sentinel was created. Two pauses with identical
|
|
72
|
+
* provenance are still different instances if `since` differs, so a drain
|
|
73
|
+
* can tell its own sentinel from one a later actor re-created after a
|
|
74
|
+
* resume lifted it — even when a legacy global sentinel coexists (#377).
|
|
75
|
+
*/
|
|
76
|
+
pauseState(project?: string): { source: string; reason?: string; since: number } | undefined;
|
|
56
77
|
setPaused(value: boolean, project?: string): void;
|
|
57
78
|
restartDaemon(): Promise<void>;
|
|
58
79
|
sleep(ms: number): Promise<void>;
|
|
@@ -89,10 +110,17 @@ export const DEFAULT_DEPS: UpgradeDeps = {
|
|
|
89
110
|
const live = readFileSync(layout.orchestratorPath, "utf8");
|
|
90
111
|
return { kind: layout.kind, current: live === renderBriefForProject(project, policy) };
|
|
91
112
|
},
|
|
113
|
+
projectNames: () => loadConfig().projects.map((p) => p.name),
|
|
92
114
|
daemonIdentity: () => {
|
|
93
115
|
const daemon = livingDaemon();
|
|
94
|
-
|
|
116
|
+
if (daemon === undefined) return { running: false };
|
|
117
|
+
return {
|
|
118
|
+
running: true,
|
|
119
|
+
project: daemon.project,
|
|
120
|
+
generation: `${daemon.pid}@${daemon.startedAt}`,
|
|
121
|
+
};
|
|
95
122
|
},
|
|
123
|
+
pauseState: (project) => pauseInstance(project),
|
|
96
124
|
setPaused: (v, project) =>
|
|
97
125
|
setPaused(v, { source: "upgrade", reason: "upgrade, draining" }, project),
|
|
98
126
|
restartDaemon: async () => {
|
|
@@ -248,10 +276,110 @@ function previousHerdrInstall(source: string): readonly [string, readonly string
|
|
|
248
276
|
];
|
|
249
277
|
}
|
|
250
278
|
|
|
251
|
-
|
|
279
|
+
/**
|
|
280
|
+
* The projects one lifecycle transaction covers, and the single pause sentinel
|
|
281
|
+
* that represents them.
|
|
282
|
+
*
|
|
283
|
+
* Everything an upgrade or a draining restart replaces is host-wide: the
|
|
284
|
+
* Bun-global CLI, the omp plugin, the Herdr plugin, and the one dispatch
|
|
285
|
+
* daemon. That daemon serves *every* configured project — the generated unit's
|
|
286
|
+
* ExecStart carries no `--project` — so its restart lands on all of them at
|
|
287
|
+
* once. The scope therefore names every project whose workers must drain and
|
|
288
|
+
* whose brief must be refreshed, instead of the single project a caller
|
|
289
|
+
* happened to name (#389).
|
|
290
|
+
*
|
|
291
|
+
* `selectors` are what the rest of the module passes back to the project-aware
|
|
292
|
+
* deps. A selector of `undefined` means "the one configured project", which is
|
|
293
|
+
* how a single-project host has always addressed itself: keeping it verbatim
|
|
294
|
+
* means the single-project path still emits exactly the same `brief-upgrade`
|
|
295
|
+
* command and the same one `snapshot` call it did before.
|
|
296
|
+
*
|
|
297
|
+
* `pauseKey` is the sentinel this transaction owns. It is `undefined` for a
|
|
298
|
+
* host-wide scope, which selects the legacy *global* sentinel — and that is
|
|
299
|
+
* precisely the right instrument: `isPaused(name)` consults the global file
|
|
300
|
+
* for every project, so one write pauses the whole host, and removing it lifts
|
|
301
|
+
* exactly this transaction's pause while leaving any per-project operator hold
|
|
302
|
+
* standing.
|
|
303
|
+
*/
|
|
304
|
+
interface UpgradeScope {
|
|
305
|
+
/** Project selectors to address the project-aware deps with, never empty. */
|
|
306
|
+
selectors: readonly (string | undefined)[];
|
|
307
|
+
/** The sentinel this transaction pauses and resumes. */
|
|
308
|
+
pauseKey: string | undefined;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Decides which projects a transaction covers, and refuses the requests that
|
|
313
|
+
* cannot be honoured truthfully.
|
|
314
|
+
*
|
|
315
|
+
* Two refusals, both about a caller naming a project the daemon does not agree
|
|
316
|
+
* with:
|
|
317
|
+
*
|
|
318
|
+
* - the daemon records a *different* project — it serves that one, so a
|
|
319
|
+
* request aimed at another is simply about a different fleet;
|
|
320
|
+
* - the daemon records *no* project while several are configured. That is not
|
|
321
|
+
* "unknown", it is the multi-project daemon: it serves them all. Narrowing
|
|
322
|
+
* is refused rather than obeyed, because draining one project and then
|
|
323
|
+
* restarting the shared daemon would kill the other projects' live workers
|
|
324
|
+
* without ever counting them (#389, and the shared-daemon rule in #378).
|
|
325
|
+
*
|
|
326
|
+
* A single-configured-project host keeps the historical bare selector, so its
|
|
327
|
+
* behaviour is unchanged whether or not the daemon recorded its name.
|
|
328
|
+
*/
|
|
329
|
+
function resolveScope(deps: UpgradeDeps, verb: "upgrade" | "restart", project?: string): UpgradeScope {
|
|
330
|
+
let configured: readonly string[] = [];
|
|
331
|
+
try {
|
|
332
|
+
configured = deps.projectNames();
|
|
333
|
+
} catch {
|
|
334
|
+
// No readable config is not this function's error to raise: the caller
|
|
335
|
+
// reaches a project-aware dep moments later and fails with the real
|
|
336
|
+
// problem. Treat it as the historical bare case.
|
|
337
|
+
}
|
|
338
|
+
const daemon = deps.daemonIdentity();
|
|
339
|
+
|
|
340
|
+
if (project !== undefined) {
|
|
341
|
+
if (daemon.running && daemon.project !== undefined && daemon.project !== project) {
|
|
342
|
+
throw new Error(
|
|
343
|
+
`active daemon serves ${daemon.project}; refusing to ${verb} --project ${project} while it is running`,
|
|
344
|
+
);
|
|
345
|
+
}
|
|
346
|
+
if (daemon.running && daemon.project === undefined && configured.length > 1) {
|
|
347
|
+
throw new Error(
|
|
348
|
+
`the running daemon serves all ${String(configured.length)} configured projects ` +
|
|
349
|
+
`(${configured.join(", ")}); ${verb} replaces host-wide packages and restarts that one daemon, ` +
|
|
350
|
+
`so it cannot be narrowed to --project ${project} — re-run \`omp-conductor ${verb}\` without --project`,
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
return { selectors: [project], pauseKey: project };
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// A daemon that recorded its project serves that project alone; the rest of
|
|
357
|
+
// the config is somebody else's fleet as far as this restart is concerned.
|
|
358
|
+
if (daemon.running && daemon.project !== undefined) {
|
|
359
|
+
return { selectors: [daemon.project], pauseKey: daemon.project };
|
|
360
|
+
}
|
|
361
|
+
if (configured.length > 1) return { selectors: configured, pauseKey: undefined };
|
|
362
|
+
return { selectors: [undefined], pauseKey: undefined };
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
async function waitForDrain(
|
|
366
|
+
deps: UpgradeDeps,
|
|
367
|
+
scope: UpgradeScope,
|
|
368
|
+
deadlineAt?: number,
|
|
369
|
+
stale?: () => string | undefined,
|
|
370
|
+
): Promise<void> {
|
|
252
371
|
let last = -1;
|
|
253
372
|
while (true) {
|
|
254
|
-
|
|
373
|
+
// A transaction that is no longer the fleet's current intent exits before
|
|
374
|
+
// it waits on anyone else's workers: a resume lifted its pause or the
|
|
375
|
+
// daemon it began with was replaced, so there is nothing left to drain
|
|
376
|
+
// and nothing it may stop (#377).
|
|
377
|
+
const cancelled = stale?.();
|
|
378
|
+
if (cancelled !== undefined) throw new Error(cancelled);
|
|
379
|
+
// Every project the scope covers: a host-wide drain that counted only one
|
|
380
|
+
// would report zero while another project's worker was still writing to a
|
|
381
|
+
// worktree the restart is about to orphan (#389).
|
|
382
|
+
const workers = scope.selectors.reduce((n, s) => n + deps.snapshot(s).liveWorkers, 0);
|
|
255
383
|
if (workers === 0) return;
|
|
256
384
|
if (deadlineAt !== undefined && Date.now() >= deadlineAt) {
|
|
257
385
|
throw new Error(
|
|
@@ -264,10 +392,71 @@ async function waitForDrain(deps: UpgradeDeps, project?: string, deadlineAt?: nu
|
|
|
264
392
|
}
|
|
265
393
|
}
|
|
266
394
|
|
|
395
|
+
/**
|
|
396
|
+
* What a draining restart captured when it began: the restart-owned pause and
|
|
397
|
+
* the daemon instance it is entitled to stop. Every later fence check compares
|
|
398
|
+
* live state against this snapshot, so the destructive restart call only ever
|
|
399
|
+
* acts on the world the request began in (#377).
|
|
400
|
+
*/
|
|
401
|
+
interface RestartBegun {
|
|
402
|
+
/** The durable pause sentinel this request relies on, proved readable as an instance. */
|
|
403
|
+
pauseToken: { source: string; reason?: string; since: number };
|
|
404
|
+
/** The daemon instance the restart targets, with its generation identity. */
|
|
405
|
+
daemon: ReturnType<UpgradeDeps["daemonIdentity"]>;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Why a draining restart can no longer act, or `undefined` when it still owns
|
|
410
|
+
* the pause and the daemon generation it began with. The pause check is the
|
|
411
|
+
* *restart-owned sentinel instance* — `drainAndRestart` proves ownership up
|
|
412
|
+
* front, so a token that cannot be re-read (sentinel removed, unreadable, or
|
|
413
|
+
* re-created — even under the same source and reason) means this request's own
|
|
414
|
+
* pause no longer exists and it must cancel. The generation check is the
|
|
415
|
+
* later-daemon fence: a daemon created after this request began — by
|
|
416
|
+
* `restart --now`, a crash-restart, or an operator start — must never be
|
|
417
|
+
* stopped by it.
|
|
418
|
+
*/
|
|
419
|
+
function restartFenceProblem(
|
|
420
|
+
deps: UpgradeDeps,
|
|
421
|
+
scope: UpgradeScope,
|
|
422
|
+
begun: RestartBegun,
|
|
423
|
+
): string | undefined {
|
|
424
|
+
if (!deps.layers(scope.pauseKey).paused) {
|
|
425
|
+
return "restart cancelled: dispatch was resumed while the drain was in progress — nothing was restarted";
|
|
426
|
+
}
|
|
427
|
+
const owned = deps.pauseState(scope.pauseKey);
|
|
428
|
+
if (
|
|
429
|
+
owned === undefined ||
|
|
430
|
+
owned.since !== begun.pauseToken.since ||
|
|
431
|
+
owned.source !== begun.pauseToken.source ||
|
|
432
|
+
owned.reason !== begun.pauseToken.reason
|
|
433
|
+
) {
|
|
434
|
+
return "restart cancelled: the restart-owned pause was lifted and replaced while the drain was in progress — nothing was restarted";
|
|
435
|
+
}
|
|
436
|
+
const now = deps.daemonIdentity();
|
|
437
|
+
if (now.running !== begun.daemon.running || (begun.daemon.running && now.generation !== begun.daemon.generation)) {
|
|
438
|
+
return "restart cancelled: the daemon instance was replaced while the drain was in progress — the newer daemon was left running";
|
|
439
|
+
}
|
|
440
|
+
return undefined;
|
|
441
|
+
}
|
|
442
|
+
|
|
267
443
|
/**
|
|
268
444
|
* Pause, drain, restart, restore — the trusted restart transaction, minus the
|
|
269
445
|
* install/verify steps of {@link upgradeConductor}.
|
|
270
446
|
*
|
|
447
|
+
* Generation-scoped (#377): the transaction captures the restart-owned pause
|
|
448
|
+
* and the daemon generation when it begins and revalidates both on every drain
|
|
449
|
+
* poll and once more immediately before the destructive restart call. A
|
|
450
|
+
* `resume`, a `restart --now`, or any other action that lifts the pause or
|
|
451
|
+
* replaces the daemon while the drain waits makes the transaction stale: it
|
|
452
|
+
* exits nonzero without stopping anything, so a timed-out or disconnected
|
|
453
|
+
* restart caller cannot leave a delayed stop aimed at a later daemon
|
|
454
|
+
* generation. The stale transaction restores nothing and leaves whatever
|
|
455
|
+
* pause the fleet now has exactly as it found it. Ownership is proven up
|
|
456
|
+
* front: a sentinel that cannot be read as an instance at transaction start
|
|
457
|
+
* aborts before any drain or restart, because a lock the fence cannot compare
|
|
458
|
+
* against is a lock the fence cannot protect.
|
|
459
|
+
*
|
|
271
460
|
* Mirrors the upgrade's fail-closed posture: on ANY throw after the pause the
|
|
272
461
|
* fleet stays paused (no resume in a catch) and the error rethrows, so a
|
|
273
462
|
* systemd-owned restart that fails or a drain that outlives `timeoutMs` cannot
|
|
@@ -278,11 +467,34 @@ export async function drainAndRestart(
|
|
|
278
467
|
deps: UpgradeDeps,
|
|
279
468
|
o: { project?: string; timeoutMs: number },
|
|
280
469
|
): Promise<void> {
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
470
|
+
// Host-wide by default (#389): the daemon this restarts serves every
|
|
471
|
+
// configured project, so the drain counts every project's workers and the
|
|
472
|
+
// pause covers all of them.
|
|
473
|
+
const scope = resolveScope(deps, "restart", o.project);
|
|
474
|
+
const initial = deps.layers(scope.pauseKey);
|
|
475
|
+
if (!initial.paused) deps.setPaused(true, scope.pauseKey);
|
|
476
|
+
// The lock must be provable NOW. A pause that cannot be read as an instance
|
|
477
|
+
// — sentinel malformed or unreadable — would make the whole fence fail open
|
|
478
|
+
// (nothing to compare against), so the transaction refuses before waiting
|
|
479
|
+
// on anything it cannot act upon either way.
|
|
480
|
+
const pauseToken = deps.pauseState(scope.pauseKey);
|
|
481
|
+
if (pauseToken === undefined) {
|
|
482
|
+
throw new Error(
|
|
483
|
+
"restart cancelled: cannot prove the restart-owned pause — the active pause sentinel is unreadable or malformed; nothing was restarted",
|
|
484
|
+
);
|
|
485
|
+
}
|
|
486
|
+
const begun: RestartBegun = {
|
|
487
|
+
pauseToken,
|
|
488
|
+
daemon: deps.daemonIdentity(),
|
|
489
|
+
};
|
|
490
|
+
const stale = () => restartFenceProblem(deps, scope, begun);
|
|
491
|
+
await waitForDrain(deps, scope, Date.now() + o.timeoutMs, stale);
|
|
492
|
+
// The drain completed; the world may have moved on while it did. Re-prove
|
|
493
|
+
// the pause and the generation an instant before the destructive call.
|
|
494
|
+
const cancelled = stale();
|
|
495
|
+
if (cancelled !== undefined) throw new Error(cancelled);
|
|
284
496
|
await deps.restartDaemon();
|
|
285
|
-
if (!initial.paused) deps.setPaused(false,
|
|
497
|
+
if (!initial.paused) deps.setPaused(false, scope.pauseKey);
|
|
286
498
|
}
|
|
287
499
|
|
|
288
500
|
function recoveryProblem(
|
|
@@ -308,19 +520,32 @@ function recoveryProblem(
|
|
|
308
520
|
async function waitForRecovery(
|
|
309
521
|
deps: UpgradeDeps,
|
|
310
522
|
initial: FleetLayers,
|
|
311
|
-
|
|
523
|
+
scope: UpgradeScope,
|
|
312
524
|
): Promise<void> {
|
|
313
525
|
let problem = "recovery did not settle";
|
|
314
526
|
for (let attempt = 0; attempt < RECOVERY_ATTEMPTS; attempt += 1) {
|
|
315
|
-
|
|
527
|
+
const live = scope.selectors.reduce((n, s) => n + deps.snapshot(s).liveWorkers, 0);
|
|
528
|
+
problem = recoveryProblem(deps.layers(scope.pauseKey), initial, live) ?? "";
|
|
316
529
|
if (problem === "") return;
|
|
317
530
|
await deps.sleep(RECOVERY_POLL_MS);
|
|
318
531
|
}
|
|
319
532
|
throw new Error(`upgrade verification failed: ${problem}`);
|
|
320
533
|
}
|
|
321
534
|
|
|
535
|
+
/** One project's brief, as the transaction found it. */
|
|
536
|
+
interface ScopedBrief {
|
|
537
|
+
/** The selector to address the project-aware deps and the CLI with. */
|
|
538
|
+
selector: string | undefined;
|
|
539
|
+
kind: BriefLayout["kind"];
|
|
540
|
+
current: boolean;
|
|
541
|
+
}
|
|
542
|
+
|
|
322
543
|
/**
|
|
323
|
-
* Brings
|
|
544
|
+
* Brings every in-scope brief up with the freshly installed package.
|
|
545
|
+
*
|
|
546
|
+
* Each project owns its own ORCHESTRATOR.md, so a host-wide upgrade migrates
|
|
547
|
+
* them one at a time rather than leaving every project but one on a floor the
|
|
548
|
+
* installed package no longer emits (#389).
|
|
324
549
|
*
|
|
325
550
|
* `brief-upgrade --migrate --apply` is the cross-version ABI: this code runs
|
|
326
551
|
* from the *old* CLI while the new one is already installed, so the verb has to
|
|
@@ -334,29 +559,27 @@ async function waitForRecovery(
|
|
|
334
559
|
* fleet on a single-file brief whose floor no longer tracks the package —
|
|
335
560
|
* the exact drift this verb exists to end — so it fails loudly and rolls back.
|
|
336
561
|
*/
|
|
337
|
-
async function
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
352
|
-
deps.log(`brief: brief-upgrade unavailable in the target CLI (${msg}) — overlay recomposes each tick, continuing`);
|
|
562
|
+
async function upgradeBriefs(deps: UpgradeDeps, briefs: readonly ScopedBrief[]): Promise<void> {
|
|
563
|
+
for (const brief of briefs) {
|
|
564
|
+
const selected = brief.selector === undefined ? [] : ["--project", brief.selector];
|
|
565
|
+
if (brief.kind === "legacy-handwritten") {
|
|
566
|
+
// Ungated on purpose: a hand-written brief cannot be migrated without it.
|
|
567
|
+
await mustRun(deps, "omp-conductor", ["brief-upgrade", "--retrofit", "--apply", ...selected]);
|
|
568
|
+
}
|
|
569
|
+
try {
|
|
570
|
+
await mustRun(deps, "omp-conductor", ["brief-upgrade", "--migrate", "--apply", ...selected]);
|
|
571
|
+
} catch (err) {
|
|
572
|
+
if (brief.kind !== "overlay") throw err;
|
|
573
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
574
|
+
deps.log(`brief: brief-upgrade unavailable in the target CLI (${msg}) — overlay recomposes each tick, continuing`);
|
|
575
|
+
}
|
|
353
576
|
}
|
|
354
577
|
}
|
|
355
578
|
|
|
356
579
|
async function rollbackUpgrade(
|
|
357
580
|
deps: UpgradeDeps,
|
|
358
581
|
previous: InstalledSurfaces,
|
|
359
|
-
|
|
582
|
+
scope: UpgradeScope,
|
|
360
583
|
installTouched: boolean,
|
|
361
584
|
briefChanged: boolean,
|
|
362
585
|
herdrReloadStarted: boolean,
|
|
@@ -466,7 +689,10 @@ async function rollbackUpgrade(
|
|
|
466
689
|
if (briefChanged) {
|
|
467
690
|
try {
|
|
468
691
|
deps.log("rollback: ORCHESTRATOR.md package floor");
|
|
469
|
-
await
|
|
692
|
+
await upgradeBriefs(
|
|
693
|
+
deps,
|
|
694
|
+
scope.selectors.map((selector) => ({ selector, kind: "overlay" as const, current: false })),
|
|
695
|
+
);
|
|
470
696
|
} catch (err) {
|
|
471
697
|
failures.push(err instanceof Error ? err.message : String(err));
|
|
472
698
|
}
|
|
@@ -515,23 +741,21 @@ export async function upgradeConductor(
|
|
|
515
741
|
}
|
|
516
742
|
|
|
517
743
|
const requested = options.version === undefined ? `${PACKAGE}@latest` : `${PACKAGE}@${options.version}`;
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
throw new Error(
|
|
522
|
-
`active daemon serves ${active}; refusing to upgrade --project ${options.project} while it is running`,
|
|
523
|
-
);
|
|
524
|
-
}
|
|
525
|
-
const project = options.project ?? daemon.project;
|
|
744
|
+
// Host-wide by default (#389): one daemon serves every configured project,
|
|
745
|
+
// so an upgrade that restarts it drains them all and refreshes every brief.
|
|
746
|
+
const scope = resolveScope(deps, "upgrade", options.project);
|
|
526
747
|
|
|
527
748
|
const release = parseRegistry(
|
|
528
749
|
(await mustRun(deps, "npm", ["view", requested, "version", "gitHead", "--json"])).stdout,
|
|
529
750
|
);
|
|
530
|
-
const initial = deps.layers(
|
|
751
|
+
const initial = deps.layers(scope.pauseKey);
|
|
531
752
|
const surfaces = await inspectSurfaces(deps);
|
|
532
|
-
const
|
|
753
|
+
const briefs: ScopedBrief[] = scope.selectors.map((selector) => ({
|
|
754
|
+
selector,
|
|
755
|
+
...deps.brief(selector),
|
|
756
|
+
}));
|
|
533
757
|
const installNeeded = !surfacesCurrent(surfaces, release.version, release.gitHead);
|
|
534
|
-
if (!installNeeded &&
|
|
758
|
+
if (!installNeeded && briefs.every((b) => b.current)) {
|
|
535
759
|
return {
|
|
536
760
|
previousVersion: surfaces.cliVersion,
|
|
537
761
|
...release,
|
|
@@ -562,17 +786,24 @@ export async function upgradeConductor(
|
|
|
562
786
|
// either — the in-memory restore below still covers a clean rollback.
|
|
563
787
|
}
|
|
564
788
|
|
|
565
|
-
|
|
789
|
+
const missing = briefs.find((b) => b.kind === "missing");
|
|
790
|
+
if (missing !== undefined) {
|
|
791
|
+
throw new Error(
|
|
792
|
+
missing.selector === undefined
|
|
793
|
+
? "no ORCHESTRATOR.md exists for the configured project"
|
|
794
|
+
: `no ORCHESTRATOR.md exists for project ${missing.selector}`,
|
|
795
|
+
);
|
|
796
|
+
}
|
|
566
797
|
if (initial.herdr === "unknown") throw new Error("cannot determine whether herdr-fleet.service is active");
|
|
567
798
|
|
|
568
799
|
deps.log(`target release: omp-conductor@${release.version} (${release.gitHead})`);
|
|
569
800
|
if (!initial.paused) {
|
|
570
801
|
deps.log("safety: pausing new issue claims");
|
|
571
|
-
deps.setPaused(true,
|
|
802
|
+
deps.setPaused(true, scope.pauseKey);
|
|
572
803
|
}
|
|
573
804
|
deps.log("drain: waiting for live omp worker sessions");
|
|
574
805
|
try {
|
|
575
|
-
await waitForDrain(deps,
|
|
806
|
+
await waitForDrain(deps, scope);
|
|
576
807
|
} catch (err) {
|
|
577
808
|
const failure = err instanceof Error ? err.message : String(err);
|
|
578
809
|
throw new Error(
|
|
@@ -610,7 +841,7 @@ export async function upgradeConductor(
|
|
|
610
841
|
|
|
611
842
|
deps.log("brief: refreshing managed ORCHESTRATOR.md floor while preserving POLICY.md");
|
|
612
843
|
briefChanged = true;
|
|
613
|
-
await
|
|
844
|
+
await upgradeBriefs(deps, briefs);
|
|
614
845
|
|
|
615
846
|
if (initial.herdr === "active") {
|
|
616
847
|
herdrReloadStarted = true;
|
|
@@ -635,13 +866,13 @@ export async function upgradeConductor(
|
|
|
635
866
|
const active = await mustRun(deps, "systemctl", ["is-active", HERDR_UNIT]);
|
|
636
867
|
if (active.stdout.trim() !== "active") throw new Error(`${HERDR_UNIT} is not active after restart`);
|
|
637
868
|
}
|
|
638
|
-
await waitForRecovery(deps, initial,
|
|
869
|
+
await waitForRecovery(deps, initial, scope);
|
|
639
870
|
deps.log("verify 2/2: recovered fleet remains stable");
|
|
640
871
|
await deps.sleep(1_000);
|
|
641
|
-
await waitForRecovery(deps, initial,
|
|
872
|
+
await waitForRecovery(deps, initial, scope);
|
|
642
873
|
|
|
643
|
-
if (!initial.paused) deps.setPaused(false,
|
|
644
|
-
const restoredDispatch = deps.layers(
|
|
874
|
+
if (!initial.paused) deps.setPaused(false, scope.pauseKey);
|
|
875
|
+
const restoredDispatch = deps.layers(scope.pauseKey).dispatch;
|
|
645
876
|
if (restoredDispatch !== initial.dispatch) {
|
|
646
877
|
throw new Error(
|
|
647
878
|
`dispatch state is ${restoredDispatch}; expected to restore ${initial.dispatch}`,
|
|
@@ -649,9 +880,9 @@ export async function upgradeConductor(
|
|
|
649
880
|
}
|
|
650
881
|
} catch (err) {
|
|
651
882
|
try {
|
|
652
|
-
if (!deps.layers(
|
|
883
|
+
if (!deps.layers(scope.pauseKey).paused) deps.setPaused(true, scope.pauseKey);
|
|
653
884
|
} catch {
|
|
654
|
-
deps.setPaused(true,
|
|
885
|
+
deps.setPaused(true, scope.pauseKey);
|
|
655
886
|
}
|
|
656
887
|
const failure = err instanceof Error ? err.message : String(err);
|
|
657
888
|
deps.log(`upgrade failed: ${failure}`);
|
|
@@ -659,7 +890,7 @@ export async function upgradeConductor(
|
|
|
659
890
|
await rollbackUpgrade(
|
|
660
891
|
deps,
|
|
661
892
|
surfaces,
|
|
662
|
-
|
|
893
|
+
scope,
|
|
663
894
|
installTouched,
|
|
664
895
|
briefChanged,
|
|
665
896
|
herdrReloadStarted,
|
package/src/worker.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* sliding into a merge queue.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { createSession, disposeSession } from "./omp.ts";
|
|
13
|
+
import { createSession, disposeSession, SessionAdmissionClosedError, type AgentSessionLike } from "./omp.ts";
|
|
14
14
|
import type { ReleaseBlockContext } from "./release-policy.ts";
|
|
15
15
|
import type { Caps, ReleaseShape, ResolvedGrants, RunState } from "./types.ts";
|
|
16
16
|
|
|
@@ -123,6 +123,13 @@ export interface WorkerOpts {
|
|
|
123
123
|
* exists; absent = no pause surface (tests, one-shot callers).
|
|
124
124
|
*/
|
|
125
125
|
onPauseControl?: (control: WorkerPauseControl) => void;
|
|
126
|
+
/**
|
|
127
|
+
* Pre-spawn admission gate, forwarded to `createSession` (#374): the daemon
|
|
128
|
+
* closes it the moment its stop begins, so a stop that lands while the
|
|
129
|
+
* session socket is binding refuses the spawn and the run settles `stopped`
|
|
130
|
+
* instead of `failed`. Absent, the spawn always proceeds.
|
|
131
|
+
*/
|
|
132
|
+
maySpawn?: () => boolean;
|
|
126
133
|
}
|
|
127
134
|
|
|
128
135
|
/**
|
|
@@ -253,20 +260,39 @@ export async function runWorker(
|
|
|
253
260
|
const now = deps.now ?? Date.now;
|
|
254
261
|
const schedule = deps.schedule ?? scheduleWallClock;
|
|
255
262
|
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
263
|
+
let session: AgentSessionLike;
|
|
264
|
+
try {
|
|
265
|
+
session = await deps.createSession({
|
|
266
|
+
cwd: o.cwd,
|
|
267
|
+
...(o.sessionDir === undefined ? {} : { sessionDir: o.sessionDir }),
|
|
268
|
+
...(o.model === undefined ? {} : { model: o.model }),
|
|
269
|
+
// Prevention half of #24: as a worker, structured file tools cannot leave
|
|
270
|
+
// this worktree, and no release grant can ever reach this session (#122).
|
|
271
|
+
role: "worker",
|
|
272
|
+
...(o.releaseGrants === undefined ? {} : { releaseGrants: o.releaseGrants }),
|
|
273
|
+
...(o.onReleaseBlocked === undefined ? {} : { onReleaseBlocked: o.onReleaseBlocked }),
|
|
274
|
+
...(o.onSpawn === undefined ? {} : { onSpawn: o.onSpawn }),
|
|
275
|
+
...(o.socketPath === undefined ? {} : { socketPath: o.socketPath }),
|
|
276
|
+
...(o.verbSocketPath === undefined ? {} : { verbSocketPath: o.verbSocketPath }),
|
|
277
|
+
...(o.onChildLog === undefined ? {} : { onChildLog: o.onChildLog }),
|
|
278
|
+
...(o.maySpawn === undefined ? {} : { maySpawn: o.maySpawn }),
|
|
279
|
+
});
|
|
280
|
+
} catch (err) {
|
|
281
|
+
// The pre-spawn gate closed (#374): a daemon stop landed while the session
|
|
282
|
+
// socket was binding, so no child was ever created. That is a stop, not a
|
|
283
|
+
// failure — reporting `failed` here would charge an attempt against a
|
|
284
|
+
// worker the shutdown refused to start. `onSpawn` was never called.
|
|
285
|
+
if (err instanceof SessionAdmissionClosedError) {
|
|
286
|
+
return {
|
|
287
|
+
state: "stopped",
|
|
288
|
+
turns: 0,
|
|
289
|
+
spendUsd: 0,
|
|
290
|
+
report: "",
|
|
291
|
+
stoppedReason: "daemon shutdown began before the worker session started",
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
throw err;
|
|
295
|
+
}
|
|
270
296
|
|
|
271
297
|
// Before the first turn, not after the last: a caller that only learns the
|
|
272
298
|
// transcript path from the result learns it once the run it wanted to watch
|