omp-conductor 0.3.7 → 0.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -24
- package/package.json +1 -1
- package/src/briefs/orchestrator.md +16 -7
- package/src/briefs/worker.md +1 -1
- package/src/cli.ts +28 -14
- package/src/config.ts +16 -5
- package/src/daemon.ts +147 -48
- package/src/escalate.ts +26 -4
- package/src/lifecycle.ts +203 -12
- package/src/omp.ts +5 -0
- package/src/orchestrator-tick.ts +42 -2
- package/src/plugin.ts +32 -2
- package/src/setup.ts +3 -1
- package/src/types.ts +5 -2
- package/src/worker.ts +37 -5
- package/src/worktree.ts +109 -12
package/src/daemon.ts
CHANGED
|
@@ -11,7 +11,7 @@ import { createHash } from "node:crypto";
|
|
|
11
11
|
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
12
12
|
import { dirname, join, relative } from "node:path";
|
|
13
13
|
import { configPath, findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
|
|
14
|
-
import { createEscalator } from "./escalate.ts";
|
|
14
|
+
import { createEscalator, escalationIssueRef } from "./escalate.ts";
|
|
15
15
|
import { graphHint } from "./graph.ts";
|
|
16
16
|
import { livingDaemon } from "./lifecycle.ts";
|
|
17
17
|
import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
|
|
@@ -369,7 +369,7 @@ async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<b
|
|
|
369
369
|
await d.escalate(e);
|
|
370
370
|
return true;
|
|
371
371
|
} catch (err) {
|
|
372
|
-
log(`escalation for
|
|
372
|
+
log(`escalation for ${escalationIssueRef(e.issue)} could not be delivered: ${errText(err)}`);
|
|
373
373
|
return false;
|
|
374
374
|
}
|
|
375
375
|
}
|
|
@@ -393,13 +393,22 @@ export function salvageLines(outcome: SalvageOutcome, worktree: string): string[
|
|
|
393
393
|
];
|
|
394
394
|
}
|
|
395
395
|
|
|
396
|
-
|
|
396
|
+
const where =
|
|
397
397
|
`WIP committed to ${outcome.branch} @ ${outcome.sha}` +
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
398
|
+
(outcome.pushed
|
|
399
|
+
? " and pushed — the work outlives this worktree"
|
|
400
|
+
: ` but NOT pushed (${outcome.pushError ?? "no reason given"}) — it lives only in this host's mirror`);
|
|
401
|
+
// Manifest belongs in the escalation too: opening the commit is how the
|
|
402
|
+
// orchestrator talked itself into scrubbing a worker tree (#38).
|
|
403
|
+
const n = outcome.files.length;
|
|
404
|
+
const count = `${n} file${n === 1 ? "" : "s"}`;
|
|
405
|
+
const manifest =
|
|
406
|
+
outcome.newPaths.length === 0
|
|
407
|
+
? `${count} (all modifications to tracked paths)`
|
|
408
|
+
: `${count}; new: ${outcome.newPaths.slice(0, 12).join(", ")}${
|
|
409
|
+
outcome.newPaths.length > 12 ? `, … +${outcome.newPaths.length - 12} more` : ""
|
|
410
|
+
}`;
|
|
411
|
+
return [where, manifest, kept];
|
|
403
412
|
}
|
|
404
413
|
|
|
405
414
|
/**
|
|
@@ -444,10 +453,35 @@ export async function buildBrief(
|
|
|
444
453
|
r: Routed,
|
|
445
454
|
branch: string,
|
|
446
455
|
worktree: string,
|
|
456
|
+
opts: { continuation?: boolean; defaultBranch?: string } = {},
|
|
447
457
|
): Promise<string> {
|
|
448
458
|
// Read per dispatch rather than caching: editing the brief then takes effect
|
|
449
459
|
// on the next issue instead of needing a daemon restart.
|
|
450
460
|
const template = await Bun.file(BRIEF_TEMPLATE_PATH).text();
|
|
461
|
+
const defaultBranch = opts.defaultBranch ?? r.repo.defaultBranch;
|
|
462
|
+
const continuation =
|
|
463
|
+
opts.continuation === true
|
|
464
|
+
? [
|
|
465
|
+
"",
|
|
466
|
+
"## Continuation — do not start from zero",
|
|
467
|
+
"",
|
|
468
|
+
`You are **resuming** issue #${r.issue.number}. Branch \`${branch}\` already exists`,
|
|
469
|
+
"and was reattached with prior commits (and possibly a salvaged WIP tip).",
|
|
470
|
+
"Before writing anything:",
|
|
471
|
+
"",
|
|
472
|
+
"```bash",
|
|
473
|
+
"git log --oneline origin/" + defaultBranch + "..HEAD",
|
|
474
|
+
"git diff --stat origin/" + defaultBranch + "...HEAD",
|
|
475
|
+
"git status --porcelain",
|
|
476
|
+
"```",
|
|
477
|
+
"",
|
|
478
|
+
"Read that history. **Do not recreate work that already exists.** Finish",
|
|
479
|
+
"what remains against the same acceptance criteria. If a prior attempt",
|
|
480
|
+
"left a `wip(#…): … auto-salvaged` commit, treat it as your starting",
|
|
481
|
+
"point, not as trash to rewrite from scratch.",
|
|
482
|
+
"",
|
|
483
|
+
].join("\n")
|
|
484
|
+
: "";
|
|
451
485
|
return renderBrief(template, {
|
|
452
486
|
ISSUE_NUMBER: String(r.issue.number),
|
|
453
487
|
ISSUE_TITLE: r.issue.title,
|
|
@@ -461,6 +495,7 @@ export async function buildBrief(
|
|
|
461
495
|
// placeholder sits flush against the next list item in the template, so an
|
|
462
496
|
// unconfigured render leaves no blank line where a hint would have gone.
|
|
463
497
|
GRAPH_HINT: graphHint(r.repo),
|
|
498
|
+
CONTINUATION: continuation,
|
|
464
499
|
});
|
|
465
500
|
}
|
|
466
501
|
|
|
@@ -513,14 +548,14 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
513
548
|
// would cost a second network fetch per attempt.
|
|
514
549
|
const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
|
|
515
550
|
await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
|
|
516
|
-
|
|
517
|
-
worktreePath = await addWorktree(
|
|
551
|
+
const provisioned = await addWorktree(
|
|
518
552
|
r.repo,
|
|
519
553
|
project.mirrorRoot,
|
|
520
554
|
project.workspaceRoot,
|
|
521
555
|
issue,
|
|
522
556
|
branch,
|
|
523
557
|
);
|
|
558
|
+
worktreePath = provisioned.path;
|
|
524
559
|
|
|
525
560
|
// The SDK names the transcript itself, so the daemon supplies the parent
|
|
526
561
|
// directory and learns the real path back from the result. Inventing one
|
|
@@ -529,10 +564,16 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
529
564
|
mkdirSync(sessionDir, { recursive: true });
|
|
530
565
|
store.updateRun(runId, { worktree: worktreePath, state: "running" });
|
|
531
566
|
|
|
532
|
-
log(
|
|
567
|
+
log(
|
|
568
|
+
`#${issue} attempt ${attempt} → ${r.repo.name} ${branch}` +
|
|
569
|
+
(provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
|
|
570
|
+
);
|
|
533
571
|
|
|
534
572
|
const result = await runWorker({
|
|
535
|
-
brief: await buildBrief(project, r, branch, worktreePath
|
|
573
|
+
brief: await buildBrief(project, r, branch, worktreePath, {
|
|
574
|
+
continuation: provisioned.reattached,
|
|
575
|
+
defaultBranch: r.repo.defaultBranch,
|
|
576
|
+
}),
|
|
536
577
|
cwd: worktreePath,
|
|
537
578
|
caps,
|
|
538
579
|
sessionDir,
|
|
@@ -572,30 +613,61 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
572
613
|
detail: [`${r.issue.title}`, r.issue.url, "", result.report].join("\n"),
|
|
573
614
|
});
|
|
574
615
|
} else if (result.state === "failed" || result.state === "killed") {
|
|
575
|
-
|
|
576
|
-
//
|
|
577
|
-
//
|
|
616
|
+
// Turns-cap with attempts left: salvage, put the issue back on the queue,
|
|
617
|
+
// and skip the failed label so the next tick reclaims as a continuation
|
|
618
|
+
// instead of burning a human triage cycle (#50 / #21).
|
|
619
|
+
const continueTurns =
|
|
620
|
+
result.killedBy === "turns" && attempt < caps.maxAttemptsPerIssue;
|
|
578
621
|
const salvaged = await salvage(issue, attempt, endedBy(result.killedBy), worktreePath);
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
issue,
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
:
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
622
|
+
|
|
623
|
+
if (continueTurns) {
|
|
624
|
+
await tracker.removeLabel(issue, inProgress);
|
|
625
|
+
await tracker.addLabel(issue, project.queueLabel);
|
|
626
|
+
log(
|
|
627
|
+
`#${issue} turns-cap on attempt ${attempt}/${caps.maxAttemptsPerIssue} — ` +
|
|
628
|
+
`salvaged and re-queued for continuation`,
|
|
629
|
+
);
|
|
630
|
+
await safeEscalate(d, {
|
|
631
|
+
tier: 1,
|
|
632
|
+
project: project.name,
|
|
633
|
+
issue,
|
|
634
|
+
runId,
|
|
635
|
+
summary: `#${issue} hit the turns cap on attempt ${attempt} — auto-requeued for continuation`,
|
|
636
|
+
detail: [
|
|
637
|
+
`${r.issue.title}`,
|
|
638
|
+
r.issue.url,
|
|
639
|
+
...salvaged,
|
|
640
|
+
`Session: ${result.sessionFile ?? "(no transcript)"}`,
|
|
641
|
+
"",
|
|
642
|
+
"The queue label is back on; the next tick should reattach the branch",
|
|
643
|
+
"and open a continuation brief. No failed label was applied.",
|
|
644
|
+
"",
|
|
645
|
+
result.report,
|
|
646
|
+
].join("\n"),
|
|
647
|
+
});
|
|
648
|
+
} else {
|
|
649
|
+
await swapLabel(tracker, issue, inProgress, project.stateLabels.failed);
|
|
650
|
+
await safeEscalate(d, {
|
|
651
|
+
tier: 1,
|
|
652
|
+
project: project.name,
|
|
653
|
+
issue,
|
|
654
|
+
runId,
|
|
655
|
+
// The dedup key includes the summary, so the attempt number is what
|
|
656
|
+
// lets a genuine second failure page again while a tick that keeps
|
|
657
|
+
// seeing the same dead issue stays quiet.
|
|
658
|
+
summary: result.killedBy
|
|
659
|
+
? `#${issue} was killed on attempt ${attempt} by the ${result.killedBy} cap`
|
|
660
|
+
: `#${issue} failed on attempt ${attempt}`,
|
|
661
|
+
detail: [
|
|
662
|
+
`${r.issue.title}`,
|
|
663
|
+
r.issue.url,
|
|
664
|
+
...salvaged,
|
|
665
|
+
`Session: ${result.sessionFile ?? "(no transcript)"}`,
|
|
666
|
+
"",
|
|
667
|
+
result.report,
|
|
668
|
+
].join("\n"),
|
|
669
|
+
});
|
|
670
|
+
}
|
|
599
671
|
} else {
|
|
600
672
|
// pushed-green: the PR belongs to a human now. The in-progress label
|
|
601
673
|
// stays on until the merge closes the issue, which is also what keeps
|
|
@@ -923,9 +995,10 @@ async function tick(d: Deps): Promise<void> {
|
|
|
923
995
|
|
|
924
996
|
// Spend is the one cap that stops the fleet instead of merely deferring work.
|
|
925
997
|
// A loop that is burning money has to halt itself; waiting for a human to
|
|
926
|
-
// notice tomorrow is how a runaway becomes expensive.
|
|
998
|
+
// notice tomorrow is how a runaway becomes expensive. `null` means the
|
|
999
|
+
// operator opted out — turns and wall-clock still brake every run (#46).
|
|
927
1000
|
const spent = store.spendSince(project.name, since);
|
|
928
|
-
if (spent >= caps.dailySpendUsd) {
|
|
1001
|
+
if (caps.dailySpendUsd !== null && spent >= caps.dailySpendUsd) {
|
|
929
1002
|
setPaused(true);
|
|
930
1003
|
await safeEscalate(d, {
|
|
931
1004
|
tier: 2,
|
|
@@ -1011,7 +1084,9 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
1011
1084
|
"caps",
|
|
1012
1085
|
` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
|
|
1013
1086
|
` issues today ${s.runsToday}`,
|
|
1014
|
-
|
|
1087
|
+
s.caps.dailySpendUsd === null
|
|
1088
|
+
? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
|
|
1089
|
+
: ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
|
|
1015
1090
|
` worker max turns ${s.caps.workerMaxTurns}`,
|
|
1016
1091
|
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
1017
1092
|
` attempts per issue ${s.caps.maxAttemptsPerIssue}`,
|
|
@@ -1029,6 +1104,15 @@ export function formatStatus(s: StatusSnapshot): string {
|
|
|
1029
1104
|
);
|
|
1030
1105
|
}
|
|
1031
1106
|
}
|
|
1107
|
+
// Deploy hint: a restart while workers are live orphans them (salvage runs
|
|
1108
|
+
// first — #35). Prefer pause + drain to zero live workers when you can wait.
|
|
1109
|
+
if (s.liveWorkers > 0) {
|
|
1110
|
+
lines.push(
|
|
1111
|
+
"",
|
|
1112
|
+
`deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
|
|
1113
|
+
`pause and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
|
|
1114
|
+
);
|
|
1115
|
+
}
|
|
1032
1116
|
return lines.join("\n");
|
|
1033
1117
|
}
|
|
1034
1118
|
|
|
@@ -1085,18 +1169,18 @@ export function armConductor(): void {
|
|
|
1085
1169
|
}
|
|
1086
1170
|
|
|
1087
1171
|
/**
|
|
1088
|
-
* Settles
|
|
1172
|
+
* Settles `claimed`/`running` rows left by a dead daemon process and, before
|
|
1173
|
+
* marking each one `orphaned`, salvages any dirty worktree.
|
|
1089
1174
|
*
|
|
1090
|
-
*
|
|
1091
|
-
*
|
|
1092
|
-
*
|
|
1093
|
-
*
|
|
1094
|
-
*
|
|
1095
|
-
* (found live, after a host restart killed two workers mid-run).
|
|
1175
|
+
* Found live after a host restart killed two workers mid-run, and again on
|
|
1176
|
+
* every package deploy that restarted while workers were live (#35): without
|
|
1177
|
+
* the salvage call the next attempt's `worktree remove --force` destroyed
|
|
1178
|
+
* uncommitted edits that had no other copy. Cap-kills already salvaged (#27);
|
|
1179
|
+
* this is the same call site for the restart path.
|
|
1096
1180
|
*
|
|
1097
1181
|
* Only the rows change. The issue keeps its in-progress label — that label is
|
|
1098
1182
|
* the crash guard against double-dispatch, and deciding what a dead worker's
|
|
1099
|
-
* remains are worth (an open PR?
|
|
1183
|
+
* remains are worth (an open PR? a salvaged sha? a clean tree?) is the
|
|
1100
1184
|
* orchestrator's drain-duty judgement, not something to automate here. The
|
|
1101
1185
|
* rows also keep counting toward `maxAttemptsPerIssue`, so a loop of deaths
|
|
1102
1186
|
* still escalates instead of retrying forever.
|
|
@@ -1107,12 +1191,21 @@ export function armConductor(): void {
|
|
|
1107
1191
|
* them is {@link settlePushedGreen}, on the tick, by asking the tracker what
|
|
1108
1192
|
* became of the PR — the one question a restart cannot answer by inference.
|
|
1109
1193
|
*/
|
|
1110
|
-
export function reconcileOrphanedRuns(
|
|
1194
|
+
export async function reconcileOrphanedRuns(
|
|
1195
|
+
store: Store,
|
|
1196
|
+
project: string,
|
|
1197
|
+
): Promise<RunRecord[]> {
|
|
1111
1198
|
// Live runs only: `pushed-green` holds no process, so it cannot be orphaned by
|
|
1112
1199
|
// a process dying — it is finished work waiting on a human merge.
|
|
1113
1200
|
const stale = store.liveRuns(project);
|
|
1114
1201
|
const endedAt = Date.now();
|
|
1115
1202
|
for (const r of stale) {
|
|
1203
|
+
// Salvage before the row flips: the worktree path is on the record, and
|
|
1204
|
+
// salvageWip is a no-op for a missing/clean tree. Reason string matches
|
|
1205
|
+
// the cap-kill wording so triage reads the same either way.
|
|
1206
|
+
if (r.worktree !== "") {
|
|
1207
|
+
await salvage(r.issue, r.attempt, "a daemon restart", r.worktree);
|
|
1208
|
+
}
|
|
1116
1209
|
store.updateRun(r.id, { state: "orphaned", endedAt });
|
|
1117
1210
|
}
|
|
1118
1211
|
return stale;
|
|
@@ -1141,7 +1234,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1141
1234
|
// daemon must not orphan that daemon's real, live workers).
|
|
1142
1235
|
const alive = livingDaemon();
|
|
1143
1236
|
if (alive === undefined || alive.pid === process.pid) {
|
|
1144
|
-
for (const r of reconcileOrphanedRuns(store, project.name)) {
|
|
1237
|
+
for (const r of await reconcileOrphanedRuns(store, project.name)) {
|
|
1145
1238
|
log(
|
|
1146
1239
|
`#${r.issue} orphaned by a previous daemon (attempt ${r.attempt}, was ${r.state}, worktree ${r.worktree}) — ` +
|
|
1147
1240
|
`slot freed; the ${project.stateLabels.inProgress} label stays until the orchestrator triages what the worker left`,
|
|
@@ -1283,5 +1376,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
1283
1376
|
await orchestrator?.dispose();
|
|
1284
1377
|
store.close();
|
|
1285
1378
|
log("stopped");
|
|
1379
|
+
// A handled SIGTERM still leaves some runtimes with a non-zero default
|
|
1380
|
+
// (historically 128+signal). Under systemd `Restart=on-failure` that looks
|
|
1381
|
+
// like a crash and the unit comes straight back — the exact failure mode
|
|
1382
|
+
// `omp-conductor stop` hit on the reference fleet. Force success so a
|
|
1383
|
+
// graceful drain is not a restart.
|
|
1384
|
+
process.exitCode = 0;
|
|
1286
1385
|
}
|
|
1287
1386
|
}
|
package/src/escalate.ts
CHANGED
|
@@ -45,11 +45,21 @@ export interface Escalator {
|
|
|
45
45
|
* send, turning a cosmetic problem into a lost escalation. Plain text is also
|
|
46
46
|
* valid Markdown, so the same string renders fine as an issue comment.
|
|
47
47
|
*/
|
|
48
|
+
/**
|
|
49
|
+
* Fleet-scoped pages (integrity tripwire, spend-cap halt, stall) use issue `0`
|
|
50
|
+
* as a sentinel — there is no tracker issue. Rendering that as `#0` made
|
|
51
|
+
* Telegram pages and daemon logs look like a bug (#52). Keep the sentinel in
|
|
52
|
+
* the typed field; only the human-facing label changes here.
|
|
53
|
+
*/
|
|
54
|
+
export function escalationIssueRef(issue: number): string {
|
|
55
|
+
return issue === 0 ? "fleet" : `#${issue}`;
|
|
56
|
+
}
|
|
57
|
+
|
|
48
58
|
export function formatEscalation(e: Escalation, project: string): string {
|
|
49
59
|
const lines = [
|
|
50
60
|
`omp-conductor · tier ${e.tier} escalation`,
|
|
51
61
|
`project: ${project}`,
|
|
52
|
-
`issue:
|
|
62
|
+
`issue: ${escalationIssueRef(e.issue)}`,
|
|
53
63
|
`summary: ${e.summary}`,
|
|
54
64
|
];
|
|
55
65
|
if (e.detail) lines.push(`detail: ${e.detail}`);
|
|
@@ -99,18 +109,21 @@ export function createEscalator(
|
|
|
99
109
|
const onTurnFailed = async (cause: unknown): Promise<void> => {
|
|
100
110
|
if (!p.escalation.fallbackToIssueComment) {
|
|
101
111
|
warn(
|
|
102
|
-
`tier 1 escalation on
|
|
112
|
+
`tier 1 escalation on ${escalationIssueRef(e.issue)} was accepted by the orchestrator but its ` +
|
|
103
113
|
`turn failed (${errText(cause)}), and no fallback is configured — left unmarked ` +
|
|
104
114
|
`so the next tick retries`,
|
|
105
115
|
);
|
|
106
116
|
return;
|
|
107
117
|
}
|
|
108
118
|
try {
|
|
119
|
+
if (e.issue === 0) {
|
|
120
|
+
throw new Error("fleet-scoped escalation has no issue to comment on");
|
|
121
|
+
}
|
|
109
122
|
await tracker.comment(e.issue, text);
|
|
110
123
|
store.markNotified(key);
|
|
111
124
|
} catch (err) {
|
|
112
125
|
warn(
|
|
113
|
-
`tier 1 escalation on
|
|
126
|
+
`tier 1 escalation on ${escalationIssueRef(e.issue)} failed after acceptance ` +
|
|
114
127
|
`(${errText(cause)}) and its issue-comment fallback failed too ` +
|
|
115
128
|
`(${errText(err)}) — left unmarked so the next tick retries`,
|
|
116
129
|
);
|
|
@@ -154,10 +167,19 @@ export function createEscalator(
|
|
|
154
167
|
if (!p.escalation.fallbackToIssueComment) {
|
|
155
168
|
throw new Error(
|
|
156
169
|
`no escalation transport configured for project "${p.name}": tier ${e.tier} ` +
|
|
157
|
-
`escalation on
|
|
170
|
+
`escalation on ${escalationIssueRef(e.issue)} (${e.summary}) could not be delivered — ` +
|
|
158
171
|
`set escalation.telegramChatId or escalation.fallbackToIssueComment`,
|
|
159
172
|
);
|
|
160
173
|
}
|
|
174
|
+
// Fleet pages (issue 0) have nowhere to comment. Falling through here used
|
|
175
|
+
// to call `issues/0/comments` and surface as a confusing "#0" delivery
|
|
176
|
+
// failure after Telegram had already been tried (#52).
|
|
177
|
+
if (e.issue === 0) {
|
|
178
|
+
throw new Error(
|
|
179
|
+
`fleet-scoped tier ${e.tier} escalation (${e.summary}) has no issue to comment on — ` +
|
|
180
|
+
`configure escalation.telegramChatId so integrity/spend/stall pages can reach an operator`,
|
|
181
|
+
);
|
|
182
|
+
}
|
|
161
183
|
await tracker.comment(e.issue, text);
|
|
162
184
|
store.markNotified(key);
|
|
163
185
|
},
|
package/src/lifecycle.ts
CHANGED
|
@@ -10,16 +10,30 @@
|
|
|
10
10
|
* operator real time are `start` refusing against a ghost and `status`
|
|
11
11
|
* reporting a daemon that died hours ago.
|
|
12
12
|
*
|
|
13
|
+
* When the live pid is the MainPID of the `omp-conductor.service` unit,
|
|
14
|
+
* `stop`/`restart` go through `systemctl` rather than a raw `SIGTERM`. A
|
|
15
|
+
* raw signal against a unit with `Restart=on-failure` is read as a crash and
|
|
16
|
+
* the unit comes straight back — the bug that made `omp-conductor stop` look
|
|
17
|
+
* like a no-op on a systemd-managed host.
|
|
18
|
+
*
|
|
13
19
|
* Deliberately free of every other module in this package: nothing here opens
|
|
14
20
|
* the store, loads the config or talks to `gh`, so `stop` and `status` keep
|
|
15
21
|
* working when the config is the very thing that is broken.
|
|
16
22
|
*/
|
|
17
23
|
|
|
18
|
-
import { spawn } from "node:child_process";
|
|
24
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
19
25
|
import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
20
26
|
import { homedir } from "node:os";
|
|
21
27
|
import { join } from "node:path";
|
|
22
28
|
|
|
29
|
+
/**
|
|
30
|
+
* The systemd unit name operators are expected to install for a supervised
|
|
31
|
+
* daemon. Hardcoded rather than configured: one name on the host is the
|
|
32
|
+
* whole point of a unit, and a wrong name would silently fall back to the
|
|
33
|
+
* SIGTERM path and reintroduce the restart loop this module exists to avoid.
|
|
34
|
+
*/
|
|
35
|
+
export const SYSTEMD_UNIT = "omp-conductor.service";
|
|
36
|
+
|
|
23
37
|
/**
|
|
24
38
|
* Mirrors `DEFAULT_PORT` in ./daemon.ts. Duplicated rather than imported so
|
|
25
39
|
* this module stays free of the dispatcher's dependency tree; exported so the
|
|
@@ -271,23 +285,67 @@ export async function startDaemon(o: { port?: number; project?: string } = {}):
|
|
|
271
285
|
}
|
|
272
286
|
|
|
273
287
|
/**
|
|
274
|
-
*
|
|
288
|
+
* How the last stop actually landed. Callers print this so an operator can
|
|
289
|
+
* tell a supervised stop from a bare SIGTERM without reading the journal.
|
|
290
|
+
*/
|
|
291
|
+
export type StopResult =
|
|
292
|
+
| { kind: "stopped"; pid: number; via: "systemctl" | "signal" }
|
|
293
|
+
| { kind: "not-running" };
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Stops the daemon.
|
|
275
297
|
*
|
|
276
|
-
* `
|
|
277
|
-
*
|
|
278
|
-
*
|
|
279
|
-
*
|
|
280
|
-
*
|
|
281
|
-
*
|
|
298
|
+
* Prefer `systemctl stop omp-conductor.service` when that unit's MainPID is
|
|
299
|
+
* the live daemon: systemd then owns the stop and will not schedule a restart
|
|
300
|
+
* for the exit it just requested. A raw `SIGTERM` against a unit with
|
|
301
|
+
* `Restart=on-failure` is what made `omp-conductor stop` look like a no-op on
|
|
302
|
+
* the reference fleet — exit 143 was a "failure", and the unit came back five
|
|
303
|
+
* seconds later.
|
|
304
|
+
*
|
|
305
|
+
* Falls back to `SIGTERM` then `SIGKILL` when there is no unit, the unit is
|
|
306
|
+
* not running, or its MainPID is somebody else (a hand-started daemon next to
|
|
307
|
+
* a stopped unit). The grace period is a deadline, not a clean drain: a tick
|
|
308
|
+
* with a worker in flight can run for that worker's whole wall clock.
|
|
282
309
|
*/
|
|
283
|
-
export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<
|
|
310
|
+
export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopResult> {
|
|
284
311
|
const rec = livingDaemon();
|
|
285
312
|
if (rec === undefined) {
|
|
286
313
|
// `livingDaemon` already cleared a stale file; this covers the unparseable
|
|
287
|
-
// one it refused to read.
|
|
314
|
+
// one it refused to read. Still ask systemd: a unit can be running with
|
|
315
|
+
// no pidfile (boot race, wiped runtime dir) and stop must still land.
|
|
316
|
+
const unitPid = systemdMainPid();
|
|
317
|
+
if (unitPid !== undefined && (await stopViaSystemd(undefined))) {
|
|
318
|
+
const deadline = Date.now() + (o.timeoutMs ?? STOP_TIMEOUT_MS);
|
|
319
|
+
while (isAlive(unitPid) && Date.now() < deadline) await sleep(100);
|
|
320
|
+
if (isAlive(unitPid)) {
|
|
321
|
+
throw new Error(
|
|
322
|
+
`daemon pid ${unitPid} still alive after systemctl stop ${SYSTEMD_UNIT} — ` +
|
|
323
|
+
`check \`systemctl status ${SYSTEMD_UNIT}\``,
|
|
324
|
+
);
|
|
325
|
+
}
|
|
326
|
+
clearRecord();
|
|
327
|
+
return { kind: "stopped", pid: unitPid, via: "systemctl" };
|
|
328
|
+
}
|
|
288
329
|
clearRecord();
|
|
289
|
-
return "not-running";
|
|
330
|
+
return { kind: "not-running" };
|
|
290
331
|
}
|
|
332
|
+
|
|
333
|
+
if (await stopViaSystemd(rec.pid)) {
|
|
334
|
+
// Wait out the unit: systemctl stop is synchronous for Type=simple, but
|
|
335
|
+
// a slow drain still holds the old pid briefly and the next start would
|
|
336
|
+
// refuse against it.
|
|
337
|
+
const deadline = Date.now() + (o.timeoutMs ?? STOP_TIMEOUT_MS);
|
|
338
|
+
while (isAlive(rec.pid) && Date.now() < deadline) await sleep(100);
|
|
339
|
+
if (isAlive(rec.pid)) {
|
|
340
|
+
throw new Error(
|
|
341
|
+
`daemon pid ${rec.pid} still alive after systemctl stop ${SYSTEMD_UNIT} — ` +
|
|
342
|
+
`check \`systemctl status ${SYSTEMD_UNIT}\``,
|
|
343
|
+
);
|
|
344
|
+
}
|
|
345
|
+
clearRecord();
|
|
346
|
+
return { kind: "stopped", pid: rec.pid, via: "systemctl" };
|
|
347
|
+
}
|
|
348
|
+
|
|
291
349
|
const gone = await terminate(rec.pid, o.timeoutMs ?? STOP_TIMEOUT_MS);
|
|
292
350
|
if (!gone) {
|
|
293
351
|
// The record stays: something is still holding that pid, and forgetting
|
|
@@ -295,13 +353,146 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<"stopp
|
|
|
295
353
|
throw new Error(`daemon pid ${rec.pid} survived SIGTERM and SIGKILL — it may belong to another user`);
|
|
296
354
|
}
|
|
297
355
|
clearRecord();
|
|
298
|
-
return "stopped";
|
|
356
|
+
return { kind: "stopped", pid: rec.pid, via: "signal" };
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* Restarts the daemon.
|
|
361
|
+
*
|
|
362
|
+
* Same ownership rule as {@link stopDaemon}: when the unit owns the live pid,
|
|
363
|
+
* `systemctl restart` keeps systemd in charge of the replacement process so
|
|
364
|
+
* the new MainPID is still the unit's. Falling back to stop+start for a
|
|
365
|
+
* hand-started daemon would leave the unit dead and the new process
|
|
366
|
+
* unsupervised — fine for a laptop, wrong for the host that installed a unit
|
|
367
|
+
* specifically so a crash comes back.
|
|
368
|
+
*
|
|
369
|
+
* Returns the record of the process that is now answering `/healthz`.
|
|
370
|
+
*/
|
|
371
|
+
export async function restartDaemon(
|
|
372
|
+
o: { port?: number; project?: string; timeoutMs?: number } = {},
|
|
373
|
+
): Promise<{ previous: DaemonRecord | undefined; record: DaemonRecord; via: "systemctl" | "cli" }> {
|
|
374
|
+
const previous = livingDaemon();
|
|
375
|
+
const unitPid = systemdMainPid();
|
|
376
|
+
const unitOwns =
|
|
377
|
+
unitPid !== undefined && (previous === undefined || previous.pid === unitPid);
|
|
378
|
+
|
|
379
|
+
if (unitOwns) {
|
|
380
|
+
const ran = systemctl(["restart", SYSTEMD_UNIT]);
|
|
381
|
+
if (ran.ok) {
|
|
382
|
+
// systemctl restart returns once the new MainPID is up; the pidfile is
|
|
383
|
+
// written by the daemon itself on boot, so wait for that rather than
|
|
384
|
+
// inventing a record from the unit alone.
|
|
385
|
+
const deadline = Date.now() + READY_TIMEOUT_MS;
|
|
386
|
+
for (;;) {
|
|
387
|
+
const rec = livingDaemon();
|
|
388
|
+
if (rec !== undefined) {
|
|
389
|
+
const health = await healthCheck(rec.port);
|
|
390
|
+
if (health.ok) return { previous, record: rec, via: "systemctl" };
|
|
391
|
+
}
|
|
392
|
+
if (Date.now() >= deadline) break;
|
|
393
|
+
await sleep(READY_POLL_MS);
|
|
394
|
+
}
|
|
395
|
+
throw new Error(
|
|
396
|
+
`systemctl restart ${SYSTEMD_UNIT} returned, but the daemon never answered /healthz`,
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
// Unit exists and owns the pid but systemctl refused (permissions, dbus
|
|
400
|
+
// down). Fall through to the signal path rather than stranding the
|
|
401
|
+
// operator with "restart failed" and a still-running daemon they cannot
|
|
402
|
+
// reach through the unit.
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
await stopDaemon({ timeoutMs: o.timeoutMs });
|
|
406
|
+
const record = await startDaemon({ port: o.port ?? previous?.port, project: o.project ?? previous?.project });
|
|
407
|
+
return { previous, record, via: "cli" };
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/**
|
|
411
|
+
* The MainPID of {@link SYSTEMD_UNIT}, or `undefined` when systemd is absent,
|
|
412
|
+
* the unit is unknown, or it is not running. Never throws: a missing binary
|
|
413
|
+
* or a dbus blip is "no unit", and stop falls back to SIGTERM.
|
|
414
|
+
*/
|
|
415
|
+
export function systemdMainPid(unit = SYSTEMD_UNIT): number | undefined {
|
|
416
|
+
const ran = systemctl(["show", unit, "--property=MainPID", "--property=ActiveState", "--value"]);
|
|
417
|
+
if (!ran.ok) return undefined;
|
|
418
|
+
// `systemctl show --value` prints one property per line, MainPID then
|
|
419
|
+
// ActiveState, in the order requested. Tolerate either order and blank
|
|
420
|
+
// lines so a future systemctl rearrange does not silently disable the path.
|
|
421
|
+
const lines = ran.stdout
|
|
422
|
+
.split("\n")
|
|
423
|
+
.map((l) => l.trim())
|
|
424
|
+
.filter((l) => l.length > 0);
|
|
425
|
+
let pid: number | undefined;
|
|
426
|
+
let active: string | undefined;
|
|
427
|
+
for (const line of lines) {
|
|
428
|
+
if (/^\d+$/.test(line)) {
|
|
429
|
+
const n = Number(line);
|
|
430
|
+
if (Number.isInteger(n) && n > 1) pid = n;
|
|
431
|
+
} else {
|
|
432
|
+
active = line;
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
if (active !== undefined && active !== "active" && active !== "reactivating") return undefined;
|
|
436
|
+
return pid;
|
|
299
437
|
}
|
|
300
438
|
|
|
301
439
|
// ---------------------------------------------------------------------------
|
|
302
440
|
// internals
|
|
303
441
|
// ---------------------------------------------------------------------------
|
|
304
442
|
|
|
443
|
+
/**
|
|
444
|
+
* Ask systemd to stop the unit, but only when it actually owns `pid`.
|
|
445
|
+
*
|
|
446
|
+
* `pid === undefined` means "no pidfile" — still stop the unit if it is
|
|
447
|
+
* active, because that is the only process that could be the daemon. A unit
|
|
448
|
+
* whose MainPID is some other process is left alone: stopping it would take
|
|
449
|
+
* down a neighbour, and the signal path handles *our* pid.
|
|
450
|
+
*/
|
|
451
|
+
async function stopViaSystemd(pid: number | undefined): Promise<boolean> {
|
|
452
|
+
const main = systemdMainPid();
|
|
453
|
+
if (main === undefined) return false;
|
|
454
|
+
if (pid !== undefined && main !== pid) return false;
|
|
455
|
+
const ran = systemctl(["stop", SYSTEMD_UNIT]);
|
|
456
|
+
return ran.ok;
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/**
|
|
460
|
+
* Run one `systemctl` invocation. Captures output and never throws: absence
|
|
461
|
+
* of the binary, a missing unit, or a permission error are all "not ok", and
|
|
462
|
+
* the caller decides whether to fall back.
|
|
463
|
+
*
|
|
464
|
+
* The default shells out. Tests replace it with {@link setSystemctlForTest}
|
|
465
|
+
* so the ownership decision is exercised without a real systemd.
|
|
466
|
+
*/
|
|
467
|
+
export type SystemctlFn = (args: string[]) => { ok: boolean; stdout: string; stderr: string };
|
|
468
|
+
|
|
469
|
+
function defaultSystemctl(args: string[]): { ok: boolean; stdout: string; stderr: string } {
|
|
470
|
+
try {
|
|
471
|
+
const res = spawnSync("systemctl", args, {
|
|
472
|
+
encoding: "utf8",
|
|
473
|
+
// A hung dbus is not worth blocking stop on; the signal path is right there.
|
|
474
|
+
timeout: 15_000,
|
|
475
|
+
env: process.env,
|
|
476
|
+
});
|
|
477
|
+
if (res.error) return { ok: false, stdout: "", stderr: res.error.message };
|
|
478
|
+
return {
|
|
479
|
+
ok: res.status === 0,
|
|
480
|
+
stdout: res.stdout ?? "",
|
|
481
|
+
stderr: res.stderr ?? "",
|
|
482
|
+
};
|
|
483
|
+
} catch (err) {
|
|
484
|
+
return { ok: false, stdout: "", stderr: err instanceof Error ? err.message : String(err) };
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
let systemctl: SystemctlFn = defaultSystemctl;
|
|
489
|
+
|
|
490
|
+
/** Test-only: replace the `systemctl` runner. Pass `undefined` to restore. */
|
|
491
|
+
export function setSystemctlForTest(fn: SystemctlFn | undefined): void {
|
|
492
|
+
systemctl = fn ?? defaultSystemctl;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
|
|
305
496
|
function sleep(ms: number): Promise<void> {
|
|
306
497
|
return new Promise<void>((resolve) => {
|
|
307
498
|
const t = setTimeout(resolve, ms);
|
package/src/omp.ts
CHANGED
|
@@ -162,6 +162,11 @@ export async function createSession(opts: {
|
|
|
162
162
|
// sharing it fails to start. The daemon runs `maxConcurrentWorkers`
|
|
163
163
|
// (2 by default) workers at once, which makes this the normal path.
|
|
164
164
|
agentRegistry: new mod.AgentRegistry(),
|
|
165
|
+
// Default true in the SDK, but pass it explicitly so a harness change
|
|
166
|
+
// cannot silently strip MCP from workers. Discovery walks cwd + user
|
|
167
|
+
// agentDir (~/.omp/agent/mcp.json) — without this, workers grep-only and
|
|
168
|
+
// burn the turns cap on discovery (#29).
|
|
169
|
+
enableMCP: true,
|
|
165
170
|
});
|
|
166
171
|
const raw = asRawSession(created);
|
|
167
172
|
// Surfaced rather than swallowed: this is how a quiet downgrade to a weaker
|