omp-conductor 0.19.5 → 0.19.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli.ts +2 -0
- package/src/command-help.ts +18 -8
- package/src/command-manifest.ts +22 -5
- package/src/commands/context.ts +1 -0
- package/src/commands/report.ts +63 -0
- package/src/commands/restore-db.ts +1 -1
- package/src/commands/snapshot-db.ts +67 -0
- package/src/commands/upgrade-install.ts +1 -1
- package/src/commands/upgrade.ts +2 -2
- package/src/commands/verb.ts +1 -0
- package/src/commands/watch.ts +14 -2
- package/src/config.ts +17 -0
- package/src/daemon.ts +23 -3
- package/src/doctor.ts +36 -3
- package/src/fleet.ts +51 -3
- package/src/orchestrator-tick.ts +14 -3
- package/src/setup-host.ts +10 -10
- package/src/setup-wizard.ts +1 -1
- package/src/status-render.ts +10 -0
- package/src/store.ts +296 -22
- package/src/types.ts +32 -3
- package/src/upgrade-journal.ts +35 -1
- package/src/upgrade-verify.ts +27 -6
- package/src/upgrade.ts +301 -22
- package/src/verbs/actions.ts +6 -1
- package/src/verbs/server.ts +54 -20
- package/systemd/omp-conductor-recover.sh +55 -7
- package/systemd/recover-unit-test.sh +99 -6
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "omp-conductor",
|
|
3
|
-
"version": "0.19.
|
|
3
|
+
"version": "0.19.6",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",
|
package/src/cli.ts
CHANGED
|
@@ -32,6 +32,7 @@ import { reportCommand } from "./commands/report.ts";
|
|
|
32
32
|
import { releaseCompositionCommand } from "./commands/release-composition.ts";
|
|
33
33
|
import { restartCommand } from "./commands/restart.ts";
|
|
34
34
|
import { restoreDbCommand } from "./commands/restore-db.ts";
|
|
35
|
+
import { snapshotDbCommand } from "./commands/snapshot-db.ts";
|
|
35
36
|
import { resumeCommand } from "./commands/resume.ts";
|
|
36
37
|
import { setupCommand } from "./commands/setup.ts";
|
|
37
38
|
import { startCommand } from "./commands/start.ts";
|
|
@@ -300,6 +301,7 @@ export function commandHandlers(ctx: CommandContext): Record<string, CommandHand
|
|
|
300
301
|
intake: () => intakeCommand(ctx),
|
|
301
302
|
friction: () => frictionCommand(ctx),
|
|
302
303
|
"restore-db": () => restoreDbCommand(ctx),
|
|
304
|
+
"snapshot-db": () => snapshotDbCommand(ctx),
|
|
303
305
|
resume: () => resumeCommand(ctx),
|
|
304
306
|
"brief-upgrade": () => briefUpgradeCommand(ctx),
|
|
305
307
|
help: () => helpCommand(USAGE),
|
package/src/command-help.ts
CHANGED
|
@@ -6,10 +6,11 @@ export const COMMAND_DETAILS = ` setup interview, then write config.json, th
|
|
|
6
6
|
the project already exists, a chooser of which area to amend. Naming
|
|
7
7
|
an area positionally skips that chooser and amends only that area:
|
|
8
8
|
${AMEND_AREA_IDS.join(", ")}.
|
|
9
|
-
upgrade
|
|
10
|
-
brief as one pinned release.
|
|
11
|
-
workers, reloads, verifies twice,
|
|
12
|
-
state. Run it from a shell outside
|
|
9
|
+
upgrade discover the installed CLI package root, then update it with the omp
|
|
10
|
+
plugin, Herdr recovery plugin, and brief as one pinned release.
|
|
11
|
+
Pauses only new claims, drains live workers, reloads, verifies twice,
|
|
12
|
+
and restores the prior dispatch state. Run it from a shell outside
|
|
13
|
+
the target Herdr session.
|
|
13
14
|
--bootstrap SHA --source PATH installs an exact commit instead of a
|
|
14
15
|
published version, for when the installed conductor is what is
|
|
15
16
|
broken: the source must be a checkout AT that sha, the package's own
|
|
@@ -66,6 +67,10 @@ export const COMMAND_DETAILS = ` setup interview, then write config.json, th
|
|
|
66
67
|
--probe-telegram sends through the report transport. Exit code 0
|
|
67
68
|
only when nothing failed; --json prints the stable CI shape. Run
|
|
68
69
|
it after install and after every upgrade.
|
|
70
|
+
snapshot-db
|
|
71
|
+
publish a transaction-consistent conductor.db snapshot with SQLite
|
|
72
|
+
VACUUM INTO, record it in the durable event ledger, and apply the
|
|
73
|
+
daemon cadence's bounded retention. Safe while the daemon writes.
|
|
69
74
|
restore-db
|
|
70
75
|
put conductor.db back to a restorable snapshot, replacing the live
|
|
71
76
|
file and dropping stale -wal/-shm sidecars. Refuses while a daemon
|
|
@@ -153,6 +158,9 @@ export const COMMAND_DETAILS = ` setup interview, then write config.json, th
|
|
|
153
158
|
sending now and holding, exactly as for a daemon escalation. An
|
|
154
159
|
identical retry is refused only while the earlier handoff is still
|
|
155
160
|
undelivered, and admitted again once it lands (#453).
|
|
161
|
+
report withdraw removes only an unsent pending report or unassigned
|
|
162
|
+
held notice, requires a reason, preserves an audit row, and refuses
|
|
163
|
+
sending/delivered/failed/already-withdrawn handoffs.
|
|
156
164
|
message deliver one direct Telegram message to this project's own chat and
|
|
157
165
|
forum topic, resolved from config rather than from whichever chat
|
|
158
166
|
last wrote to the session. This is how a locally injected tick
|
|
@@ -180,10 +188,12 @@ export const COMMAND_DETAILS = ` setup interview, then write config.json, th
|
|
|
180
188
|
watch set a condition or carry note for the orchestrator itself, with no
|
|
181
189
|
human in the loop — the same no-sleep surface \`decision open
|
|
182
190
|
--resolves-when\` used to serve, given its own verb so it never renders
|
|
183
|
-
as a question put to the operator.
|
|
184
|
-
condition
|
|
185
|
-
|
|
186
|
-
|
|
191
|
+
as a question put to the operator. The running daemon checks a
|
|
192
|
+
\`--resolves-when\` condition; an armed or ungated heartbeat wakes the
|
|
193
|
+
orchestrator tick when one is met. With no daemon the durable row is
|
|
194
|
+
unmonitored. With ticks disarmed the condition may still be marked met,
|
|
195
|
+
but no tick can be woken until the heartbeat returns. It is listed
|
|
196
|
+
under its own heading, never under "Open operator decisions", and has
|
|
187
197
|
no seven-day expiry. \`watch list\` shows open watches; \`watch
|
|
188
198
|
withdraw <id>\` ends one with a recorded reason — the verb that
|
|
189
199
|
creates a watch is the verb that ends it (watches are decision rows,
|
package/src/command-manifest.ts
CHANGED
|
@@ -202,6 +202,17 @@ export const COMMAND_MANIFEST: readonly CommandManifestEntry[] = [
|
|
|
202
202
|
toggle("--probe-telegram", "send the self-identified Telegram health probe"),
|
|
203
203
|
],
|
|
204
204
|
},
|
|
205
|
+
{
|
|
206
|
+
name: "snapshot-db",
|
|
207
|
+
description: "publish a transaction-consistent conductor.db snapshot",
|
|
208
|
+
scope: "host",
|
|
209
|
+
usage: ["snapshot-db"],
|
|
210
|
+
flags: [],
|
|
211
|
+
details:
|
|
212
|
+
"Publish one transaction-consistent conductor.db snapshot using SQLite VACUUM INTO, " +
|
|
213
|
+
"record the action in every configured project's durable event ledger, and apply the " +
|
|
214
|
+
"daemon cadence's bounded retention. Safe while the daemon is writing.",
|
|
215
|
+
},
|
|
205
216
|
{
|
|
206
217
|
name: "restore-db",
|
|
207
218
|
description: "restore conductor.db from a safe snapshot",
|
|
@@ -403,14 +414,18 @@ export const COMMAND_MANIFEST: readonly CommandManifestEntry[] = [
|
|
|
403
414
|
scope: "project",
|
|
404
415
|
usage: [
|
|
405
416
|
"report --text TEXT [--kind material|digest|tier2|decision-needed|fleet-stopped|confirmed-failure] [--events IDS] [--notices IDS] [--project NAME]",
|
|
417
|
+
"report withdraw <id> --reason TEXT [--project NAME]",
|
|
406
418
|
],
|
|
419
|
+
subcommands: [{ name: "withdraw", description: "withdraw an unsent report or held notice" }],
|
|
407
420
|
flags: [
|
|
408
421
|
value("--text", "rendered report text"),
|
|
409
422
|
value("--kind", "material, digest, or escalation category"),
|
|
410
423
|
value("--events", "comma-separated event ledger ids"),
|
|
411
424
|
value("--notices", "comma-separated notice ledger ids"),
|
|
425
|
+
value("--reason", "why a pending handoff is obsolete"),
|
|
412
426
|
project(),
|
|
413
427
|
],
|
|
428
|
+
positionals: [{ name: "action" }, { name: "id" }],
|
|
414
429
|
},
|
|
415
430
|
{
|
|
416
431
|
name: "message",
|
|
@@ -460,11 +475,13 @@ export const COMMAND_MANIFEST: readonly CommandManifestEntry[] = [
|
|
|
460
475
|
"watch list [--project NAME] [--json]",
|
|
461
476
|
"watch withdraw <id> [--reason TEXT] [--project NAME]",
|
|
462
477
|
],
|
|
463
|
-
details: `add records a row
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
478
|
+
details: `add records a durable row with no operator answer needed. The running daemon
|
|
479
|
+
checks conditions; an armed or ungated heartbeat wakes the orchestrator tick when one is met.
|
|
480
|
+
With no daemon the row is unmonitored. With ticks disarmed the daemon may still mark a
|
|
481
|
+
condition met, but no tick can be woken until the heartbeat returns. list shows open watches,
|
|
482
|
+
oldest first. withdraw ends one with a recorded reason — the verb that creates a watch is the
|
|
483
|
+
verb that ends it. A watch whose PR condition can no longer be observed (the PR merged or
|
|
484
|
+
closed first) is withdrawn by the daemon itself.`,
|
|
468
485
|
subcommands: [
|
|
469
486
|
{ name: "add", description: "record a watch" },
|
|
470
487
|
{ name: "list", description: "list open watches" },
|
package/src/commands/context.ts
CHANGED
package/src/commands/report.ts
CHANGED
|
@@ -22,7 +22,70 @@ import {
|
|
|
22
22
|
type ReportKind,
|
|
23
23
|
} from "../types.ts";
|
|
24
24
|
|
|
25
|
+
async function withdrawHandoff(ctx: CommandContext): Promise<void> {
|
|
26
|
+
const id = ctx.argv[2];
|
|
27
|
+
const reason = ctx.flag("reason")?.trim();
|
|
28
|
+
if (id === undefined || id.startsWith("--")) {
|
|
29
|
+
process.stderr.write("omp-conductor: report withdraw needs the report or held-notice id\n");
|
|
30
|
+
process.exit(2);
|
|
31
|
+
}
|
|
32
|
+
if (reason === undefined || reason.length === 0 || reason.startsWith("--")) {
|
|
33
|
+
process.stderr.write("omp-conductor: report withdraw needs --reason with why the handoff is obsolete\n");
|
|
34
|
+
process.exit(2);
|
|
35
|
+
}
|
|
36
|
+
for (let i = 3; i < ctx.argv.length; i++) {
|
|
37
|
+
const token = ctx.argv[i];
|
|
38
|
+
if (token === "--project" || token === "--reason") {
|
|
39
|
+
i += 1;
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
if (token?.startsWith("--project=") || token?.startsWith("--reason=")) continue;
|
|
43
|
+
process.stderr.write(`omp-conductor: report withdraw: unexpected argument "${token ?? ""}"\n`);
|
|
44
|
+
process.exit(2);
|
|
45
|
+
}
|
|
46
|
+
const sessionRole = process.env[SESSION_ROLE_ENV];
|
|
47
|
+
if (sessionRole === "worker") {
|
|
48
|
+
process.stderr.write("omp-conductor: report withdraw is an operator/orchestrator action, not a worker action\n");
|
|
49
|
+
process.exit(3);
|
|
50
|
+
}
|
|
51
|
+
const project = findProject(loadConfig(), ctx.projectFlag);
|
|
52
|
+
const inHerdrPane =
|
|
53
|
+
process.env["HERDR_ENV"] !== undefined && process.env["HERDR_ENV"] !== "";
|
|
54
|
+
const actor = sessionRole ?? (inHerdrPane ? "orchestrator" : "operator");
|
|
55
|
+
const store = openStore(dbPath());
|
|
56
|
+
try {
|
|
57
|
+
const result = store.withdrawHandoff(
|
|
58
|
+
id,
|
|
59
|
+
project.name,
|
|
60
|
+
actor,
|
|
61
|
+
reason,
|
|
62
|
+
Date.now(),
|
|
63
|
+
);
|
|
64
|
+
if (result.kind === "not-found") {
|
|
65
|
+
process.stderr.write(`omp-conductor: no report or held notice ${id} for ${project.name}\n`);
|
|
66
|
+
process.exit(1);
|
|
67
|
+
}
|
|
68
|
+
if (result.kind === "refused") {
|
|
69
|
+
process.stderr.write(
|
|
70
|
+
`omp-conductor: cannot withdraw ${result.target} ${id}: state is ${result.state}; ` +
|
|
71
|
+
"only an unsent pending report or unassigned held notice may be withdrawn\n",
|
|
72
|
+
);
|
|
73
|
+
process.exit(1);
|
|
74
|
+
}
|
|
75
|
+
process.stdout.write(
|
|
76
|
+
`${result.withdrawal.target} ${id} withdrawn for ${project.name} — ${reason}\n`,
|
|
77
|
+
);
|
|
78
|
+
} finally {
|
|
79
|
+
store.close();
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
|
|
25
84
|
export async function reportCommand(ctx: CommandContext): Promise<void> {
|
|
85
|
+
if (ctx.argv[1] === "withdraw") {
|
|
86
|
+
await withdrawHandoff(ctx);
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
26
89
|
const rawText = ctx.flag("text");
|
|
27
90
|
const body = rawText?.trim();
|
|
28
91
|
if (body === undefined || body.length === 0 || rawText?.startsWith("--") === true) {
|
|
@@ -128,7 +128,7 @@ export async function restoreDbCommand(ctx: CommandContext): Promise<void> {
|
|
|
128
128
|
const snapshot = requested === undefined ? newestSnapshot(snapDir) : resolve(requested);
|
|
129
129
|
if (snapshot === undefined) {
|
|
130
130
|
throw new Error(
|
|
131
|
-
`no conductor.db snapshot in ${snapDir} —
|
|
131
|
+
`no conductor.db snapshot in ${snapDir} — run \`omp-conductor snapshot-db\` or pass an explicit snapshot path`,
|
|
132
132
|
);
|
|
133
133
|
}
|
|
134
134
|
if (!existsSync(snapshot)) {
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/** `snapshot-db` — publish one transaction-consistent conductor.db snapshot. */
|
|
2
|
+
|
|
3
|
+
import type { CommandContext } from "./context.ts";
|
|
4
|
+
import { dbBackupDirFor, loadConfig } from "../config.ts";
|
|
5
|
+
import type { Store } from "../types.ts";
|
|
6
|
+
import {
|
|
7
|
+
DB_SNAPSHOT_RETENTION,
|
|
8
|
+
dbPath,
|
|
9
|
+
openStore,
|
|
10
|
+
pruneDbSnapshots,
|
|
11
|
+
snapshotDb,
|
|
12
|
+
} from "../store.ts";
|
|
13
|
+
|
|
14
|
+
export async function snapshotDbCommand(ctx: CommandContext): Promise<void> {
|
|
15
|
+
if (ctx.projectFlag !== undefined) {
|
|
16
|
+
process.stderr.write("omp-conductor: snapshot-db does not take a project — the store is host-wide\n");
|
|
17
|
+
process.exit(2);
|
|
18
|
+
}
|
|
19
|
+
if (ctx.argv.length > 1) {
|
|
20
|
+
process.stderr.write(`omp-conductor: snapshot-db: unexpected argument "${ctx.argv[1]}"\n`);
|
|
21
|
+
process.exit(2);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const cfg = loadConfig();
|
|
25
|
+
const directory = dbBackupDirFor(cfg);
|
|
26
|
+
const published = snapshotDb(dbPath(), directory);
|
|
27
|
+
// Publication is already durable. Print its path before secondary accounting
|
|
28
|
+
// so a ledger/prune failure cannot turn success into an invisible file.
|
|
29
|
+
process.stdout.write(`snapshot published: ${published}\n`);
|
|
30
|
+
|
|
31
|
+
let store: Store | undefined;
|
|
32
|
+
try {
|
|
33
|
+
const opened = openStore(dbPath());
|
|
34
|
+
store = opened;
|
|
35
|
+
const now = Date.now();
|
|
36
|
+
const eventIds = cfg.projects.map((project) =>
|
|
37
|
+
opened.recordMaterialEvent({
|
|
38
|
+
project: project.name,
|
|
39
|
+
category: "db-snapshot",
|
|
40
|
+
summary: "published manual conductor.db snapshot",
|
|
41
|
+
evidence: published,
|
|
42
|
+
occurredAt: now,
|
|
43
|
+
recordedAt: now,
|
|
44
|
+
}).id,
|
|
45
|
+
);
|
|
46
|
+
process.stdout.write(`ledger events recorded: ${eventIds.join(", ")}\n`);
|
|
47
|
+
} catch (err) {
|
|
48
|
+
process.stderr.write(
|
|
49
|
+
`warning: snapshot published, but its ledger event could not be recorded: ${
|
|
50
|
+
err instanceof Error ? err.message : String(err)
|
|
51
|
+
}\n`,
|
|
52
|
+
);
|
|
53
|
+
} finally {
|
|
54
|
+
store?.close();
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
try {
|
|
58
|
+
const pruned = pruneDbSnapshots(directory, DB_SNAPSHOT_RETENTION);
|
|
59
|
+
process.stdout.write(`retention ${DB_SNAPSHOT_RETENTION} (${pruned} pruned)\n`);
|
|
60
|
+
} catch (err) {
|
|
61
|
+
process.stderr.write(
|
|
62
|
+
`warning: snapshot published, but retention pruning failed: ${
|
|
63
|
+
err instanceof Error ? err.message : String(err)
|
|
64
|
+
}\n`,
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
@@ -22,7 +22,7 @@ const result = await runDetachedUpgrade({
|
|
|
22
22
|
});
|
|
23
23
|
process.stdout.write(
|
|
24
24
|
`install complete — awaiting first-tick verification:\n` +
|
|
25
|
-
`
|
|
25
|
+
` omp-conductor CLI omp-conductor@${result.version}\n` +
|
|
26
26
|
` omp plugin omp-conductor@${result.version}\n` +
|
|
27
27
|
` Herdr plugin herdr-conductor@${result.gitHead}\n` +
|
|
28
28
|
` orchestrator brief managed ORCHESTRATOR.md floor current; POLICY.md preserved\n` +
|
package/src/commands/upgrade.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `upgrade` — update the
|
|
2
|
+
* `upgrade` — update the discovered CLI package, omp plugin, Herdr recovery plugin and brief as one pinned release.
|
|
3
3
|
*
|
|
4
4
|
* Moved out of cli.ts's switch by the per-verb module split (#462);
|
|
5
5
|
* only the case wrapper, the injected `ctx` lookups and the imports
|
|
@@ -33,7 +33,7 @@ const result = await upgradeConductor({
|
|
|
33
33
|
const spec = sha === undefined ? `omp-conductor@${result.version}` : `${result.version} @ ${result.gitHead.slice(0, 12)}`;
|
|
34
34
|
process.stdout.write(
|
|
35
35
|
`${result.alreadyCurrent ? "already current" : sha === undefined ? "upgrade complete" : "bootstrap complete"}:\n` +
|
|
36
|
-
`
|
|
36
|
+
` omp-conductor CLI ${spec}\n` +
|
|
37
37
|
` omp plugin ${spec}\n` +
|
|
38
38
|
` Herdr plugin herdr-conductor@${result.gitHead}\n` +
|
|
39
39
|
` orchestrator brief managed ORCHESTRATOR.md floor current; POLICY.md preserved\n` +
|
package/src/commands/verb.ts
CHANGED
package/src/commands/watch.ts
CHANGED
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
import type { CommandContext } from "./context.ts";
|
|
22
22
|
import { findProject, loadConfig } from "../config.ts";
|
|
23
23
|
import { CONDITION_FORMS, parseCondition } from "../decisions.ts";
|
|
24
|
+
import { watchMonitoringForProject } from "../fleet.ts";
|
|
24
25
|
import { shellQuote } from "../shell.ts";
|
|
25
26
|
import { dbPath, openStore } from "../store.ts";
|
|
26
27
|
|
|
@@ -94,7 +95,12 @@ export async function watchCommand(ctx: CommandContext): Promise<void> {
|
|
|
94
95
|
...(condition === undefined ? {} : { condition }),
|
|
95
96
|
at: Date.now(),
|
|
96
97
|
});
|
|
97
|
-
const
|
|
98
|
+
const monitoring = watchMonitoringForProject(project.name);
|
|
99
|
+
const wake = monitoring.monitored
|
|
100
|
+
? condition === undefined
|
|
101
|
+
? "read by the next tick"
|
|
102
|
+
: "the daemon checks it and wakes the next available tick when it is met"
|
|
103
|
+
: `durable but currently unmonitored (${monitoring.reason}); it becomes wake-capable automatically when the daemon and heartbeat are active`;
|
|
98
104
|
// The verb that ends the row names the project it was created in: this
|
|
99
105
|
// command is copied, and on a host with several projects the bare form is
|
|
100
106
|
// ambiguous (#810).
|
|
@@ -107,6 +113,7 @@ export async function watchCommand(ctx: CommandContext): Promise<void> {
|
|
|
107
113
|
if (sub === "list" || sub === undefined) {
|
|
108
114
|
assertKnownWatchArgs(ctx, "list", 2);
|
|
109
115
|
const open = store.openDecisions(project.name).filter((d) => d.kind === "watch");
|
|
116
|
+
const monitoring = watchMonitoringForProject(project.name);
|
|
110
117
|
const now = Date.now();
|
|
111
118
|
const watches = open.map((d) => ({
|
|
112
119
|
id: d.id,
|
|
@@ -116,13 +123,18 @@ export async function watchCommand(ctx: CommandContext): Promise<void> {
|
|
|
116
123
|
note: d.question,
|
|
117
124
|
}));
|
|
118
125
|
if (ctx.argv.includes("--json")) {
|
|
119
|
-
process.stdout.write(`${JSON.stringify({ project: project.name, watches }, null, 2)}\n`);
|
|
126
|
+
process.stdout.write(`${JSON.stringify({ project: project.name, monitoring, watches }, null, 2)}\n`);
|
|
120
127
|
return;
|
|
121
128
|
}
|
|
122
129
|
if (watches.length === 0) {
|
|
123
130
|
process.stdout.write("no watches\n");
|
|
124
131
|
return;
|
|
125
132
|
}
|
|
133
|
+
process.stdout.write(
|
|
134
|
+
monitoring.monitored
|
|
135
|
+
? "watch monitoring: active\n"
|
|
136
|
+
: `watch monitoring: durable, unmonitored (${monitoring.reason})\n`,
|
|
137
|
+
);
|
|
126
138
|
for (const watch of watches) {
|
|
127
139
|
process.stdout.write(
|
|
128
140
|
`${watch.id} ${watch.ageHours}h blocks:${watch.blocks ?? "-"} ` +
|
package/src/config.ts
CHANGED
|
@@ -320,6 +320,23 @@ export function resolveReleaseGrants(p: ProjectConfig): ResolvedGrants {
|
|
|
320
320
|
return grants;
|
|
321
321
|
}
|
|
322
322
|
|
|
323
|
+
/** The one install holder shared by every project on this host, or the
|
|
324
|
+
* per-project evidence explaining why no shared holder exists (#1018). */
|
|
325
|
+
export function resolveSharedInstallAuthority(projects: readonly ProjectConfig[]): {
|
|
326
|
+
holder?: ResolvedGrants["install"];
|
|
327
|
+
entries: readonly { project: string; holder: ResolvedGrants["install"] }[];
|
|
328
|
+
} {
|
|
329
|
+
const entries = projects.map((project) => ({
|
|
330
|
+
project: project.name,
|
|
331
|
+
holder: resolveReleaseGrants(project).install,
|
|
332
|
+
}));
|
|
333
|
+
const holders = new Set(entries.map((entry) => entry.holder));
|
|
334
|
+
return {
|
|
335
|
+
...(holders.size === 1 ? { holder: entries[0]!.holder } : {}),
|
|
336
|
+
entries,
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
|
|
323
340
|
/**
|
|
324
341
|
* The gating conditions for one project, complete (#129).
|
|
325
342
|
*
|
package/src/daemon.ts
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
loadConfig,
|
|
18
18
|
resolveCaps,
|
|
19
19
|
resolveReleaseGrants,
|
|
20
|
+
resolveSharedInstallAuthority,
|
|
20
21
|
resolveReview,
|
|
21
22
|
stateDir,
|
|
22
23
|
} from "./config.ts";
|
|
@@ -171,6 +172,7 @@ import type {
|
|
|
171
172
|
DigestBacklog,
|
|
172
173
|
Escalation,
|
|
173
174
|
InterruptCategory,
|
|
175
|
+
HandoffWithdrawal,
|
|
174
176
|
IssueComment,
|
|
175
177
|
IssueSnapshot,
|
|
176
178
|
PrDiff,
|
|
@@ -451,6 +453,7 @@ interface Deps {
|
|
|
451
453
|
export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbActions">): VerbDeps {
|
|
452
454
|
return {
|
|
453
455
|
project: () => findProject(loadConfig(), d.project.name),
|
|
456
|
+
projects: () => loadConfig().projects,
|
|
454
457
|
store: d.store,
|
|
455
458
|
tracker: d.tracker,
|
|
456
459
|
actions: d.verbActions ?? githubVerbActions(d.project),
|
|
@@ -6605,6 +6608,8 @@ export interface StatusSnapshot {
|
|
|
6605
6608
|
* unknown outcome, or written off. An empty list is the only honest way to
|
|
6606
6609
|
* say "everything authored this cycle actually went out" (#123). */
|
|
6607
6610
|
openReports: ReportRecord[];
|
|
6611
|
+
/** Recent successful report/notice withdrawals, newest first. */
|
|
6612
|
+
handoffWithdrawals?: HandoffWithdrawal[];
|
|
6608
6613
|
/** Ordinary outcomes and deferred escalations not yet associated with an
|
|
6609
6614
|
* accepted digest report. */
|
|
6610
6615
|
digestBacklog: DigestBacklog;
|
|
@@ -6791,6 +6796,7 @@ export function statusSnapshotFromStore(
|
|
|
6791
6796
|
quarantinedRuns: store.quarantinedRuns(p.name),
|
|
6792
6797
|
turnOverrides: store.listTurnOverrides(p.name),
|
|
6793
6798
|
openReports: store.openReports(p.name),
|
|
6799
|
+
handoffWithdrawals: store.handoffWithdrawals(p.name),
|
|
6794
6800
|
digestBacklog: store.digestBacklog(p.name),
|
|
6795
6801
|
verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
|
|
6796
6802
|
liveWorkers: live.length,
|
|
@@ -7199,11 +7205,18 @@ interface ProjectRuntime {
|
|
|
7199
7205
|
reportPass?: Promise<void>;
|
|
7200
7206
|
}
|
|
7201
7207
|
|
|
7202
|
-
function orchestratorStandingOrders(
|
|
7208
|
+
export function orchestratorStandingOrders(
|
|
7209
|
+
project: ProjectConfig,
|
|
7210
|
+
projects: readonly ProjectConfig[],
|
|
7211
|
+
): {
|
|
7203
7212
|
brief: string;
|
|
7204
7213
|
releaseGrants: ResolvedGrants;
|
|
7205
7214
|
} {
|
|
7206
|
-
const
|
|
7215
|
+
const installAuthority = resolveSharedInstallAuthority(projects);
|
|
7216
|
+
const releaseGrants = {
|
|
7217
|
+
...resolveReleaseGrants(project),
|
|
7218
|
+
install: installAuthority.holder ?? "human",
|
|
7219
|
+
};
|
|
7207
7220
|
const grantedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] === "orchestrator");
|
|
7208
7221
|
const deniedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] !== "orchestrator");
|
|
7209
7222
|
return {
|
|
@@ -7230,6 +7243,13 @@ function orchestratorStandingOrders(project: ProjectConfig): {
|
|
|
7230
7243
|
? "every release and deploy shape is mechanically blocked for you"
|
|
7231
7244
|
: `you may invoke ${grantedShapes.join(", ")} — and only by the procedure in your POLICY.md`
|
|
7232
7245
|
}.` + (deniedShapes.length === 0 ? "" : ` Blocked: ${deniedShapes.join(", ")}.`),
|
|
7246
|
+
...(installAuthority.holder === undefined
|
|
7247
|
+
? [
|
|
7248
|
+
`Host-global install authority conflicts: ${installAuthority.entries
|
|
7249
|
+
.map((entry) => `${entry.project}=${entry.holder}`)
|
|
7250
|
+
.join(", ")}. Install is blocked.`,
|
|
7251
|
+
]
|
|
7252
|
+
: []),
|
|
7233
7253
|
"Handle each escalation below before the next one.",
|
|
7234
7254
|
].join("\n"),
|
|
7235
7255
|
};
|
|
@@ -7432,7 +7452,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
7432
7452
|
log(`skipping orphan reconciliation: daemon pid ${alive.pid} is alive and owns the active runs`);
|
|
7433
7453
|
}
|
|
7434
7454
|
|
|
7435
|
-
const { brief, releaseGrants } = orchestratorStandingOrders(project);
|
|
7455
|
+
const { brief, releaseGrants } = orchestratorStandingOrders(project, cfg.projects);
|
|
7436
7456
|
let orchestrator: OrchestratorHandle | undefined;
|
|
7437
7457
|
let orchestratorVerbs: VerbListener | undefined;
|
|
7438
7458
|
/** First start-failure cause, surfaced by the orchestrator-down incident (#288). */
|
package/src/doctor.ts
CHANGED
|
@@ -85,6 +85,7 @@ import {
|
|
|
85
85
|
import {
|
|
86
86
|
herdrConductorPluginConfigDir,
|
|
87
87
|
planHostRuntime,
|
|
88
|
+
RECOVER_SERVICE_NAME,
|
|
88
89
|
STAGED_SERVICE_NAME,
|
|
89
90
|
SYSTEMD_UNIT_DIR,
|
|
90
91
|
tickCwdForProject,
|
|
@@ -229,6 +230,8 @@ export interface DoctorDeps {
|
|
|
229
230
|
hasSystemd?: () => boolean;
|
|
230
231
|
/** The installed unit text at an absolute path, or undefined when absent. */
|
|
231
232
|
readUnit?: (path: string) => string | undefined;
|
|
233
|
+
/** Whether one installed systemd unit is currently failed; undefined when unreadable. */
|
|
234
|
+
unitFailed?: (name: string) => boolean | undefined;
|
|
232
235
|
/** stat one path; undefined when it does not exist. */
|
|
233
236
|
stat?: (path: string) => Stats | undefined;
|
|
234
237
|
/** uid of a username (`id -u`), or undefined when it cannot be resolved. */
|
|
@@ -397,6 +400,18 @@ function defaultReadUnit(path: string): string | undefined {
|
|
|
397
400
|
}
|
|
398
401
|
}
|
|
399
402
|
|
|
403
|
+
export function systemdStateFailed(state: string): boolean | undefined {
|
|
404
|
+
const normalized = state.trim();
|
|
405
|
+
if (normalized === "") return undefined;
|
|
406
|
+
return normalized === "failed";
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
function defaultUnitFailed(name: string): boolean | undefined {
|
|
410
|
+
const ran = spawnSync("systemctl", ["is-failed", name], { encoding: "utf8" });
|
|
411
|
+
if (ran.error !== undefined) return undefined;
|
|
412
|
+
return systemdStateFailed(ran.stdout ?? "");
|
|
413
|
+
}
|
|
414
|
+
|
|
400
415
|
function defaultStat(path: string): Stats | undefined {
|
|
401
416
|
try {
|
|
402
417
|
return statSync(path);
|
|
@@ -660,7 +675,7 @@ function dbBackupProbe(probes: Probes, cfg: ConductorConfig | undefined): Findin
|
|
|
660
675
|
return warnFinding(
|
|
661
676
|
"db-backup",
|
|
662
677
|
"conductor.db has no snapshot — losing it loses the audit trail with no recovery path",
|
|
663
|
-
"
|
|
678
|
+
"run `omp-conductor snapshot-db` (the daemon cadence uses the same SQLite snapshot primitive)",
|
|
664
679
|
);
|
|
665
680
|
}
|
|
666
681
|
const newestMtime = snapshots
|
|
@@ -670,7 +685,7 @@ function dbBackupProbe(probes: Probes, cfg: ConductorConfig | undefined): Findin
|
|
|
670
685
|
return warnFinding(
|
|
671
686
|
"db-backup",
|
|
672
687
|
`conductor.db is newer than every snapshot (${snapshots.length} file(s)) — the newest writes were never snapshot`,
|
|
673
|
-
"
|
|
688
|
+
"run `omp-conductor snapshot-db` so restore-db has the current state",
|
|
674
689
|
);
|
|
675
690
|
}
|
|
676
691
|
return passFinding("db-backup", `conductor.db snapshot is fresh (${snapshots.length} file(s))`);
|
|
@@ -934,6 +949,22 @@ function recoveryProbe(probes: Probes): Finding {
|
|
|
934
949
|
);
|
|
935
950
|
}
|
|
936
951
|
|
|
952
|
+
function recoveryStateProbe(probes: Probes): Finding {
|
|
953
|
+
const id = "systemd-recovery-state";
|
|
954
|
+
if (!probes.hasSystemd()) return passFinding(id, "no systemd directory on this host — nothing to check");
|
|
955
|
+
if (probes.readUnit(join(SYSTEMD_UNIT_DIR, RECOVER_SERVICE_NAME)) === undefined) {
|
|
956
|
+
return passFinding(id, "recovery unit is not installed — systemd-recovery owns that finding");
|
|
957
|
+
}
|
|
958
|
+
const failed = probes.unitFailed(RECOVER_SERVICE_NAME);
|
|
959
|
+
if (failed === false) return passFinding(id, `${RECOVER_SERVICE_NAME} is not failed`);
|
|
960
|
+
const fix =
|
|
961
|
+
`inspect \`systemctl status ${RECOVER_SERVICE_NAME}\` and \`journalctl -u ${RECOVER_SERVICE_NAME}\`; ` +
|
|
962
|
+
`fix the root cause, then run \`systemctl reset-failed ${RECOVER_SERVICE_NAME}\` and rerun doctor`;
|
|
963
|
+
return failed === true
|
|
964
|
+
? warnFinding(id, `${RECOVER_SERVICE_NAME} is in failed state — its last recovery did not complete`, fix)
|
|
965
|
+
: warnFinding(id, `${RECOVER_SERVICE_NAME} state could not be read`, fix);
|
|
966
|
+
}
|
|
967
|
+
|
|
937
968
|
function installedUnitUser(probes: Probes): string | undefined {
|
|
938
969
|
const unit = probes.readUnit(join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME));
|
|
939
970
|
if (unit === undefined) return undefined;
|
|
@@ -1713,7 +1744,7 @@ function parseEnvKey(text: string, key: string): string | undefined {
|
|
|
1713
1744
|
|
|
1714
1745
|
/**
|
|
1715
1746
|
* Install-surface parity (#904). omp-conductor installs onto three surfaces —
|
|
1716
|
-
* the
|
|
1747
|
+
* the discovered CLI package tree, the omp plugin, and the herdr recovery
|
|
1717
1748
|
* plugin — and only a live `upgrade` invocation ever compared them. On a
|
|
1718
1749
|
* fleet whose installs are manual they diverge silently: measured on this
|
|
1719
1750
|
* host on 2026-08-22, the omp plugin sat on the withdrawn 0.18.1 release for
|
|
@@ -2002,6 +2033,7 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
|
|
|
2002
2033
|
// the first one stands in for the rendering seam.
|
|
2003
2034
|
findings.push(unitProbe(probes, projects[0], cfg));
|
|
2004
2035
|
findings.push(recoveryProbe(probes));
|
|
2036
|
+
findings.push(recoveryStateProbe(probes));
|
|
2005
2037
|
findings.push(ownershipProbe(probes));
|
|
2006
2038
|
// #541 seam checks, host-global: the live herdr config and the plugin's
|
|
2007
2039
|
// config.env are single files on the host, not per-project facts.
|
|
@@ -2106,6 +2138,7 @@ export function defaultProbes(): Probes {
|
|
|
2106
2138
|
repoLabels: defaultRepoLabels,
|
|
2107
2139
|
hasSystemd: () => existsSync(SYSTEMD_UNIT_DIR),
|
|
2108
2140
|
readUnit: defaultReadUnit,
|
|
2141
|
+
unitFailed: defaultUnitFailed,
|
|
2109
2142
|
stat: defaultStat,
|
|
2110
2143
|
uidOf: defaultUidOf,
|
|
2111
2144
|
dbIntegrity: defaultDbIntegrity,
|