omp-conductor 0.19.4 → 0.19.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli.ts +2 -0
- package/src/command-help.ts +18 -8
- package/src/command-manifest.ts +22 -5
- package/src/commands/context.ts +1 -0
- package/src/commands/report.ts +63 -0
- package/src/commands/restore-db.ts +1 -1
- package/src/commands/resume.ts +5 -0
- package/src/commands/snapshot-db.ts +67 -0
- package/src/commands/upgrade-install.ts +1 -1
- package/src/commands/upgrade.ts +2 -2
- package/src/commands/verb.ts +1 -0
- package/src/commands/watch.ts +14 -2
- package/src/config.ts +17 -0
- package/src/daemon.ts +54 -8
- package/src/dashboard/controls.ts +3 -0
- package/src/doctor.ts +36 -3
- package/src/fleet.ts +51 -3
- package/src/orchestrator-tick.ts +14 -3
- package/src/setup-host.ts +10 -10
- package/src/setup-wizard.ts +1 -1
- package/src/status-render.ts +21 -0
- package/src/store.ts +296 -22
- package/src/types.ts +32 -3
- package/src/upgrade-journal.ts +37 -1
- package/src/upgrade-verify.ts +272 -37
- package/src/upgrade.ts +324 -23
- package/src/verbs/actions.ts +6 -1
- package/src/verbs/server.ts +54 -20
- package/systemd/omp-conductor-recover.sh +55 -7
- package/systemd/recover-unit-test.sh +99 -6
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "omp-conductor",
|
|
3
|
-
"version": "0.19.
|
|
3
|
+
"version": "0.19.6",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",
|
package/src/cli.ts
CHANGED
|
@@ -32,6 +32,7 @@ import { reportCommand } from "./commands/report.ts";
|
|
|
32
32
|
import { releaseCompositionCommand } from "./commands/release-composition.ts";
|
|
33
33
|
import { restartCommand } from "./commands/restart.ts";
|
|
34
34
|
import { restoreDbCommand } from "./commands/restore-db.ts";
|
|
35
|
+
import { snapshotDbCommand } from "./commands/snapshot-db.ts";
|
|
35
36
|
import { resumeCommand } from "./commands/resume.ts";
|
|
36
37
|
import { setupCommand } from "./commands/setup.ts";
|
|
37
38
|
import { startCommand } from "./commands/start.ts";
|
|
@@ -300,6 +301,7 @@ export function commandHandlers(ctx: CommandContext): Record<string, CommandHand
|
|
|
300
301
|
intake: () => intakeCommand(ctx),
|
|
301
302
|
friction: () => frictionCommand(ctx),
|
|
302
303
|
"restore-db": () => restoreDbCommand(ctx),
|
|
304
|
+
"snapshot-db": () => snapshotDbCommand(ctx),
|
|
303
305
|
resume: () => resumeCommand(ctx),
|
|
304
306
|
"brief-upgrade": () => briefUpgradeCommand(ctx),
|
|
305
307
|
help: () => helpCommand(USAGE),
|
package/src/command-help.ts
CHANGED
|
@@ -6,10 +6,11 @@ export const COMMAND_DETAILS = ` setup interview, then write config.json, th
|
|
|
6
6
|
the project already exists, a chooser of which area to amend. Naming
|
|
7
7
|
an area positionally skips that chooser and amends only that area:
|
|
8
8
|
${AMEND_AREA_IDS.join(", ")}.
|
|
9
|
-
upgrade
|
|
10
|
-
brief as one pinned release.
|
|
11
|
-
workers, reloads, verifies twice,
|
|
12
|
-
state. Run it from a shell outside
|
|
9
|
+
upgrade discover the installed CLI package root, then update it with the omp
|
|
10
|
+
plugin, Herdr recovery plugin, and brief as one pinned release.
|
|
11
|
+
Pauses only new claims, drains live workers, reloads, verifies twice,
|
|
12
|
+
and restores the prior dispatch state. Run it from a shell outside
|
|
13
|
+
the target Herdr session.
|
|
13
14
|
--bootstrap SHA --source PATH installs an exact commit instead of a
|
|
14
15
|
published version, for when the installed conductor is what is
|
|
15
16
|
broken: the source must be a checkout AT that sha, the package's own
|
|
@@ -66,6 +67,10 @@ export const COMMAND_DETAILS = ` setup interview, then write config.json, th
|
|
|
66
67
|
--probe-telegram sends through the report transport. Exit code 0
|
|
67
68
|
only when nothing failed; --json prints the stable CI shape. Run
|
|
68
69
|
it after install and after every upgrade.
|
|
70
|
+
snapshot-db
|
|
71
|
+
publish a transaction-consistent conductor.db snapshot with SQLite
|
|
72
|
+
VACUUM INTO, record it in the durable event ledger, and apply the
|
|
73
|
+
daemon cadence's bounded retention. Safe while the daemon writes.
|
|
69
74
|
restore-db
|
|
70
75
|
put conductor.db back to a restorable snapshot, replacing the live
|
|
71
76
|
file and dropping stale -wal/-shm sidecars. Refuses while a daemon
|
|
@@ -153,6 +158,9 @@ export const COMMAND_DETAILS = ` setup interview, then write config.json, th
|
|
|
153
158
|
sending now and holding, exactly as for a daemon escalation. An
|
|
154
159
|
identical retry is refused only while the earlier handoff is still
|
|
155
160
|
undelivered, and admitted again once it lands (#453).
|
|
161
|
+
report withdraw removes only an unsent pending report or unassigned
|
|
162
|
+
held notice, requires a reason, preserves an audit row, and refuses
|
|
163
|
+
sending/delivered/failed/already-withdrawn handoffs.
|
|
156
164
|
message deliver one direct Telegram message to this project's own chat and
|
|
157
165
|
forum topic, resolved from config rather than from whichever chat
|
|
158
166
|
last wrote to the session. This is how a locally injected tick
|
|
@@ -180,10 +188,12 @@ export const COMMAND_DETAILS = ` setup interview, then write config.json, th
|
|
|
180
188
|
watch set a condition or carry note for the orchestrator itself, with no
|
|
181
189
|
human in the loop — the same no-sleep surface \`decision open
|
|
182
190
|
--resolves-when\` used to serve, given its own verb so it never renders
|
|
183
|
-
as a question put to the operator.
|
|
184
|
-
condition
|
|
185
|
-
|
|
186
|
-
|
|
191
|
+
as a question put to the operator. The running daemon checks a
|
|
192
|
+
\`--resolves-when\` condition; an armed or ungated heartbeat wakes the
|
|
193
|
+
orchestrator tick when one is met. With no daemon the durable row is
|
|
194
|
+
unmonitored. With ticks disarmed the condition may still be marked met,
|
|
195
|
+
but no tick can be woken until the heartbeat returns. It is listed
|
|
196
|
+
under its own heading, never under "Open operator decisions", and has
|
|
187
197
|
no seven-day expiry. \`watch list\` shows open watches; \`watch
|
|
188
198
|
withdraw <id>\` ends one with a recorded reason — the verb that
|
|
189
199
|
creates a watch is the verb that ends it (watches are decision rows,
|
package/src/command-manifest.ts
CHANGED
|
@@ -202,6 +202,17 @@ export const COMMAND_MANIFEST: readonly CommandManifestEntry[] = [
|
|
|
202
202
|
toggle("--probe-telegram", "send the self-identified Telegram health probe"),
|
|
203
203
|
],
|
|
204
204
|
},
|
|
205
|
+
{
|
|
206
|
+
name: "snapshot-db",
|
|
207
|
+
description: "publish a transaction-consistent conductor.db snapshot",
|
|
208
|
+
scope: "host",
|
|
209
|
+
usage: ["snapshot-db"],
|
|
210
|
+
flags: [],
|
|
211
|
+
details:
|
|
212
|
+
"Publish one transaction-consistent conductor.db snapshot using SQLite VACUUM INTO, " +
|
|
213
|
+
"record the action in every configured project's durable event ledger, and apply the " +
|
|
214
|
+
"daemon cadence's bounded retention. Safe while the daemon is writing.",
|
|
215
|
+
},
|
|
205
216
|
{
|
|
206
217
|
name: "restore-db",
|
|
207
218
|
description: "restore conductor.db from a safe snapshot",
|
|
@@ -403,14 +414,18 @@ export const COMMAND_MANIFEST: readonly CommandManifestEntry[] = [
|
|
|
403
414
|
scope: "project",
|
|
404
415
|
usage: [
|
|
405
416
|
"report --text TEXT [--kind material|digest|tier2|decision-needed|fleet-stopped|confirmed-failure] [--events IDS] [--notices IDS] [--project NAME]",
|
|
417
|
+
"report withdraw <id> --reason TEXT [--project NAME]",
|
|
406
418
|
],
|
|
419
|
+
subcommands: [{ name: "withdraw", description: "withdraw an unsent report or held notice" }],
|
|
407
420
|
flags: [
|
|
408
421
|
value("--text", "rendered report text"),
|
|
409
422
|
value("--kind", "material, digest, or escalation category"),
|
|
410
423
|
value("--events", "comma-separated event ledger ids"),
|
|
411
424
|
value("--notices", "comma-separated notice ledger ids"),
|
|
425
|
+
value("--reason", "why a pending handoff is obsolete"),
|
|
412
426
|
project(),
|
|
413
427
|
],
|
|
428
|
+
positionals: [{ name: "action" }, { name: "id" }],
|
|
414
429
|
},
|
|
415
430
|
{
|
|
416
431
|
name: "message",
|
|
@@ -460,11 +475,13 @@ export const COMMAND_MANIFEST: readonly CommandManifestEntry[] = [
|
|
|
460
475
|
"watch list [--project NAME] [--json]",
|
|
461
476
|
"watch withdraw <id> [--reason TEXT] [--project NAME]",
|
|
462
477
|
],
|
|
463
|
-
details: `add records a row
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
478
|
+
details: `add records a durable row with no operator answer needed. The running daemon
|
|
479
|
+
checks conditions; an armed or ungated heartbeat wakes the orchestrator tick when one is met.
|
|
480
|
+
With no daemon the row is unmonitored. With ticks disarmed the daemon may still mark a
|
|
481
|
+
condition met, but no tick can be woken until the heartbeat returns. list shows open watches,
|
|
482
|
+
oldest first. withdraw ends one with a recorded reason — the verb that creates a watch is the
|
|
483
|
+
verb that ends it. A watch whose PR condition can no longer be observed (the PR merged or
|
|
484
|
+
closed first) is withdrawn by the daemon itself.`,
|
|
468
485
|
subcommands: [
|
|
469
486
|
{ name: "add", description: "record a watch" },
|
|
470
487
|
{ name: "list", description: "list open watches" },
|
package/src/commands/context.ts
CHANGED
package/src/commands/report.ts
CHANGED
|
@@ -22,7 +22,70 @@ import {
|
|
|
22
22
|
type ReportKind,
|
|
23
23
|
} from "../types.ts";
|
|
24
24
|
|
|
25
|
+
async function withdrawHandoff(ctx: CommandContext): Promise<void> {
|
|
26
|
+
const id = ctx.argv[2];
|
|
27
|
+
const reason = ctx.flag("reason")?.trim();
|
|
28
|
+
if (id === undefined || id.startsWith("--")) {
|
|
29
|
+
process.stderr.write("omp-conductor: report withdraw needs the report or held-notice id\n");
|
|
30
|
+
process.exit(2);
|
|
31
|
+
}
|
|
32
|
+
if (reason === undefined || reason.length === 0 || reason.startsWith("--")) {
|
|
33
|
+
process.stderr.write("omp-conductor: report withdraw needs --reason with why the handoff is obsolete\n");
|
|
34
|
+
process.exit(2);
|
|
35
|
+
}
|
|
36
|
+
for (let i = 3; i < ctx.argv.length; i++) {
|
|
37
|
+
const token = ctx.argv[i];
|
|
38
|
+
if (token === "--project" || token === "--reason") {
|
|
39
|
+
i += 1;
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
if (token?.startsWith("--project=") || token?.startsWith("--reason=")) continue;
|
|
43
|
+
process.stderr.write(`omp-conductor: report withdraw: unexpected argument "${token ?? ""}"\n`);
|
|
44
|
+
process.exit(2);
|
|
45
|
+
}
|
|
46
|
+
const sessionRole = process.env[SESSION_ROLE_ENV];
|
|
47
|
+
if (sessionRole === "worker") {
|
|
48
|
+
process.stderr.write("omp-conductor: report withdraw is an operator/orchestrator action, not a worker action\n");
|
|
49
|
+
process.exit(3);
|
|
50
|
+
}
|
|
51
|
+
const project = findProject(loadConfig(), ctx.projectFlag);
|
|
52
|
+
const inHerdrPane =
|
|
53
|
+
process.env["HERDR_ENV"] !== undefined && process.env["HERDR_ENV"] !== "";
|
|
54
|
+
const actor = sessionRole ?? (inHerdrPane ? "orchestrator" : "operator");
|
|
55
|
+
const store = openStore(dbPath());
|
|
56
|
+
try {
|
|
57
|
+
const result = store.withdrawHandoff(
|
|
58
|
+
id,
|
|
59
|
+
project.name,
|
|
60
|
+
actor,
|
|
61
|
+
reason,
|
|
62
|
+
Date.now(),
|
|
63
|
+
);
|
|
64
|
+
if (result.kind === "not-found") {
|
|
65
|
+
process.stderr.write(`omp-conductor: no report or held notice ${id} for ${project.name}\n`);
|
|
66
|
+
process.exit(1);
|
|
67
|
+
}
|
|
68
|
+
if (result.kind === "refused") {
|
|
69
|
+
process.stderr.write(
|
|
70
|
+
`omp-conductor: cannot withdraw ${result.target} ${id}: state is ${result.state}; ` +
|
|
71
|
+
"only an unsent pending report or unassigned held notice may be withdrawn\n",
|
|
72
|
+
);
|
|
73
|
+
process.exit(1);
|
|
74
|
+
}
|
|
75
|
+
process.stdout.write(
|
|
76
|
+
`${result.withdrawal.target} ${id} withdrawn for ${project.name} — ${reason}\n`,
|
|
77
|
+
);
|
|
78
|
+
} finally {
|
|
79
|
+
store.close();
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
|
|
25
84
|
export async function reportCommand(ctx: CommandContext): Promise<void> {
|
|
85
|
+
if (ctx.argv[1] === "withdraw") {
|
|
86
|
+
await withdrawHandoff(ctx);
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
26
89
|
const rawText = ctx.flag("text");
|
|
27
90
|
const body = rawText?.trim();
|
|
28
91
|
if (body === undefined || body.length === 0 || rawText?.startsWith("--") === true) {
|
|
@@ -128,7 +128,7 @@ export async function restoreDbCommand(ctx: CommandContext): Promise<void> {
|
|
|
128
128
|
const snapshot = requested === undefined ? newestSnapshot(snapDir) : resolve(requested);
|
|
129
129
|
if (snapshot === undefined) {
|
|
130
130
|
throw new Error(
|
|
131
|
-
`no conductor.db snapshot in ${snapDir} —
|
|
131
|
+
`no conductor.db snapshot in ${snapDir} — run \`omp-conductor snapshot-db\` or pass an explicit snapshot path`,
|
|
132
132
|
);
|
|
133
133
|
}
|
|
134
134
|
if (!existsSync(snapshot)) {
|
package/src/commands/resume.ts
CHANGED
|
@@ -13,10 +13,15 @@
|
|
|
13
13
|
|
|
14
14
|
import type { CommandContext } from "./context.ts";
|
|
15
15
|
import { armState, clearPaneHaltIfResolvable, releaseHold } from "../fleet.ts";
|
|
16
|
+
import { acknowledgeUpgradeRecovery } from "../upgrade-verify.ts";
|
|
16
17
|
import { wakeDispatch } from "../wake.ts";
|
|
17
18
|
|
|
18
19
|
export async function resumeCommand(ctx: CommandContext): Promise<void> {
|
|
19
20
|
for (const project of ctx.targetProjects()) {
|
|
21
|
+
// `releaseHold(project)` also removes the host-global sentinel. Record an
|
|
22
|
+
// unresolved recovery acknowledgement first even when a project-local pause
|
|
23
|
+
// shadows that global fence; the helper is a no-op when nothing is owed.
|
|
24
|
+
acknowledgeUpgradeRecovery();
|
|
20
25
|
releaseHold(project.name);
|
|
21
26
|
const pin = clearPaneHaltIfResolvable(project.name);
|
|
22
27
|
const arm = armState(project.name);
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/** `snapshot-db` — publish one transaction-consistent conductor.db snapshot. */
|
|
2
|
+
|
|
3
|
+
import type { CommandContext } from "./context.ts";
|
|
4
|
+
import { dbBackupDirFor, loadConfig } from "../config.ts";
|
|
5
|
+
import type { Store } from "../types.ts";
|
|
6
|
+
import {
|
|
7
|
+
DB_SNAPSHOT_RETENTION,
|
|
8
|
+
dbPath,
|
|
9
|
+
openStore,
|
|
10
|
+
pruneDbSnapshots,
|
|
11
|
+
snapshotDb,
|
|
12
|
+
} from "../store.ts";
|
|
13
|
+
|
|
14
|
+
export async function snapshotDbCommand(ctx: CommandContext): Promise<void> {
|
|
15
|
+
if (ctx.projectFlag !== undefined) {
|
|
16
|
+
process.stderr.write("omp-conductor: snapshot-db does not take a project — the store is host-wide\n");
|
|
17
|
+
process.exit(2);
|
|
18
|
+
}
|
|
19
|
+
if (ctx.argv.length > 1) {
|
|
20
|
+
process.stderr.write(`omp-conductor: snapshot-db: unexpected argument "${ctx.argv[1]}"\n`);
|
|
21
|
+
process.exit(2);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const cfg = loadConfig();
|
|
25
|
+
const directory = dbBackupDirFor(cfg);
|
|
26
|
+
const published = snapshotDb(dbPath(), directory);
|
|
27
|
+
// Publication is already durable. Print its path before secondary accounting
|
|
28
|
+
// so a ledger/prune failure cannot turn success into an invisible file.
|
|
29
|
+
process.stdout.write(`snapshot published: ${published}\n`);
|
|
30
|
+
|
|
31
|
+
let store: Store | undefined;
|
|
32
|
+
try {
|
|
33
|
+
const opened = openStore(dbPath());
|
|
34
|
+
store = opened;
|
|
35
|
+
const now = Date.now();
|
|
36
|
+
const eventIds = cfg.projects.map((project) =>
|
|
37
|
+
opened.recordMaterialEvent({
|
|
38
|
+
project: project.name,
|
|
39
|
+
category: "db-snapshot",
|
|
40
|
+
summary: "published manual conductor.db snapshot",
|
|
41
|
+
evidence: published,
|
|
42
|
+
occurredAt: now,
|
|
43
|
+
recordedAt: now,
|
|
44
|
+
}).id,
|
|
45
|
+
);
|
|
46
|
+
process.stdout.write(`ledger events recorded: ${eventIds.join(", ")}\n`);
|
|
47
|
+
} catch (err) {
|
|
48
|
+
process.stderr.write(
|
|
49
|
+
`warning: snapshot published, but its ledger event could not be recorded: ${
|
|
50
|
+
err instanceof Error ? err.message : String(err)
|
|
51
|
+
}\n`,
|
|
52
|
+
);
|
|
53
|
+
} finally {
|
|
54
|
+
store?.close();
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
try {
|
|
58
|
+
const pruned = pruneDbSnapshots(directory, DB_SNAPSHOT_RETENTION);
|
|
59
|
+
process.stdout.write(`retention ${DB_SNAPSHOT_RETENTION} (${pruned} pruned)\n`);
|
|
60
|
+
} catch (err) {
|
|
61
|
+
process.stderr.write(
|
|
62
|
+
`warning: snapshot published, but retention pruning failed: ${
|
|
63
|
+
err instanceof Error ? err.message : String(err)
|
|
64
|
+
}\n`,
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
@@ -22,7 +22,7 @@ const result = await runDetachedUpgrade({
|
|
|
22
22
|
});
|
|
23
23
|
process.stdout.write(
|
|
24
24
|
`install complete — awaiting first-tick verification:\n` +
|
|
25
|
-
`
|
|
25
|
+
` omp-conductor CLI omp-conductor@${result.version}\n` +
|
|
26
26
|
` omp plugin omp-conductor@${result.version}\n` +
|
|
27
27
|
` Herdr plugin herdr-conductor@${result.gitHead}\n` +
|
|
28
28
|
` orchestrator brief managed ORCHESTRATOR.md floor current; POLICY.md preserved\n` +
|
package/src/commands/upgrade.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `upgrade` — update the
|
|
2
|
+
* `upgrade` — update the discovered CLI package, omp plugin, Herdr recovery plugin and brief as one pinned release.
|
|
3
3
|
*
|
|
4
4
|
* Moved out of cli.ts's switch by the per-verb module split (#462);
|
|
5
5
|
* only the case wrapper, the injected `ctx` lookups and the imports
|
|
@@ -33,7 +33,7 @@ const result = await upgradeConductor({
|
|
|
33
33
|
const spec = sha === undefined ? `omp-conductor@${result.version}` : `${result.version} @ ${result.gitHead.slice(0, 12)}`;
|
|
34
34
|
process.stdout.write(
|
|
35
35
|
`${result.alreadyCurrent ? "already current" : sha === undefined ? "upgrade complete" : "bootstrap complete"}:\n` +
|
|
36
|
-
`
|
|
36
|
+
` omp-conductor CLI ${spec}\n` +
|
|
37
37
|
` omp plugin ${spec}\n` +
|
|
38
38
|
` Herdr plugin herdr-conductor@${result.gitHead}\n` +
|
|
39
39
|
` orchestrator brief managed ORCHESTRATOR.md floor current; POLICY.md preserved\n` +
|
package/src/commands/verb.ts
CHANGED
package/src/commands/watch.ts
CHANGED
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
import type { CommandContext } from "./context.ts";
|
|
22
22
|
import { findProject, loadConfig } from "../config.ts";
|
|
23
23
|
import { CONDITION_FORMS, parseCondition } from "../decisions.ts";
|
|
24
|
+
import { watchMonitoringForProject } from "../fleet.ts";
|
|
24
25
|
import { shellQuote } from "../shell.ts";
|
|
25
26
|
import { dbPath, openStore } from "../store.ts";
|
|
26
27
|
|
|
@@ -94,7 +95,12 @@ export async function watchCommand(ctx: CommandContext): Promise<void> {
|
|
|
94
95
|
...(condition === undefined ? {} : { condition }),
|
|
95
96
|
at: Date.now(),
|
|
96
97
|
});
|
|
97
|
-
const
|
|
98
|
+
const monitoring = watchMonitoringForProject(project.name);
|
|
99
|
+
const wake = monitoring.monitored
|
|
100
|
+
? condition === undefined
|
|
101
|
+
? "read by the next tick"
|
|
102
|
+
: "the daemon checks it and wakes the next available tick when it is met"
|
|
103
|
+
: `durable but currently unmonitored (${monitoring.reason}); it becomes wake-capable automatically when the daemon and heartbeat are active`;
|
|
98
104
|
// The verb that ends the row names the project it was created in: this
|
|
99
105
|
// command is copied, and on a host with several projects the bare form is
|
|
100
106
|
// ambiguous (#810).
|
|
@@ -107,6 +113,7 @@ export async function watchCommand(ctx: CommandContext): Promise<void> {
|
|
|
107
113
|
if (sub === "list" || sub === undefined) {
|
|
108
114
|
assertKnownWatchArgs(ctx, "list", 2);
|
|
109
115
|
const open = store.openDecisions(project.name).filter((d) => d.kind === "watch");
|
|
116
|
+
const monitoring = watchMonitoringForProject(project.name);
|
|
110
117
|
const now = Date.now();
|
|
111
118
|
const watches = open.map((d) => ({
|
|
112
119
|
id: d.id,
|
|
@@ -116,13 +123,18 @@ export async function watchCommand(ctx: CommandContext): Promise<void> {
|
|
|
116
123
|
note: d.question,
|
|
117
124
|
}));
|
|
118
125
|
if (ctx.argv.includes("--json")) {
|
|
119
|
-
process.stdout.write(`${JSON.stringify({ project: project.name, watches }, null, 2)}\n`);
|
|
126
|
+
process.stdout.write(`${JSON.stringify({ project: project.name, monitoring, watches }, null, 2)}\n`);
|
|
120
127
|
return;
|
|
121
128
|
}
|
|
122
129
|
if (watches.length === 0) {
|
|
123
130
|
process.stdout.write("no watches\n");
|
|
124
131
|
return;
|
|
125
132
|
}
|
|
133
|
+
process.stdout.write(
|
|
134
|
+
monitoring.monitored
|
|
135
|
+
? "watch monitoring: active\n"
|
|
136
|
+
: `watch monitoring: durable, unmonitored (${monitoring.reason})\n`,
|
|
137
|
+
);
|
|
126
138
|
for (const watch of watches) {
|
|
127
139
|
process.stdout.write(
|
|
128
140
|
`${watch.id} ${watch.ageHours}h blocks:${watch.blocks ?? "-"} ` +
|
package/src/config.ts
CHANGED
|
@@ -320,6 +320,23 @@ export function resolveReleaseGrants(p: ProjectConfig): ResolvedGrants {
|
|
|
320
320
|
return grants;
|
|
321
321
|
}
|
|
322
322
|
|
|
323
|
+
/** The one install holder shared by every project on this host, or the
|
|
324
|
+
* per-project evidence explaining why no shared holder exists (#1018). */
|
|
325
|
+
export function resolveSharedInstallAuthority(projects: readonly ProjectConfig[]): {
|
|
326
|
+
holder?: ResolvedGrants["install"];
|
|
327
|
+
entries: readonly { project: string; holder: ResolvedGrants["install"] }[];
|
|
328
|
+
} {
|
|
329
|
+
const entries = projects.map((project) => ({
|
|
330
|
+
project: project.name,
|
|
331
|
+
holder: resolveReleaseGrants(project).install,
|
|
332
|
+
}));
|
|
333
|
+
const holders = new Set(entries.map((entry) => entry.holder));
|
|
334
|
+
return {
|
|
335
|
+
...(holders.size === 1 ? { holder: entries[0]!.holder } : {}),
|
|
336
|
+
entries,
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
|
|
323
340
|
/**
|
|
324
341
|
* The gating conditions for one project, complete (#129).
|
|
325
342
|
*
|
package/src/daemon.ts
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
loadConfig,
|
|
18
18
|
resolveCaps,
|
|
19
19
|
resolveReleaseGrants,
|
|
20
|
+
resolveSharedInstallAuthority,
|
|
20
21
|
resolveReview,
|
|
21
22
|
stateDir,
|
|
22
23
|
} from "./config.ts";
|
|
@@ -58,7 +59,7 @@ import {
|
|
|
58
59
|
setPaused,
|
|
59
60
|
} from "./pause.ts";
|
|
60
61
|
import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
|
|
61
|
-
import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
|
|
62
|
+
import { runCommand, upgradeRecoveryStatus, verifyPendingUpgrade, type UpgradeRecoveryStatus, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
|
|
62
63
|
import { inspectSurfaces, type InstalledSurfaces } from "./upgrade.ts";
|
|
63
64
|
import { checkTelegramFreshness, type TelegramFreshness } from "./telegram-freshness.ts";
|
|
64
65
|
import { processStartTimeMs } from "./upgrade-verify.ts";
|
|
@@ -171,6 +172,7 @@ import type {
|
|
|
171
172
|
DigestBacklog,
|
|
172
173
|
Escalation,
|
|
173
174
|
InterruptCategory,
|
|
175
|
+
HandoffWithdrawal,
|
|
174
176
|
IssueComment,
|
|
175
177
|
IssueSnapshot,
|
|
176
178
|
PrDiff,
|
|
@@ -451,6 +453,7 @@ interface Deps {
|
|
|
451
453
|
export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbActions">): VerbDeps {
|
|
452
454
|
return {
|
|
453
455
|
project: () => findProject(loadConfig(), d.project.name),
|
|
456
|
+
projects: () => loadConfig().projects,
|
|
454
457
|
store: d.store,
|
|
455
458
|
tracker: d.tracker,
|
|
456
459
|
actions: d.verbActions ?? githubVerbActions(d.project),
|
|
@@ -5295,6 +5298,7 @@ export function upgradeVerifyDepsFor(d: Pick<Deps, "project" | "store">): Upgrad
|
|
|
5295
5298
|
layers: (project) => fleetLayers(project),
|
|
5296
5299
|
health: healthCheck,
|
|
5297
5300
|
doctor: (name) => runDoctor(name),
|
|
5301
|
+
surfaces: () => inspectSurfaces({ run: runCommand, log, env: process.env }),
|
|
5298
5302
|
enqueue: (draft) => {
|
|
5299
5303
|
try {
|
|
5300
5304
|
d.store.enqueueReport(draft);
|
|
@@ -5304,6 +5308,21 @@ export function upgradeVerifyDepsFor(d: Pick<Deps, "project" | "store">): Upgrad
|
|
|
5304
5308
|
},
|
|
5305
5309
|
pause: pauseInstance,
|
|
5306
5310
|
resume: (project) => setPaused(false, undefined, project),
|
|
5311
|
+
holdGlobal: () => {
|
|
5312
|
+
if (pauseInstance() === undefined) {
|
|
5313
|
+
setPaused(true, { source: "upgrade-recovery", reason: "upgrade recovery verification required" });
|
|
5314
|
+
}
|
|
5315
|
+
},
|
|
5316
|
+
releaseGlobalRecovery: (expectedSince) => {
|
|
5317
|
+
const pause = pauseInstance();
|
|
5318
|
+
if (
|
|
5319
|
+
pause?.source === "upgrade-recovery" &&
|
|
5320
|
+
expectedSince !== undefined &&
|
|
5321
|
+
pause.since === expectedSince
|
|
5322
|
+
) {
|
|
5323
|
+
setPaused(false);
|
|
5324
|
+
}
|
|
5325
|
+
},
|
|
5307
5326
|
launchRollback: async (version) => {
|
|
5308
5327
|
const launched = await launchTransientUnit(
|
|
5309
5328
|
(argv) => runCommand(argv[0]!, argv.slice(1)),
|
|
@@ -5333,7 +5352,11 @@ async function verifyPendingUpgradeTick(
|
|
|
5333
5352
|
const verifyDeps = upgradeVerifyDepsFor(d);
|
|
5334
5353
|
try {
|
|
5335
5354
|
const outcome = await run(verifyDeps);
|
|
5336
|
-
if (
|
|
5355
|
+
if (
|
|
5356
|
+
outcome.handled !== "none" &&
|
|
5357
|
+
outcome.handled !== "already-closed" &&
|
|
5358
|
+
outcome.handled !== "recovery-owed"
|
|
5359
|
+
) {
|
|
5337
5360
|
log(
|
|
5338
5361
|
`upgrade journal handled: ${outcome.handled}` +
|
|
5339
5362
|
`${outcome.version === undefined ? "" : ` for omp-conductor@${outcome.version}`}` +
|
|
@@ -6480,6 +6503,8 @@ export interface StatusSnapshot {
|
|
|
6480
6503
|
* reading like a mistake (#220).
|
|
6481
6504
|
*/
|
|
6482
6505
|
pauseReason?: string;
|
|
6506
|
+
/** Unresolved host-wide upgrade recovery, rendered identically for every project. */
|
|
6507
|
+
upgradeRecovery?: UpgradeRecoveryStatus;
|
|
6483
6508
|
/**
|
|
6484
6509
|
* The project's active self-expiring drain (#484): present only while a
|
|
6485
6510
|
* fresh, valid drain record exists. Paused and drained are deliberately two
|
|
@@ -6583,6 +6608,8 @@ export interface StatusSnapshot {
|
|
|
6583
6608
|
* unknown outcome, or written off. An empty list is the only honest way to
|
|
6584
6609
|
* say "everything authored this cycle actually went out" (#123). */
|
|
6585
6610
|
openReports: ReportRecord[];
|
|
6611
|
+
/** Recent successful report/notice withdrawals, newest first. */
|
|
6612
|
+
handoffWithdrawals?: HandoffWithdrawal[];
|
|
6586
6613
|
/** Ordinary outcomes and deferred escalations not yet associated with an
|
|
6587
6614
|
* accepted digest report. */
|
|
6588
6615
|
digestBacklog: DigestBacklog;
|
|
@@ -6674,9 +6701,12 @@ export function statusSnapshotFromStore(
|
|
|
6674
6701
|
const oldestLabelOpAt = store.oldestPendingLabelOpAt(p.name);
|
|
6675
6702
|
// Read once: `status` renders the degrade row off this while it is down.
|
|
6676
6703
|
const orchestratorDown = store.orchestratorIncident(p.name);
|
|
6677
|
-
//
|
|
6678
|
-
//
|
|
6679
|
-
|
|
6704
|
+
// The journal is the authoritative recovery lifecycle. Render unresolved
|
|
6705
|
+
// recovery even when an operator/project pause currently wins provenance,
|
|
6706
|
+
// or before the verifier has re-created a missing global fence.
|
|
6707
|
+
const pause = pauseProvenance(p.name);
|
|
6708
|
+
const reason = pause?.reason;
|
|
6709
|
+
const upgradeRecovery = upgradeRecoveryStatus(readUpgradeJournal(stateDir()), stateDir());
|
|
6680
6710
|
// Same cost discipline as the pause read: the drain record is a file read,
|
|
6681
6711
|
// and the active-drain view is only built when one is actually fresh. The
|
|
6682
6712
|
// read is observational — it never mutates the record — so a status read
|
|
@@ -6741,6 +6771,7 @@ export function statusSnapshotFromStore(
|
|
|
6741
6771
|
stateDir: stateDir(),
|
|
6742
6772
|
paused: isPaused(p.name),
|
|
6743
6773
|
...(reason === undefined ? {} : { pauseReason: reason }),
|
|
6774
|
+
...(upgradeRecovery === undefined ? {} : { upgradeRecovery }),
|
|
6744
6775
|
...(drain.kind === "active"
|
|
6745
6776
|
? {
|
|
6746
6777
|
drain: {
|
|
@@ -6765,6 +6796,7 @@ export function statusSnapshotFromStore(
|
|
|
6765
6796
|
quarantinedRuns: store.quarantinedRuns(p.name),
|
|
6766
6797
|
turnOverrides: store.listTurnOverrides(p.name),
|
|
6767
6798
|
openReports: store.openReports(p.name),
|
|
6799
|
+
handoffWithdrawals: store.handoffWithdrawals(p.name),
|
|
6768
6800
|
digestBacklog: store.digestBacklog(p.name),
|
|
6769
6801
|
verbLedger: store.verbLedger(p.name, { limit: STATUS_LEDGER_SCAN }),
|
|
6770
6802
|
liveWorkers: live.length,
|
|
@@ -7173,11 +7205,18 @@ interface ProjectRuntime {
|
|
|
7173
7205
|
reportPass?: Promise<void>;
|
|
7174
7206
|
}
|
|
7175
7207
|
|
|
7176
|
-
function orchestratorStandingOrders(
|
|
7208
|
+
export function orchestratorStandingOrders(
|
|
7209
|
+
project: ProjectConfig,
|
|
7210
|
+
projects: readonly ProjectConfig[],
|
|
7211
|
+
): {
|
|
7177
7212
|
brief: string;
|
|
7178
7213
|
releaseGrants: ResolvedGrants;
|
|
7179
7214
|
} {
|
|
7180
|
-
const
|
|
7215
|
+
const installAuthority = resolveSharedInstallAuthority(projects);
|
|
7216
|
+
const releaseGrants = {
|
|
7217
|
+
...resolveReleaseGrants(project),
|
|
7218
|
+
install: installAuthority.holder ?? "human",
|
|
7219
|
+
};
|
|
7181
7220
|
const grantedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] === "orchestrator");
|
|
7182
7221
|
const deniedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] !== "orchestrator");
|
|
7183
7222
|
return {
|
|
@@ -7204,6 +7243,13 @@ function orchestratorStandingOrders(project: ProjectConfig): {
|
|
|
7204
7243
|
? "every release and deploy shape is mechanically blocked for you"
|
|
7205
7244
|
: `you may invoke ${grantedShapes.join(", ")} — and only by the procedure in your POLICY.md`
|
|
7206
7245
|
}.` + (deniedShapes.length === 0 ? "" : ` Blocked: ${deniedShapes.join(", ")}.`),
|
|
7246
|
+
...(installAuthority.holder === undefined
|
|
7247
|
+
? [
|
|
7248
|
+
`Host-global install authority conflicts: ${installAuthority.entries
|
|
7249
|
+
.map((entry) => `${entry.project}=${entry.holder}`)
|
|
7250
|
+
.join(", ")}. Install is blocked.`,
|
|
7251
|
+
]
|
|
7252
|
+
: []),
|
|
7207
7253
|
"Handle each escalation below before the next one.",
|
|
7208
7254
|
].join("\n"),
|
|
7209
7255
|
};
|
|
@@ -7406,7 +7452,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
7406
7452
|
log(`skipping orphan reconciliation: daemon pid ${alive.pid} is alive and owns the active runs`);
|
|
7407
7453
|
}
|
|
7408
7454
|
|
|
7409
|
-
const { brief, releaseGrants } = orchestratorStandingOrders(project);
|
|
7455
|
+
const { brief, releaseGrants } = orchestratorStandingOrders(project, cfg.projects);
|
|
7410
7456
|
let orchestrator: OrchestratorHandle | undefined;
|
|
7411
7457
|
let orchestratorVerbs: VerbListener | undefined;
|
|
7412
7458
|
/** First start-failure cause, surfaced by the orchestrator-down incident (#288). */
|
|
@@ -46,6 +46,7 @@ import { unblockIssue } from "../unblock.ts";
|
|
|
46
46
|
import { makeTracker } from "../tracker/github.ts";
|
|
47
47
|
import { dbPath, openStore } from "../store.ts";
|
|
48
48
|
import { loadConfig, resolveCaps } from "../config.ts";
|
|
49
|
+
import { acknowledgeUpgradeRecovery } from "../upgrade-verify.ts";
|
|
49
50
|
import type { ProjectConfig } from "../types.ts";
|
|
50
51
|
|
|
51
52
|
/** The source string every dashboard mutation attributes itself with. */
|
|
@@ -70,6 +71,7 @@ export interface ControlDeps {
|
|
|
70
71
|
project: ProjectConfig;
|
|
71
72
|
hold?: typeof hold;
|
|
72
73
|
releaseHold?: typeof releaseHold;
|
|
74
|
+
acknowledgeUpgradeRecovery?: typeof acknowledgeUpgradeRecovery;
|
|
73
75
|
disarmTicks?: typeof disarmTicks;
|
|
74
76
|
/**
|
|
75
77
|
* Forwards one request to the owning daemon's own HTTP surface. Returns
|
|
@@ -119,6 +121,7 @@ export function dashboardPause(body: unknown, d: ControlDeps): ControlOutcome {
|
|
|
119
121
|
|
|
120
122
|
/** Clear the pause. Idempotent, exactly as `resume` is on the CLI. */
|
|
121
123
|
export function dashboardResume(d: ControlDeps): ControlOutcome {
|
|
124
|
+
(d.acknowledgeUpgradeRecovery ?? acknowledgeUpgradeRecovery)();
|
|
122
125
|
(d.releaseHold ?? releaseHold)(d.project.name);
|
|
123
126
|
return { status: 200, body: { paused: false } };
|
|
124
127
|
}
|