omp-conductor 0.16.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -4
- package/REFERENCE.md +20 -11
- package/package.json +3 -1
- package/src/briefs/orchestrator.md +21 -6
- package/src/briefs/policy.md +13 -5
- package/src/clack-ui.ts +83 -0
- package/src/cli.ts +110 -381
- package/src/command-help.ts +220 -0
- package/src/command-manifest.ts +480 -0
- package/src/commands/arm.ts +7 -2
- package/src/commands/complete.ts +93 -0
- package/src/commands/context.ts +1 -0
- package/src/commands/decision.ts +17 -7
- package/src/commands/doctor.ts +18 -1
- package/src/commands/hold.ts +9 -7
- package/src/commands/ledger.ts +25 -4
- package/src/commands/setup.ts +128 -11
- package/src/commands/stats.ts +9 -5
- package/src/commands/status.ts +32 -5
- package/src/commands/tail.ts +13 -1
- package/src/commands/watch.ts +16 -7
- package/src/fleet.ts +55 -15
- package/src/orchestrator-tick.ts +46 -7
- package/src/privileged.ts +3 -0
- package/src/setup-answers.ts +135 -0
- package/src/setup-host.ts +29 -11
- package/src/setup-install.ts +2 -0
- package/src/setup-probe.ts +1 -0
- package/src/setup-wizard.ts +130 -38
- package/src/ui/progress.ts +32 -0
- package/src/ui/style.ts +11 -0
- package/src/upgrade-verify.ts +25 -3
- package/src/upgrade.ts +61 -2
- package/src/wizard-ui.ts +14 -5
- package/systemd/omp-conductor-recover.sh +1 -1
- package/systemd/recover-unit-test.sh +2 -2
package/src/commands/hold.ts
CHANGED
|
@@ -8,19 +8,21 @@
|
|
|
8
8
|
|
|
9
9
|
import type { CommandContext } from "./context.ts";
|
|
10
10
|
import { hold } from "../fleet.ts";
|
|
11
|
+
import { dim, ok } from "../ui/style.ts";
|
|
11
12
|
|
|
12
13
|
export async function holdCommand(ctx: CommandContext): Promise<void> {
|
|
13
14
|
const keepTicks = ctx.argv.includes("--keep-ticks");
|
|
14
15
|
for (const project of ctx.targetProjects()) {
|
|
15
16
|
const r = hold(project.name, "hold", keepTicks ? { keepTicks: true } : {});
|
|
16
17
|
process.stdout.write(
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
18
|
+
ok(
|
|
19
|
+
`held — claiming paused` +
|
|
20
|
+
`${r.wasPaused ? " (already paused)" : ""}` +
|
|
21
|
+
(r.disarmed === undefined
|
|
22
|
+
? "; ticks left armed — resume restores the fleet with no new arm challenge"
|
|
23
|
+
: `; ticks disarmed at ${r.disarmed.path}` +
|
|
24
|
+
`${r.disarmed.wasArmed ? "" : " (was already disarmed)"}`),
|
|
25
|
+
) + `\n${dim("daemon and pane left running; stop to stop the daemon too")}\n`,
|
|
24
26
|
);
|
|
25
27
|
}
|
|
26
28
|
}
|
package/src/commands/ledger.ts
CHANGED
|
@@ -10,6 +10,7 @@ import type { CommandContext } from "./context.ts";
|
|
|
10
10
|
import { findProject, loadConfig } from "../config.ts";
|
|
11
11
|
import { dbPath, openStore } from "../store.ts";
|
|
12
12
|
import { formatVerbLedgerEntry } from "../verbs/ledger.ts";
|
|
13
|
+
import { dim, heading } from "../ui/style.ts";
|
|
13
14
|
|
|
14
15
|
export async function ledgerCommand(ctx: CommandContext): Promise<void> {
|
|
15
16
|
const cfg = loadConfig();
|
|
@@ -35,10 +36,28 @@ try {
|
|
|
35
36
|
...(issue === undefined ? {} : { issue }),
|
|
36
37
|
limit,
|
|
37
38
|
});
|
|
39
|
+
if (ctx.argv.includes("--json")) {
|
|
40
|
+
process.stdout.write(
|
|
41
|
+
`${JSON.stringify(
|
|
42
|
+
{
|
|
43
|
+
project: p.name,
|
|
44
|
+
...(issue === undefined ? {} : { issue }),
|
|
45
|
+
entries,
|
|
46
|
+
refused: entries.filter((entry) => entry.decision === "refused").length,
|
|
47
|
+
turnOverrides: overrides,
|
|
48
|
+
},
|
|
49
|
+
null,
|
|
50
|
+
2,
|
|
51
|
+
)}\n`,
|
|
52
|
+
);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
38
55
|
if (entries.length === 0 && overrides.length === 0) {
|
|
39
56
|
process.stdout.write(
|
|
40
|
-
|
|
41
|
-
|
|
57
|
+
dim(
|
|
58
|
+
`no conductor actions recorded for ${p.name}` +
|
|
59
|
+
`${issue === undefined ? "" : ` on #${String(issue)}`}`,
|
|
60
|
+
) + "\n",
|
|
42
61
|
);
|
|
43
62
|
return;
|
|
44
63
|
}
|
|
@@ -46,13 +65,15 @@ try {
|
|
|
46
65
|
if (entries.length > 0) {
|
|
47
66
|
const refused = entries.filter((e) => e.decision === "refused").length;
|
|
48
67
|
blocks.push(
|
|
49
|
-
`${p.name} — ${entries.length} verb call(s), ${refused} refused (newest first)
|
|
68
|
+
heading(`${p.name} — ${entries.length} verb call(s), ${refused} refused (newest first)`) +
|
|
69
|
+
"\n" +
|
|
50
70
|
entries.flatMap(formatVerbLedgerEntry).join("\n"),
|
|
51
71
|
);
|
|
52
72
|
}
|
|
53
73
|
if (overrides.length > 0) {
|
|
54
74
|
blocks.push(
|
|
55
|
-
`${p.name} — ${overrides.length} turn override(s) (newest first)
|
|
75
|
+
heading(`${p.name} — ${overrides.length} turn override(s) (newest first)`) +
|
|
76
|
+
"\n" +
|
|
56
77
|
overrides
|
|
57
78
|
.map(
|
|
58
79
|
(entry) =>
|
package/src/commands/setup.ts
CHANGED
|
@@ -6,13 +6,27 @@
|
|
|
6
6
|
* changed from the original bodies.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
10
|
+
import { homedir } from "node:os";
|
|
11
|
+
import { basename, join } from "node:path";
|
|
12
|
+
|
|
9
13
|
import type { CommandContext } from "./context.ts";
|
|
14
|
+
import type { CompletionShell } from "./complete.ts";
|
|
10
15
|
import { findProject, loadConfig, resolveCaps } from "../config.ts";
|
|
11
16
|
import { AMEND_AREA_IDS, type AmendAreaId } from "../setup.ts";
|
|
12
17
|
import { runGraphInstall, runHostInstall, type InstallOutcome } from "../setup-install.ts";
|
|
13
18
|
import { DEFAULT_PROBES, NO_PROBES, setup } from "../setup-wizard.ts";
|
|
14
19
|
import { telegramStateDir } from "../fleet.ts";
|
|
15
|
-
import {
|
|
20
|
+
import {
|
|
21
|
+
answersUi,
|
|
22
|
+
guardNonInteractiveUi,
|
|
23
|
+
loadAnswersFile,
|
|
24
|
+
recordingAnswersUi,
|
|
25
|
+
saveAnswersFile,
|
|
26
|
+
type RecordedAnswersUi,
|
|
27
|
+
} from "../setup-answers.ts";
|
|
28
|
+
import { terminalUi, type WizardUi } from "../wizard-ui.ts";
|
|
29
|
+
import { interactiveUi } from "../ui/progress.ts";
|
|
16
30
|
import type { ConductorConfig, ProjectConfig } from "../types.ts";
|
|
17
31
|
|
|
18
32
|
/**
|
|
@@ -25,7 +39,7 @@ const SETUP_USAGE = `omp-conductor setup — interview, then write config.json,
|
|
|
25
39
|
and the staged host files behind one confirm.
|
|
26
40
|
|
|
27
41
|
usage:
|
|
28
|
-
omp-conductor setup [area] [--no-ai] [--project NAME]
|
|
42
|
+
omp-conductor setup [area] [--no-ai] [--answers FILE] [--save-answers FILE] [--project NAME]
|
|
29
43
|
omp-conductor setup host [NAME] [--project NAME]
|
|
30
44
|
omp-conductor setup graph [--no-seed] [--print] [--project NAME]
|
|
31
45
|
|
|
@@ -39,9 +53,11 @@ amend areas:
|
|
|
39
53
|
${AMEND_AREA_IDS.join(", ")}
|
|
40
54
|
|
|
41
55
|
flags:
|
|
42
|
-
--project NAME
|
|
43
|
-
|
|
44
|
-
--no-ai
|
|
56
|
+
--project NAME the project to configure or amend (or NAME positionally
|
|
57
|
+
for \`setup host\`)
|
|
58
|
+
--no-ai ask every question, propose nothing (no AI repo reads)
|
|
59
|
+
--answers FILE answer every prompt from JSON; never opens a prompt
|
|
60
|
+
--save-answers FILE save successful prompt answers as replayable JSON`;
|
|
45
61
|
|
|
46
62
|
/**
|
|
47
63
|
* Resolve the project an install subcommand (`setup host`, `setup graph`)
|
|
@@ -63,6 +79,66 @@ export function setupInstallProject(
|
|
|
63
79
|
return findProject(cfg, projectFlag ?? positionalName);
|
|
64
80
|
}
|
|
65
81
|
|
|
82
|
+
export async function installShellCompletions(
|
|
83
|
+
shell: Extract<CompletionShell, "zsh" | "bash">,
|
|
84
|
+
home: string = homedir(),
|
|
85
|
+
): Promise<{ scriptPath: string; rcPath: string }> {
|
|
86
|
+
const completionDir = join(home, ".omp", "conductor");
|
|
87
|
+
mkdirSync(completionDir, { recursive: true });
|
|
88
|
+
const scriptPath = join(completionDir, `completions.${shell}`);
|
|
89
|
+
writeFileSync(
|
|
90
|
+
scriptPath,
|
|
91
|
+
(await import("./complete.ts")).renderCompletionScript(shell),
|
|
92
|
+
"utf8",
|
|
93
|
+
);
|
|
94
|
+
|
|
95
|
+
const rcPath = join(home, shell === "zsh" ? ".zshrc" : ".bashrc");
|
|
96
|
+
const sourceLine =
|
|
97
|
+
`[ -f ~/.omp/conductor/completions.${shell} ] && ` +
|
|
98
|
+
`source ~/.omp/conductor/completions.${shell}`;
|
|
99
|
+
const current = existsSync(rcPath) ? readFileSync(rcPath, "utf8") : "";
|
|
100
|
+
if (!current.split(/\r?\n/).includes(sourceLine)) {
|
|
101
|
+
const separator = current.length > 0 && !current.endsWith("\n") ? "\n" : "";
|
|
102
|
+
appendFileSync(rcPath, `${separator}${sourceLine}\n`, "utf8");
|
|
103
|
+
}
|
|
104
|
+
return { scriptPath, rcPath };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
async function offerCompletionInstall(ui: WizardUi): Promise<void> {
|
|
108
|
+
const install = await ui.confirm(
|
|
109
|
+
"Install shell completions?",
|
|
110
|
+
"Writes a generated completion script and one idempotent source line to your shell rc file.",
|
|
111
|
+
{ key: "install-shell-completions" },
|
|
112
|
+
);
|
|
113
|
+
if (install !== true) return;
|
|
114
|
+
|
|
115
|
+
const detected = basename(process.env.SHELL ?? "");
|
|
116
|
+
const selected = await ui.select(
|
|
117
|
+
"Shell",
|
|
118
|
+
[
|
|
119
|
+
{ label: "zsh", description: "Install through ~/.zshrc" },
|
|
120
|
+
{ label: "bash", description: "Install through ~/.bashrc" },
|
|
121
|
+
],
|
|
122
|
+
{ key: "completion-shell", initialIndex: detected === "bash" ? 1 : 0 },
|
|
123
|
+
);
|
|
124
|
+
if (selected !== "zsh" && selected !== "bash") return;
|
|
125
|
+
const installed = await installShellCompletions(selected);
|
|
126
|
+
ui.notify(`Installed completions at ${installed.scriptPath}; sourced from ${installed.rcPath}.`);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function setupValueFlag(argv: readonly string[], name: "--answers" | "--save-answers"): string | undefined {
|
|
130
|
+
const inline = argv.find((arg) => arg.startsWith(`${name}=`));
|
|
131
|
+
const at = argv.indexOf(name);
|
|
132
|
+
if (inline !== undefined && at !== -1) throw new Error(`${name} may be passed only once`);
|
|
133
|
+
const value = inline?.slice(name.length + 1) ?? (at === -1 ? undefined : argv[at + 1]);
|
|
134
|
+
if (value === undefined) {
|
|
135
|
+
if (at !== -1) throw new Error(`${name} requires a file path`);
|
|
136
|
+
return undefined;
|
|
137
|
+
}
|
|
138
|
+
if (value.length === 0 || value.startsWith("--")) throw new Error(`${name} requires a file path`);
|
|
139
|
+
return value;
|
|
140
|
+
}
|
|
141
|
+
|
|
66
142
|
export async function setupCommand(ctx: CommandContext): Promise<void> {
|
|
67
143
|
// Help first, and only in the first trailing position: a help request
|
|
68
144
|
// must never open a UI, read config, probe GitHub or pause dispatch.
|
|
@@ -74,10 +150,22 @@ if (ctx.argv[1] === "--help" || ctx.argv[1] === "-h") {
|
|
|
74
150
|
}
|
|
75
151
|
const sub = ctx.argv[1];
|
|
76
152
|
const positional = sub !== undefined && !sub.startsWith("--") ? sub : undefined;
|
|
77
|
-
//
|
|
78
|
-
//
|
|
79
|
-
|
|
80
|
-
const
|
|
153
|
+
// Clack owns an interactive terminal. Pipes retain the byte-pinned readline
|
|
154
|
+
// protocol, while --answers installs a prompt-free driver.
|
|
155
|
+
const answersPath = setupValueFlag(ctx.argv, "--answers");
|
|
156
|
+
const saveAnswersPath = setupValueFlag(ctx.argv, "--save-answers");
|
|
157
|
+
const useClack = answersPath === undefined && interactiveUi();
|
|
158
|
+
const baseUi = answersPath
|
|
159
|
+
? answersUi(loadAnswersFile(answersPath))
|
|
160
|
+
: useClack
|
|
161
|
+
? (await import("../clack-ui.ts")).clackUi()
|
|
162
|
+
: terminalUi();
|
|
163
|
+
const guardedUi = answersPath === undefined && !process.stdin.isTTY ? guardNonInteractiveUi(baseUi) : baseUi;
|
|
164
|
+
const recording: RecordedAnswersUi | undefined =
|
|
165
|
+
saveAnswersPath === undefined ? undefined : recordingAnswersUi(guardedUi);
|
|
166
|
+
const ui = recording?.ui ?? guardedUi;
|
|
167
|
+
let saveCompletedAnswers = false;
|
|
168
|
+
let closeMessage = "Setup failed.";
|
|
81
169
|
try {
|
|
82
170
|
// `host` and `graph` are install subcommands, checked BEFORE the amend
|
|
83
171
|
// areas. They are not areas — routing them through the area validation
|
|
@@ -119,6 +207,8 @@ try {
|
|
|
119
207
|
// `staged` is a success on a host that has no systemd: the files are
|
|
120
208
|
// real, only the enable step is impossible.
|
|
121
209
|
if (outcome.kind === "refused" || outcome.kind === "failed") process.exit(1);
|
|
210
|
+
saveCompletedAnswers = outcome.kind !== "declined";
|
|
211
|
+
closeMessage = outcome.kind === "declined" ? "Setup incomplete." : "Setup complete.";
|
|
122
212
|
return;
|
|
123
213
|
}
|
|
124
214
|
let area: AmendAreaId | undefined;
|
|
@@ -136,8 +226,35 @@ try {
|
|
|
136
226
|
// is still asked, nothing is proposed. For a host with no omp peer, a
|
|
137
227
|
// private repo no probe can clone, or an operator who would rather type the
|
|
138
228
|
// gates than review a model's reading of their CI.
|
|
139
|
-
|
|
229
|
+
const completed = await setup(
|
|
230
|
+
ui,
|
|
231
|
+
ctx.projectFlag,
|
|
232
|
+
area,
|
|
233
|
+
ctx.argv.includes("--no-ai") ? NO_PROBES : DEFAULT_PROBES,
|
|
234
|
+
);
|
|
235
|
+
if (completed && process.stdin.isTTY && process.stdout.isTTY) {
|
|
236
|
+
try {
|
|
237
|
+
await offerCompletionInstall(ui);
|
|
238
|
+
} catch (err) {
|
|
239
|
+
ui.notify(
|
|
240
|
+
`Skipped completion install: ${err instanceof Error ? err.message : String(err)}`,
|
|
241
|
+
"warning",
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
saveCompletedAnswers = completed;
|
|
246
|
+
closeMessage = completed ? "Setup complete." : "Setup incomplete.";
|
|
140
247
|
} finally {
|
|
141
|
-
|
|
248
|
+
try {
|
|
249
|
+
if (saveCompletedAnswers && saveAnswersPath !== undefined && recording !== undefined) {
|
|
250
|
+
saveAnswersFile(saveAnswersPath, recording.answers);
|
|
251
|
+
ui.notify(`Saved setup answers to ${saveAnswersPath}.`);
|
|
252
|
+
}
|
|
253
|
+
} catch (err) {
|
|
254
|
+
closeMessage = "Setup failed.";
|
|
255
|
+
throw err;
|
|
256
|
+
} finally {
|
|
257
|
+
ui.close(closeMessage);
|
|
258
|
+
}
|
|
142
259
|
}
|
|
143
260
|
}
|
package/src/commands/stats.ts
CHANGED
|
@@ -13,6 +13,7 @@ import type { CommandContext } from "./context.ts";
|
|
|
13
13
|
import { findProject, loadConfig } from "../config.ts";
|
|
14
14
|
import { dbPath, openStore, utcDay } from "../store.ts";
|
|
15
15
|
import { computeStats, renderStatsHuman, type StatsWindow } from "../stats.ts";
|
|
16
|
+
import { heading } from "../ui/style.ts";
|
|
16
17
|
|
|
17
18
|
const DAY_MS = 24 * 60 * 60 * 1_000;
|
|
18
19
|
const DATE_FORM = /^\d{4}-\d{2}-\d{2}$/;
|
|
@@ -120,11 +121,14 @@ export async function statsCommand(ctx: CommandContext): Promise<void> {
|
|
|
120
121
|
ghCalls: store.ghCallsBetween(window.sinceDay, window.untilDay),
|
|
121
122
|
runs: store.statsRuns(project.name, window.sinceEpochMs),
|
|
122
123
|
});
|
|
123
|
-
process.stdout.write(
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
124
|
+
if (ctx.argv.includes("--json")) process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
125
|
+
else {
|
|
126
|
+
const human = renderStatsHuman(report);
|
|
127
|
+
const newline = human.indexOf("\n");
|
|
128
|
+
process.stdout.write(
|
|
129
|
+
newline < 0 ? heading(human) : `${heading(human.slice(0, newline))}${human.slice(newline)}`,
|
|
130
|
+
);
|
|
131
|
+
}
|
|
128
132
|
} finally {
|
|
129
133
|
store.close();
|
|
130
134
|
}
|
package/src/commands/status.ts
CHANGED
|
@@ -10,8 +10,9 @@ import type { CommandContext } from "./context.ts";
|
|
|
10
10
|
import { readFileSync } from "node:fs";
|
|
11
11
|
import { join } from "node:path";
|
|
12
12
|
import { stateDir } from "../config.ts";
|
|
13
|
-
import {
|
|
13
|
+
import { collectFleetStatus, renderFleetStatusReport } from "../fleet.ts";
|
|
14
14
|
import { STALL_MARKER_FILE } from "../orchestrator-tick.ts";
|
|
15
|
+
import { fail, heading, ok, warn } from "../ui/style.ts";
|
|
15
16
|
|
|
16
17
|
/**
|
|
17
18
|
* The orchestrator half, and the one thing `status` has ever known about the
|
|
@@ -40,9 +41,35 @@ function stallLine(): string | undefined {
|
|
|
40
41
|
return `orchestrator STALLED since ${since === "" ? "an unrecorded time" : since}${diagnosis}`;
|
|
41
42
|
}
|
|
42
43
|
|
|
44
|
+
function styleStatus(text: string): string {
|
|
45
|
+
return text
|
|
46
|
+
.split("\n")
|
|
47
|
+
.map((line) => {
|
|
48
|
+
if (
|
|
49
|
+
line === "caps" ||
|
|
50
|
+
line === "daemon" ||
|
|
51
|
+
line === "active runs" ||
|
|
52
|
+
line === "active runs (none)" ||
|
|
53
|
+
line.startsWith("project ")
|
|
54
|
+
)
|
|
55
|
+
return heading(line);
|
|
56
|
+
if (line.includes("STALLED")) return fail(line);
|
|
57
|
+
if (/^(dispatch|ticks|pane|herdr|telegram| healthz)\s+.*\b(running|healthy|ok)\b/.test(line))
|
|
58
|
+
return ok(line);
|
|
59
|
+
if (/^(dispatch|ticks|daemon)\s+.*\b(paused|stopped|not running|overdue)\b/.test(line))
|
|
60
|
+
return warn(line);
|
|
61
|
+
return line;
|
|
62
|
+
})
|
|
63
|
+
.join("\n");
|
|
64
|
+
}
|
|
65
|
+
|
|
43
66
|
export async function statusCommand(ctx: CommandContext): Promise<void> {
|
|
44
|
-
const
|
|
45
|
-
const
|
|
46
|
-
|
|
47
|
-
process.stdout.write(`${
|
|
67
|
+
const report = await collectFleetStatus(ctx.projectFlag);
|
|
68
|
+
const stalled = stallLine();
|
|
69
|
+
if (ctx.argv.includes("--json")) {
|
|
70
|
+
process.stdout.write(`${JSON.stringify({ ...report, orchestratorStall: stalled }, null, 2)}\n`);
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
const text = styleStatus(renderFleetStatusReport(report));
|
|
74
|
+
process.stdout.write(`${text}${stalled === undefined ? "\n" : `\n\n${fail(stalled)}\n`}`);
|
|
48
75
|
}
|
package/src/commands/tail.ts
CHANGED
|
@@ -13,6 +13,7 @@ import { findProject, loadConfig } from "../config.ts";
|
|
|
13
13
|
import { dbPath, LIVE_STATES, openStore } from "../store.ts";
|
|
14
14
|
import { formatTranscriptLine, prop } from "../transcript.ts";
|
|
15
15
|
import type { Store } from "../types.ts";
|
|
16
|
+
import { dim, fail, ok, warn } from "../ui/style.ts";
|
|
16
17
|
|
|
17
18
|
/** How often `tail` re-stats the transcripts it is following. */
|
|
18
19
|
const TAIL_POLL_MS = 1_000;
|
|
@@ -142,6 +143,16 @@ export function renderAdvisorLine(line: string): string | undefined {
|
|
|
142
143
|
return out.length === 0 ? undefined : `[advisor] ${out.join("\n")}`;
|
|
143
144
|
}
|
|
144
145
|
|
|
146
|
+
function styleTailLine(line: string): string {
|
|
147
|
+
if (line.startsWith("[advisor] blocker:")) return fail(line);
|
|
148
|
+
if (line.startsWith("[advisor] warning:")) return warn(line);
|
|
149
|
+
if (line.startsWith("[advisor] nit:")) return dim(line);
|
|
150
|
+
if (line === "run ended: merged" || line === "run ended: pushed-green") return ok(line);
|
|
151
|
+
if (/^run ended: (?:failed|killed|orphaned)$/.test(line)) return fail(line);
|
|
152
|
+
if (line.startsWith("run ended:")) return warn(line);
|
|
153
|
+
return line;
|
|
154
|
+
}
|
|
155
|
+
|
|
145
156
|
/**
|
|
146
157
|
* Follow one run's transcripts the way `tail -f` follows a log: the worker's
|
|
147
158
|
* own transcript plus any `__advisor*.jsonl` sitting beside it (the mid-run
|
|
@@ -173,7 +184,8 @@ export async function tailRun(
|
|
|
173
184
|
} = {},
|
|
174
185
|
): Promise<void> {
|
|
175
186
|
const store = deps.store ?? openStore(dbPath());
|
|
176
|
-
const write =
|
|
187
|
+
const write =
|
|
188
|
+
deps.write ?? ((line: string) => process.stdout.write(`${styleTailLine(line)}\n`));
|
|
177
189
|
const pollMs = deps.pollMs ?? TAIL_POLL_MS;
|
|
178
190
|
const quietMs = deps.quietMs ?? TAIL_QUIET_MS;
|
|
179
191
|
try {
|
package/src/commands/watch.ts
CHANGED
|
@@ -50,18 +50,27 @@ export async function watchCommand(ctx: CommandContext): Promise<void> {
|
|
|
50
50
|
}
|
|
51
51
|
|
|
52
52
|
if (sub === "list" || sub === undefined) {
|
|
53
|
-
const
|
|
53
|
+
const open = store.openDecisions(project.name).filter((d) => d.kind === "watch");
|
|
54
|
+
const now = Date.now();
|
|
55
|
+
const watches = open.map((d) => ({
|
|
56
|
+
id: d.id,
|
|
57
|
+
ageHours: Math.max(0, Math.round((now - d.askedAt) / 3_600_000)),
|
|
58
|
+
blocks: d.blocks ?? null,
|
|
59
|
+
condition: d.condition === undefined ? null : d.conditionMetAt === undefined ? "pending" : "met",
|
|
60
|
+
note: d.question,
|
|
61
|
+
}));
|
|
62
|
+
if (ctx.argv.includes("--json")) {
|
|
63
|
+
process.stdout.write(`${JSON.stringify({ project: project.name, watches }, null, 2)}\n`);
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
54
66
|
if (watches.length === 0) {
|
|
55
67
|
process.stdout.write("no watches\n");
|
|
56
68
|
return;
|
|
57
69
|
}
|
|
58
|
-
const
|
|
59
|
-
for (const d of watches) {
|
|
60
|
-
const condition =
|
|
61
|
-
d.condition === undefined ? "-" : d.conditionMetAt === undefined ? "pending" : "met";
|
|
62
|
-
const hours = Math.max(0, Math.round((now - d.askedAt) / 3_600_000));
|
|
70
|
+
for (const watch of watches) {
|
|
63
71
|
process.stdout.write(
|
|
64
|
-
`${
|
|
72
|
+
`${watch.id} ${watch.ageHours}h blocks:${watch.blocks ?? "-"} ` +
|
|
73
|
+
`condition:${watch.condition ?? "-"} ${watch.note}\n`,
|
|
65
74
|
);
|
|
66
75
|
}
|
|
67
76
|
return;
|
package/src/fleet.ts
CHANGED
|
@@ -37,7 +37,7 @@ import { dbPath, openStore } from "./store.ts";
|
|
|
37
37
|
import { renderBriefForProject } from "./setup.ts";
|
|
38
38
|
import type { DaemonStop, ProjectConfig, Store } from "./types.ts";
|
|
39
39
|
import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
|
|
40
|
-
import { isPaused, setPaused, statusSnapshot } from "./daemon.ts";
|
|
40
|
+
import { isPaused, setPaused, statusSnapshot, type StatusSnapshot } from "./daemon.ts";
|
|
41
41
|
import {
|
|
42
42
|
healthCheck,
|
|
43
43
|
isAlive,
|
|
@@ -1311,8 +1311,24 @@ function servedProjectName(payload: object): string | undefined {
|
|
|
1311
1311
|
return undefined;
|
|
1312
1312
|
}
|
|
1313
1313
|
|
|
1314
|
-
export
|
|
1315
|
-
|
|
1314
|
+
export type FleetStatusReport = StatusSnapshot & {
|
|
1315
|
+
observedAt: number;
|
|
1316
|
+
layers: FleetLayers;
|
|
1317
|
+
daemon: FleetDaemonProbe | undefined;
|
|
1318
|
+
telegram: TelegramHealth;
|
|
1319
|
+
codeGraph: CodeGraphHealth;
|
|
1320
|
+
brief: string | undefined;
|
|
1321
|
+
decisions: string | undefined;
|
|
1322
|
+
failureClasses: string | undefined;
|
|
1323
|
+
workerPhases: { issue: number; phase: WorkerPausePhase }[];
|
|
1324
|
+
intake: string | undefined;
|
|
1325
|
+
lastStop: DaemonStop | undefined;
|
|
1326
|
+
siblings: { project: string; live: number }[];
|
|
1327
|
+
};
|
|
1328
|
+
|
|
1329
|
+
/** Collects the complete status payload once for both text and JSON renderers. */
|
|
1330
|
+
export async function collectFleetStatus(projectName?: string): Promise<FleetStatusReport> {
|
|
1331
|
+
const snapshot = statusSnapshot(projectName);
|
|
1316
1332
|
const layers = fleetLayers(projectName);
|
|
1317
1333
|
const project = findProject(loadConfig(), projectName);
|
|
1318
1334
|
const rec = livingDaemon();
|
|
@@ -1325,7 +1341,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1325
1341
|
// Read here rather than in `statusSnapshot`, which is synchronous and used
|
|
1326
1342
|
// by callers that must not shell out. An unmetered project never spawns
|
|
1327
1343
|
// the provider at all.
|
|
1328
|
-
readPlanUsage(
|
|
1344
|
+
readPlanUsage(snapshot.caps.planUsage, sharedUsageSource()),
|
|
1329
1345
|
// Same reasoning as `planUsage`: the snapshot is synchronous, this read is
|
|
1330
1346
|
// a shell-out, and undefined on any failure — one missing row, never a
|
|
1331
1347
|
// broken report (#188).
|
|
@@ -1347,7 +1363,9 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1347
1363
|
const healthBody = projectHealth.kind === "ok" ? rawHealth?.body : undefined;
|
|
1348
1364
|
const cached = codeGraphFromHealthz(healthBody, project.name);
|
|
1349
1365
|
const codeGraph = cached ?? (await probeCodeGraph(project));
|
|
1350
|
-
const workerPhases = workerPhasesFromHealthz(healthBody, project.name)
|
|
1366
|
+
const workerPhases = [...workerPhasesFromHealthz(healthBody, project.name)].map(
|
|
1367
|
+
([issue, phase]) => ({ issue, phase }),
|
|
1368
|
+
);
|
|
1351
1369
|
// The newest host-wide stop/restart provenance (#378). Read here — not in
|
|
1352
1370
|
// `statusSnapshot`, which is synchronous and belongs to the daemon module —
|
|
1353
1371
|
// and rendered identically from either project: the daemon_stops table is
|
|
@@ -1359,32 +1377,54 @@ export async function renderStatus(projectName?: string): Promise<string> {
|
|
|
1359
1377
|
try {
|
|
1360
1378
|
lastStop = store.latestDaemonStop();
|
|
1361
1379
|
// Shared-daemon visibility (#545): every configured project other than the
|
|
1362
|
-
// one being viewed, with its live-run count.
|
|
1363
|
-
// as the provenance above, so a status reader can tell "my fleet is idle"
|
|
1364
|
-
// from "the process I am about to stop is busy".
|
|
1380
|
+
// one being viewed, with its live-run count.
|
|
1365
1381
|
siblings = loadConfig()
|
|
1366
1382
|
.projects.filter((p) => p.name !== project.name)
|
|
1367
1383
|
.map((p) => ({ project: p.name, live: store.liveRuns(p.name).length }));
|
|
1368
1384
|
} finally {
|
|
1369
1385
|
store.close();
|
|
1370
1386
|
}
|
|
1371
|
-
return
|
|
1372
|
-
|
|
1387
|
+
return {
|
|
1388
|
+
...snapshot,
|
|
1389
|
+
planUsage,
|
|
1390
|
+
github,
|
|
1391
|
+
observedAt: Date.now(),
|
|
1373
1392
|
layers,
|
|
1374
1393
|
daemon,
|
|
1375
1394
|
telegram,
|
|
1376
|
-
Date.now(),
|
|
1377
1395
|
codeGraph,
|
|
1378
|
-
briefStatusLine(project),
|
|
1379
|
-
decisionStatusLine(project.name),
|
|
1380
|
-
failureClassBlock(project.name),
|
|
1396
|
+
brief: briefStatusLine(project),
|
|
1397
|
+
decisions: decisionStatusLine(project.name),
|
|
1398
|
+
failureClasses: failureClassBlock(project.name),
|
|
1381
1399
|
workerPhases,
|
|
1382
|
-
intakeStatusLine(project.name),
|
|
1400
|
+
intake: intakeStatusLine(project.name),
|
|
1383
1401
|
lastStop,
|
|
1384
1402
|
siblings,
|
|
1403
|
+
};
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
export function renderFleetStatusReport(report: FleetStatusReport): string {
|
|
1407
|
+
return formatFleetStatus(
|
|
1408
|
+
report,
|
|
1409
|
+
report.layers,
|
|
1410
|
+
report.daemon,
|
|
1411
|
+
report.telegram,
|
|
1412
|
+
report.observedAt,
|
|
1413
|
+
report.codeGraph,
|
|
1414
|
+
report.brief,
|
|
1415
|
+
report.decisions,
|
|
1416
|
+
report.failureClasses,
|
|
1417
|
+
new Map(report.workerPhases.map(({ issue, phase }) => [issue, phase])),
|
|
1418
|
+
report.intake,
|
|
1419
|
+
report.lastStop,
|
|
1420
|
+
report.siblings,
|
|
1385
1421
|
);
|
|
1386
1422
|
}
|
|
1387
1423
|
|
|
1424
|
+
export async function renderStatus(projectName?: string): Promise<string> {
|
|
1425
|
+
return renderFleetStatusReport(await collectFleetStatus(projectName));
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1388
1428
|
/**
|
|
1389
1429
|
* One line naming the brief layout, or nothing when it cannot be read.
|
|
1390
1430
|
*
|
package/src/orchestrator-tick.ts
CHANGED
|
@@ -103,7 +103,7 @@ import {
|
|
|
103
103
|
type AskResult,
|
|
104
104
|
} from "./ask.ts";
|
|
105
105
|
import { deliverOperatorMessage } from "./reports.ts";
|
|
106
|
-
import type { RunRecord } from "./types.ts";
|
|
106
|
+
import type { RecoveryAction, RunRecord } from "./types.ts";
|
|
107
107
|
import { dbPath, openStore } from "./store.ts";
|
|
108
108
|
import { digestDue, localDayKey } from "./digest-schedule.ts";
|
|
109
109
|
import { heldNoticeId } from "./notices.ts";
|
|
@@ -471,6 +471,34 @@ export function defaultTickMessage(
|
|
|
471
471
|
* `REPORT_SCOPES` fails to compile here instead of resolving to `undefined` at
|
|
472
472
|
* the point of use.
|
|
473
473
|
*/
|
|
474
|
+
/**
|
|
475
|
+
* Which recovery actions resolve a run without leaving human work behind.
|
|
476
|
+
* Exhaustive over {@link RecoveryAction} so adding an action to the vocabulary
|
|
477
|
+
* forces a decision here: a recovered row carrying `true` is genuinely done and
|
|
478
|
+
* may be summarised as "already handled"; `false` — `escalate`, `hold`, `none` —
|
|
479
|
+
* still needs the orchestrator's Duty 1 attention, so it must never inherit the
|
|
480
|
+
* suppressive "do not re-triage" sentence (#610).
|
|
481
|
+
*/
|
|
482
|
+
const AUTONOMOUS_RECOVERY_ACTIONS: Record<RecoveryAction, boolean> = {
|
|
483
|
+
requeue: true,
|
|
484
|
+
continue: true,
|
|
485
|
+
"rerun-checks": true,
|
|
486
|
+
settle: true,
|
|
487
|
+
escalate: false,
|
|
488
|
+
hold: false,
|
|
489
|
+
none: false,
|
|
490
|
+
};
|
|
491
|
+
|
|
492
|
+
/** Bounded name-and-count summary shared by the two recovered-row groups. */
|
|
493
|
+
function recoveredSummary(group: readonly RunRecord[]): string {
|
|
494
|
+
const named = group
|
|
495
|
+
.slice(0, 5)
|
|
496
|
+
.map((r) => `${r.failureClass ?? "unknown"} #${r.issue}`)
|
|
497
|
+
.join(", ");
|
|
498
|
+
const rest = group.length > 5 ? `, +${group.length - 5} more` : "";
|
|
499
|
+
return `${group.length} (${named}${rest})`;
|
|
500
|
+
}
|
|
501
|
+
|
|
474
502
|
/**
|
|
475
503
|
* One line naming what the daemon recovered without asking (#132).
|
|
476
504
|
*
|
|
@@ -480,12 +508,23 @@ export function defaultTickMessage(
|
|
|
480
508
|
*/
|
|
481
509
|
export function recoveryDigestLine(recovered: readonly RunRecord[]): string | undefined {
|
|
482
510
|
if (recovered.length === 0) return undefined;
|
|
483
|
-
const
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
.
|
|
487
|
-
|
|
488
|
-
|
|
511
|
+
const handled: RunRecord[] = [];
|
|
512
|
+
const triage: RunRecord[] = [];
|
|
513
|
+
for (const r of recovered) {
|
|
514
|
+
if (r.recoveryAction !== undefined && AUTONOMOUS_RECOVERY_ACTIONS[r.recoveryAction]) {
|
|
515
|
+
handled.push(r);
|
|
516
|
+
} else {
|
|
517
|
+
triage.push(r);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
const lines: string[] = [];
|
|
521
|
+
if (handled.length > 0) {
|
|
522
|
+
lines.push(`Auto-recovered since last tick: ${recoveredSummary(handled)} — already handled, do not re-triage these.`);
|
|
523
|
+
}
|
|
524
|
+
if (triage.length > 0) {
|
|
525
|
+
lines.push(`Recovered but needs Duty 1 triage: ${recoveredSummary(triage)} — inspect these runs.`);
|
|
526
|
+
}
|
|
527
|
+
return lines.join("\n");
|
|
489
528
|
}
|
|
490
529
|
|
|
491
530
|
/**
|
package/src/privileged.ts
CHANGED
|
@@ -88,6 +88,8 @@ export interface RunPrivilegedOptions {
|
|
|
88
88
|
deps?: PrivilegedDeps;
|
|
89
89
|
/** Confirm title. Defaults to a generic one; callers name their verb. */
|
|
90
90
|
title?: string;
|
|
91
|
+
/** Stable answer-file key for this batch's confirmation. */
|
|
92
|
+
answerKey?: string;
|
|
91
93
|
/** Extra lines shown above the step list — what this batch is for. */
|
|
92
94
|
preamble?: readonly string[];
|
|
93
95
|
/**
|
|
@@ -201,6 +203,7 @@ export async function runPrivileged(
|
|
|
201
203
|
const go = await ui.confirm(
|
|
202
204
|
options.title ?? "Run these steps now?",
|
|
203
205
|
`${steps.length} step(s), in the order shown. Anything that fails stops the rest and prints what is left.`,
|
|
206
|
+
{ key: options.answerKey ?? "run-privileged-steps" },
|
|
204
207
|
);
|
|
205
208
|
// `undefined` is a dismissal rather than a "no", but for a batch that has not
|
|
206
209
|
// started they mean the same thing: run nothing.
|