omp-conductor 0.17.1 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/REFERENCE.md +71 -17
- package/agents/to-spec.md +90 -0
- package/package.json +2 -1
- package/schema/config.schema.json +53 -1
- package/src/admission.ts +308 -76
- package/src/ask.ts +307 -10
- package/src/backups.ts +2 -2
- package/src/board.ts +17 -3
- package/src/briefs/orchestrator.md +43 -14
- package/src/briefs/to-spec.md +84 -0
- package/src/briefs/worker.md +37 -19
- package/src/cli.ts +2 -0
- package/src/command-help.ts +19 -1
- package/src/command-manifest.ts +27 -2
- package/src/commands/context.ts +1 -0
- package/src/commands/drain.ts +176 -0
- package/src/commands/extend.ts +6 -10
- package/src/commands/status.ts +5 -1
- package/src/commands/watch.ts +110 -3
- package/src/commands/worker.ts +9 -10
- package/src/config-schema.ts +57 -0
- package/src/config.ts +102 -2
- package/src/daemon.ts +1220 -1517
- package/src/dashboard/app.js +4 -1
- package/src/dashboard/server.ts +5 -2
- package/src/decisions.ts +279 -16
- package/src/depends-on.ts +261 -1
- package/src/diff-flags.ts +425 -1
- package/src/digest-schedule.ts +37 -0
- package/src/doctor.ts +52 -0
- package/src/escalate.ts +9 -3
- package/src/failure-class.ts +43 -4
- package/src/fleet.ts +166 -24
- package/src/gitops.ts +188 -81
- package/src/graph-health.ts +55 -8
- package/src/graph.ts +379 -69
- package/src/harness-loader.ts +59 -0
- package/src/host.ts +567 -2
- package/src/lifecycle.ts +158 -6
- package/src/omp.ts +269 -20
- package/src/orchestrator-tick.ts +1489 -26
- package/src/orchestrator.ts +12 -0
- package/src/privileged.ts +1 -4
- package/src/release-policy.ts +503 -9
- package/src/routing.ts +11 -3
- package/src/session-host.ts +115 -5
- package/src/settlement.ts +1780 -0
- package/src/setup-host.ts +1205 -6
- package/src/setup-install.ts +119 -30
- package/src/setup-wizard.ts +88 -2
- package/src/setup.ts +119 -13
- package/src/shell.ts +15 -0
- package/src/status-render.ts +100 -11
- package/src/store.ts +519 -45
- package/src/to-spec.ts +387 -0
- package/src/tracker/github.ts +150 -14
- package/src/types.ts +470 -16
- package/src/upgrade-verify.ts +209 -2
- package/src/upgrade.ts +175 -1
- package/src/verbs/protocol.ts +39 -0
- package/src/verbs/server.ts +770 -40
- package/src/verbs/socket.ts +24 -5
- package/src/worker.ts +239 -9
- package/src/worktree.ts +142 -18
package/src/setup.ts
CHANGED
|
@@ -45,6 +45,7 @@ import {
|
|
|
45
45
|
resolveCaps,
|
|
46
46
|
resolvePolicy,
|
|
47
47
|
resolveReleaseGrants,
|
|
48
|
+
resolveReview,
|
|
48
49
|
SCOPE_PRESETS,
|
|
49
50
|
stateDir,
|
|
50
51
|
} from "./config.ts";
|
|
@@ -57,8 +58,10 @@ import {
|
|
|
57
58
|
DEFAULT_CAPS,
|
|
58
59
|
DEFAULT_PROJECT_POLICY,
|
|
59
60
|
DEFAULT_REPORT_SCOPE,
|
|
61
|
+
DEFAULT_REVIEW_POLICY,
|
|
60
62
|
DENIED_RELEASE_GRANTS,
|
|
61
63
|
RELEASE_SHAPES,
|
|
64
|
+
REVIEW_STRICTNESS,
|
|
62
65
|
WEEKDAYS,
|
|
63
66
|
type ArmProof,
|
|
64
67
|
type BaseFreshness,
|
|
@@ -75,6 +78,8 @@ import {
|
|
|
75
78
|
type ReleaseRequirement,
|
|
76
79
|
type ReportScopeChoice,
|
|
77
80
|
type ReportingPolicy,
|
|
81
|
+
type ReviewPolicy,
|
|
82
|
+
type ReviewStrictness,
|
|
78
83
|
type Weekday,
|
|
79
84
|
type WeeklyAvailability,
|
|
80
85
|
type RepoTarget,
|
|
@@ -138,7 +143,7 @@ export interface SetupAnswers {
|
|
|
138
143
|
/** Tracker repo as `owner/repo` — the only spelling `gh` takes without a host. */
|
|
139
144
|
trackerRepo: string;
|
|
140
145
|
queueLabel: string;
|
|
141
|
-
stateLabels: { inProgress: string; blocked: string; failed: string };
|
|
146
|
+
stateLabels: { inProgress: string; blocked: string; failed: string; backlog: string };
|
|
142
147
|
routingLabelPrefix: string;
|
|
143
148
|
targetRepos: {
|
|
144
149
|
name: string;
|
|
@@ -215,6 +220,13 @@ export interface SetupAnswers {
|
|
|
215
220
|
* leave it to be defaulted by whichever reader gets there first.
|
|
216
221
|
*/
|
|
217
222
|
armProof: ArmProof;
|
|
223
|
+
/**
|
|
224
|
+
* How green PRs are reviewed and returned (#678): the strictness level and
|
|
225
|
+
* the hard ceiling on review rounds per PR lifecycle. Always complete: the
|
|
226
|
+
* wizard asks about it in the policy area, so an answers object can never
|
|
227
|
+
* leave it to be defaulted by whichever reader gets there first.
|
|
228
|
+
*/
|
|
229
|
+
review: ReviewPolicy;
|
|
218
230
|
/**
|
|
219
231
|
* Hand-edited recovery merge authorizations carried through setup unchanged.
|
|
220
232
|
* The wizard never grants one; forgetting them during an unrelated amend
|
|
@@ -305,7 +317,12 @@ export interface LabelPlan {
|
|
|
305
317
|
* cannot drift apart: one spelling of "ready-for-agent" in the package. */
|
|
306
318
|
export const SETUP_DEFAULTS = {
|
|
307
319
|
queueLabel: "ready-for-agent",
|
|
308
|
-
stateLabels: {
|
|
320
|
+
stateLabels: {
|
|
321
|
+
inProgress: "agent:in-progress",
|
|
322
|
+
blocked: "agent:blocked",
|
|
323
|
+
failed: "agent:failed",
|
|
324
|
+
backlog: "backlog",
|
|
325
|
+
},
|
|
309
326
|
routingLabelPrefix: "repo:",
|
|
310
327
|
defaultBranch: "main",
|
|
311
328
|
/** Both authorities start with the human; the wizard asks to move each one. */
|
|
@@ -314,6 +331,8 @@ export const SETUP_DEFAULTS = {
|
|
|
314
331
|
releaseGrants: DENIED_RELEASE_GRANTS,
|
|
315
332
|
/** The strictest reading of the prose these conditions replaced (#129). */
|
|
316
333
|
policy: DEFAULT_PROJECT_POLICY,
|
|
334
|
+
/** The recommended review strictness and its default ceiling (#678). */
|
|
335
|
+
review: DEFAULT_REVIEW_POLICY,
|
|
317
336
|
/** The daemon runs its own triage session unless an operator already runs one. */
|
|
318
337
|
orchestratorMode: "embedded",
|
|
319
338
|
} as const;
|
|
@@ -405,6 +424,35 @@ export const ARM_PROOF_CHOICES: { readonly [K in ArmProof]: string } = {
|
|
|
405
424
|
"claim-only": "anyone who can invoke the already-privileged `omp-conductor arm` command can start dispatch once the live claim and poller pass",
|
|
406
425
|
};
|
|
407
426
|
|
|
427
|
+
/**
|
|
428
|
+
* What each review strictness blocks, in the operator's words, and the single
|
|
429
|
+
* canonical phrasing of the three bars (#678).
|
|
430
|
+
*
|
|
431
|
+
* Mapped over the closed union for the reason {@link MERGE_DUTY} is: a fourth
|
|
432
|
+
* level fails to compile here instead of reaching a wizard with no question for
|
|
433
|
+
* it, an amend row that cannot describe it, and a rendered brief that shows it
|
|
434
|
+
* blank. Each sentence is self-contained — "Low's bar" would reference prose
|
|
435
|
+
* the brief may not be rendering — and it is the *same* text the wizard shows
|
|
436
|
+
* one level at a time and the composed orchestrator brief carries for the
|
|
437
|
+
* configured level, so the definition an operator chose is the threshold a
|
|
438
|
+
* session enforces.
|
|
439
|
+
*
|
|
440
|
+
* The two consumers read this map rather than their own prose: the setup
|
|
441
|
+
* dialog explains every level from it, and {@link reviewDuty} renders the
|
|
442
|
+
* effective one from it. "One canonical implementation, not duplicated prose
|
|
443
|
+
* branches" is this object.
|
|
444
|
+
*/
|
|
445
|
+
export const REVIEW_STRICTNESS_CHOICES: { readonly [K in ReviewStrictness]: string } = {
|
|
446
|
+
low: "correctness, security, data-loss or explicit acceptance-criteria failures — anything else stays a review comment",
|
|
447
|
+
medium:
|
|
448
|
+
"correctness, security, data-loss or explicit acceptance-criteria failures, plus material maintainability or " +
|
|
449
|
+
"reliability defects likely to become incidents within six months — anything else stays a review comment",
|
|
450
|
+
high:
|
|
451
|
+
"correctness, security, data-loss or explicit acceptance-criteria failures, plus material maintainability or " +
|
|
452
|
+
"reliability defects likely to become incidents within six months, plus concrete quality defects — never " +
|
|
453
|
+
"subjective style churn or unbounded refactoring",
|
|
454
|
+
};
|
|
455
|
+
|
|
408
456
|
export const RELEASE_REQUIREMENT_CHOICES: { readonly [K in ReleaseRequirement]: string } = {
|
|
409
457
|
"runs-settled": "every run of the released repo actually merged, not merely reached a green PR",
|
|
410
458
|
"fleet-runs-settled": "every run in the project actually merged — suite-wide strictness for shapes that consume several repos",
|
|
@@ -465,6 +513,34 @@ export const MERGE_DUTY: { readonly [K in ProjectConfig["authority"]["merge"]]:
|
|
|
465
513
|
" one is a hard boundary, not a preference.",
|
|
466
514
|
};
|
|
467
515
|
|
|
516
|
+
/**
|
|
517
|
+
* Duty 1's review-return contract, worded from `project.review` (#678).
|
|
518
|
+
*
|
|
519
|
+
* One canonical implementation rather than prose branches in the template: the
|
|
520
|
+
* configured level's bar is the same {@link REVIEW_STRICTNESS_CHOICES} text
|
|
521
|
+
* the setup dialog explains each level with, and the ceiling is the configured
|
|
522
|
+
* number — so the strictness an operator chose in setup and the threshold this
|
|
523
|
+
* session enforces cannot diverge. The paragraph names `conductor_pr_review`
|
|
524
|
+
* (the #677 verb) as the only return path and states the three ceiling
|
|
525
|
+
* behaviours: leave the PR open, record the unresolved findings, escalate once.
|
|
526
|
+
*/
|
|
527
|
+
export function reviewDuty(p: ProjectConfig): string {
|
|
528
|
+
const { strictness, maxRounds } = resolveReview(p);
|
|
529
|
+
return (
|
|
530
|
+
"\n" +
|
|
531
|
+
"**Review policy:** before you merge (or settle) a green PR, review it at\n" +
|
|
532
|
+
`this project's strictness. The level is **${strictness}**: ` +
|
|
533
|
+
`${REVIEW_STRICTNESS_CHOICES[strictness]}. ` +
|
|
534
|
+
"`conductor_pr_review` is the verb for findings at or above that bar, and\n" +
|
|
535
|
+
"for nothing else — anything below it stays a review comment on the PR,\n" +
|
|
536
|
+
"never a return. Every corrected head gets a fresh review, up to a hard\n" +
|
|
537
|
+
`ceiling of ${maxRounds} ${maxRounds === 1 ? "round" : "rounds"} per PR lifecycle, visible in ` +
|
|
538
|
+
"`omp-conductor status` as `review-revision N`. At the ceiling\n" +
|
|
539
|
+
"`conductor_pr_review` refuses: leave the PR open, record the unresolved\n" +
|
|
540
|
+
"findings, and escalate once — never a further round."
|
|
541
|
+
);
|
|
542
|
+
}
|
|
543
|
+
|
|
468
544
|
/**
|
|
469
545
|
* The Promotion paragraph, worded from `authority.promotion`.
|
|
470
546
|
*
|
|
@@ -506,13 +582,15 @@ const POLICY_TEMPLATE_PATH = join(import.meta.dir, "briefs", "policy.md");
|
|
|
506
582
|
const REQUIRED_SCOPES = ["repo", "project"] as const;
|
|
507
583
|
|
|
508
584
|
/** GitHub's own palette, so the tracker reads at a glance: green means queued,
|
|
509
|
-
* blue means moving, amber means waiting on you, red means it gave up,
|
|
510
|
-
*
|
|
585
|
+
* blue means moving, amber means waiting on you, red means it gave up, grey
|
|
586
|
+
* means the operator parked it, and purple means routing — the operator's
|
|
587
|
+
* input, the label the loop never writes. */
|
|
511
588
|
const LABEL_COLOURS = {
|
|
512
589
|
queue: "0e8a16",
|
|
513
590
|
inProgress: "1d76db",
|
|
514
591
|
blocked: "fbca04",
|
|
515
592
|
failed: "b60205",
|
|
593
|
+
backlog: "8b949e",
|
|
516
594
|
routing: "6f42c1",
|
|
517
595
|
} as const;
|
|
518
596
|
|
|
@@ -596,14 +674,15 @@ export async function checkTokenScopes(): Promise<ScopeCheck> {
|
|
|
596
674
|
|
|
597
675
|
/**
|
|
598
676
|
* Every label setup wants on the tracker, before existence is known: the queue
|
|
599
|
-
* and state labels the loop reads and writes,
|
|
600
|
-
*
|
|
601
|
-
*
|
|
677
|
+
* and state labels the loop reads and writes, the operator's park label (which
|
|
678
|
+
* the loop must never write), plus one routing label per routed repo. Pure —
|
|
679
|
+
* the prefix and the repo keys are already answered at this point in the
|
|
680
|
+
* interview — which is what lets a test pin the whole set without `gh`.
|
|
602
681
|
*
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
682
|
+
* The park label is provisioned like the rest but owned the other way round:
|
|
683
|
+
* it exists so the operator has a gesture, and the conductor never applies or
|
|
684
|
+
* removes it (#507). Creating it queues nothing either — applying one, with
|
|
685
|
+
* the queue label, stays the operator's sign-off.
|
|
607
686
|
*/
|
|
608
687
|
export function wantedLabels(a: SetupAnswers): Omit<LabelPlan, "exists">[] {
|
|
609
688
|
return [
|
|
@@ -627,6 +706,11 @@ export function wantedLabels(a: SetupAnswers): Omit<LabelPlan, "exists">[] {
|
|
|
627
706
|
colour: LABEL_COLOURS.failed,
|
|
628
707
|
description: "The conductor gave up on this issue after its retry budget",
|
|
629
708
|
},
|
|
709
|
+
{
|
|
710
|
+
name: a.stateLabels.backlog,
|
|
711
|
+
colour: LABEL_COLOURS.backlog,
|
|
712
|
+
description: "Parked by the operator — never claimed, only the operator may set or clear it",
|
|
713
|
+
},
|
|
630
714
|
...a.targetRepos.map((r) => ({
|
|
631
715
|
name: `${a.routingLabelPrefix}${r.name}`,
|
|
632
716
|
colour: LABEL_COLOURS.routing,
|
|
@@ -901,6 +985,10 @@ export function buildProject(a: SetupAnswers): ProjectConfig {
|
|
|
901
985
|
name: a.projectName,
|
|
902
986
|
tracker: { kind: "github", repo: a.trackerRepo },
|
|
903
987
|
queueLabel: a.queueLabel,
|
|
988
|
+
// The park label is now an answer like the other three: the wizard offers
|
|
989
|
+
// it and every answers constructor carries it, so writing the answers as
|
|
990
|
+
// they stand is the single source — slice 1's hardcoded default bridge is
|
|
991
|
+
// gone (#507 slice 2).
|
|
904
992
|
stateLabels: { ...a.stateLabels },
|
|
905
993
|
routing: { labelPrefix: a.routingLabelPrefix, repos },
|
|
906
994
|
caps,
|
|
@@ -928,6 +1016,10 @@ export function buildProject(a: SetupAnswers): ProjectConfig {
|
|
|
928
1016
|
// policy has a line in the file to point at — and the recovery playbook
|
|
929
1017
|
// can read which proof the project opted into (#613).
|
|
930
1018
|
arm: { proof: a.armProof },
|
|
1019
|
+
// Written out in full for the same reason as `arm`: the file then carries
|
|
1020
|
+
// the strictness and the ceiling an operator chose, without anyone having
|
|
1021
|
+
// to know a migration rule (#678).
|
|
1022
|
+
review: { ...a.review },
|
|
931
1023
|
...(a.recoveryMerges === undefined
|
|
932
1024
|
? {}
|
|
933
1025
|
: { recoveryMerges: a.recoveryMerges.map((entry) => ({ ...entry })) }),
|
|
@@ -1018,6 +1110,10 @@ export function defaultAnswers(projectName: string, opts: { added?: boolean } =
|
|
|
1018
1110
|
// The strict reading, matching the loader's absent-key default: a project
|
|
1019
1111
|
// that never answered keeps today's authenticated challenge round-trip.
|
|
1020
1112
|
armProof: DEFAULT_ARM_PROOF,
|
|
1113
|
+
// First-run default: the recommended strictness and its ceiling, so a
|
|
1114
|
+
// fresh project opens on the answer the issue recommends — and the loader
|
|
1115
|
+
// materialises the same value for a config that has no key at all.
|
|
1116
|
+
review: { ...SETUP_DEFAULTS.review },
|
|
1021
1117
|
orchestratorMode: SETUP_DEFAULTS.orchestratorMode,
|
|
1022
1118
|
reportScope: SETUP_DEFAULT_REPORT_SCOPE,
|
|
1023
1119
|
writeOrchestratorBrief: false,
|
|
@@ -1117,6 +1213,10 @@ export function answersFromProject(p: ProjectConfig): SetupAnswers {
|
|
|
1117
1213
|
// The loader materialises `arm` complete, so this is the answer the project
|
|
1118
1214
|
// actually has — never a default guessed at the amend prompt.
|
|
1119
1215
|
armProof: resolveArmProof(p),
|
|
1216
|
+
// Same reason as `armProof`: the loader materialises `review` complete, so
|
|
1217
|
+
// a re-run preserves the configured strictness and ceiling rather than
|
|
1218
|
+
// opening on a default the operator already answered.
|
|
1219
|
+
review: resolveReview(p),
|
|
1120
1220
|
orchestratorMode: p.escalation.orchestrator,
|
|
1121
1221
|
reportScope: reportScopeFromPolicy(p.reporting),
|
|
1122
1222
|
writeOrchestratorBrief: false,
|
|
@@ -1184,6 +1284,7 @@ function briefVarsForProject(p: ProjectConfig): Record<string, string> {
|
|
|
1184
1284
|
QUEUE_LABEL: p.queueLabel,
|
|
1185
1285
|
RELEASES_DEFAULT: RELEASES_DEFAULTS[`${p.authority.merge}/${p.authority.release}`],
|
|
1186
1286
|
MERGE_DUTY: MERGE_DUTY[p.authority.merge],
|
|
1287
|
+
REVIEW_DUTY: reviewDuty(p),
|
|
1187
1288
|
PROMOTION_DUTY: PROMOTION_DUTIES[p.authority.promotion ?? DEFAULT_AUTHORITY.promotion],
|
|
1188
1289
|
POLICY_SOURCE: policySourceLine(p),
|
|
1189
1290
|
};
|
|
@@ -1693,6 +1794,9 @@ export function summarisePlan(
|
|
|
1693
1794
|
` environments ${a.policy.release.environments.join(", ") || "none declared — every deploy target is refused"}`,
|
|
1694
1795
|
);
|
|
1695
1796
|
|
|
1797
|
+
lines.push("", "review", ` strictness ${a.review.strictness} — ${REVIEW_STRICTNESS_CHOICES[a.review.strictness]}`);
|
|
1798
|
+
lines.push(` max rounds/PR ${a.review.maxRounds} — at the ceiling the PR is left open, the findings recorded, and it is escalated once`);
|
|
1799
|
+
|
|
1696
1800
|
const reporting = project.reporting as ReportingPolicy;
|
|
1697
1801
|
const briefPath = orchestratorBriefPath(a);
|
|
1698
1802
|
lines.push(
|
|
@@ -1909,16 +2013,18 @@ export const AMEND_AREAS: {
|
|
|
1909
2013
|
},
|
|
1910
2014
|
policy: {
|
|
1911
2015
|
name: "merge & release preconditions",
|
|
1912
|
-
asks: "the checks, base freshness, draft rule and behind-base action for a merge, then what a release requires and what it ships",
|
|
2016
|
+
asks: "the checks, base freshness, draft rule and behind-base action for a merge, then what a release requires and what it ships — and how green PRs are reviewed, and how many rounds a PR may be returned",
|
|
1913
2017
|
describe: (p) => {
|
|
1914
2018
|
const policy = resolvePolicy(p);
|
|
2019
|
+
const review = resolveReview(p);
|
|
1915
2020
|
// Counted rather than listed: this row is elided at 96 characters, and the
|
|
1916
2021
|
// full table is in the plan summary the consent screen shows next.
|
|
1917
2022
|
return (
|
|
1918
2023
|
`merge: ${policy.merge.requiredChecks.length === 0 ? "every check" : `${policy.merge.requiredChecks.length} check(s)`}, ` +
|
|
1919
2024
|
`base ${policy.merge.baseFreshness}, drafts ${policy.merge.drafts}, behind → ${policy.merge.whenBehindBase}; ` +
|
|
1920
2025
|
`release: ${policy.release.requires.length} must-land, ${policy.release.artefacts.length} artefact(s), ` +
|
|
1921
|
-
`${policy.release.environments.length} env(s)`
|
|
2026
|
+
`${policy.release.environments.length} env(s); ` +
|
|
2027
|
+
`review ${review.strictness} ×${review.maxRounds}`
|
|
1922
2028
|
);
|
|
1923
2029
|
},
|
|
1924
2030
|
},
|
package/src/shell.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One CLI argument rendered exactly as an operator would type it, for surfaces
|
|
3
|
+
* that print executable commands.
|
|
4
|
+
*
|
|
5
|
+
* A command a surface prints is a command the reader copies, so an argument
|
|
6
|
+
* that could mean something else is a defect: a project named `odd $fleet`
|
|
7
|
+
* must not render `--project odd $fleet` (two arguments, and a `$` that the
|
|
8
|
+
* shell would read). Safe characters pass through unquoted; anything else is
|
|
9
|
+
* wrapped in single quotes, folding an embedded quote with the standard
|
|
10
|
+
* `'\''` — the same convention the privileged-step renderer applies to its
|
|
11
|
+
* printed `sudo …` lines (#810).
|
|
12
|
+
*/
|
|
13
|
+
export function shellQuote(value: string): string {
|
|
14
|
+
return /^[A-Za-z0-9_@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
|
|
15
|
+
}
|
package/src/status-render.ts
CHANGED
|
@@ -22,9 +22,9 @@ import type { DaemonStop } from "./types.ts";
|
|
|
22
22
|
import { settlementFlagSummary } from "./diff-flags.ts";
|
|
23
23
|
import type { CodeGraphHealth } from "./graph-health.ts";
|
|
24
24
|
import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
|
|
25
|
-
import { formatOrchestratorDown } from "./orchestrator-down.ts";
|
|
25
|
+
import { formatDownDuration, formatOrchestratorDown } from "./orchestrator-down.ts";
|
|
26
26
|
import { planUsageLine } from "./usage.ts";
|
|
27
|
-
import { SYSTEMD_UNIT, type UnitOwnership } from "./lifecycle.ts";
|
|
27
|
+
import { HEALTH_TIMEOUT_MS, SYSTEMD_UNIT, type UnitOwnership } from "./lifecycle.ts";
|
|
28
28
|
import type { WorkerPausePhase } from "./worker.ts";
|
|
29
29
|
import { formatRss, rssBytesFromHealthz } from "./host.ts";
|
|
30
30
|
import {
|
|
@@ -32,12 +32,12 @@ import {
|
|
|
32
32
|
formatDispatchSummary,
|
|
33
33
|
formatFreezes,
|
|
34
34
|
formatReleaseGrants,
|
|
35
|
-
formatSalvagedRuns,
|
|
36
35
|
isPaused,
|
|
37
36
|
pausedAt,
|
|
38
37
|
pauseProvenance,
|
|
39
38
|
type StatusSnapshot,
|
|
40
39
|
} from "./daemon.ts";
|
|
40
|
+
import { formatQuarantinedRuns, formatSalvagedRuns } from "./settlement.ts";
|
|
41
41
|
|
|
42
42
|
// layered status
|
|
43
43
|
// ---------------------------------------------------------------------------
|
|
@@ -90,6 +90,14 @@ export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()):
|
|
|
90
90
|
`code graph ${graph.status} ${indexed}/${graph.repos.length} repos indexed`,
|
|
91
91
|
` indexer ${graph.prerequisites.indexer}`,
|
|
92
92
|
` MCP mount ${graph.prerequisites.mcpMount}`,
|
|
93
|
+
// The runtime half of the mount finding (#726): what dispatched sessions
|
|
94
|
+
// actually held in their registries. Rendered only when a run recorded
|
|
95
|
+
// it — "observed" is never claimed without evidence.
|
|
96
|
+
...(graph.session.recorded === 0
|
|
97
|
+
? []
|
|
98
|
+
: [
|
|
99
|
+
` session observed in ${graph.session.recorded} run(s): present ${graph.session.present} / absent ${graph.session.absent}`,
|
|
100
|
+
]),
|
|
93
101
|
` timer ${graph.timer.enabled} / ${graph.timer.active}`,
|
|
94
102
|
` refresh ${refresh}`,
|
|
95
103
|
...graph.reasons.map((reason) => ` - ${reason}`),
|
|
@@ -108,6 +116,7 @@ export type DaemonProjectHealth =
|
|
|
108
116
|
| { kind: "stopped" }
|
|
109
117
|
| { kind: "ok" }
|
|
110
118
|
| { kind: "unreachable" }
|
|
119
|
+
| { kind: "unresponsive" }
|
|
111
120
|
| { kind: "other-project"; serves?: string };
|
|
112
121
|
|
|
113
122
|
/**
|
|
@@ -122,6 +131,14 @@ export type FleetDaemonProbe = {
|
|
|
122
131
|
unit?: UnitOwnership;
|
|
123
132
|
};
|
|
124
133
|
|
|
134
|
+
/**
|
|
135
|
+
* #685: the healthz timeout is a probe outcome, never a death verdict — the
|
|
136
|
+
* same discipline the indexer probe uses when it reports `probe timed out`
|
|
137
|
+
* instead of declaring the indexer missing.
|
|
138
|
+
*/
|
|
139
|
+
const UNRESPONSIVE_HEALTHZ =
|
|
140
|
+
`unresponsive (healthz timed out after ${HEALTH_TIMEOUT_MS / 1000}s)`;
|
|
141
|
+
|
|
125
142
|
function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
|
|
126
143
|
if (probe === undefined) return "unprobed";
|
|
127
144
|
switch (probe.project.kind) {
|
|
@@ -129,6 +146,8 @@ function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string {
|
|
|
129
146
|
return "stopped";
|
|
130
147
|
case "ok":
|
|
131
148
|
return "ok";
|
|
149
|
+
case "unresponsive":
|
|
150
|
+
return UNRESPONSIVE_HEALTHZ;
|
|
132
151
|
case "unreachable":
|
|
133
152
|
return "unreachable — the process is up but not serving";
|
|
134
153
|
case "other-project":
|
|
@@ -239,8 +258,13 @@ export function formatFleetStatus(
|
|
|
239
258
|
// not this project's daemon facts (#379).
|
|
240
259
|
const rss =
|
|
241
260
|
daemon?.project.kind === "ok" ? rssBytesFromHealthz(daemon.body) : undefined;
|
|
261
|
+
// Three states, never two (#685): `not running` is reserved for a pid
|
|
262
|
+
// that is actually gone, and a timed-out `/healthz` is its own state —
|
|
263
|
+
// a healthy fleet that answers slowly must not read as stopped.
|
|
264
|
+
const state =
|
|
265
|
+
daemon?.project.kind === "unresponsive" ? UNRESPONSIVE_HEALTHZ : "running";
|
|
242
266
|
daemonBlock = [
|
|
243
|
-
|
|
267
|
+
`daemon ${state}`,
|
|
244
268
|
` pid ${layers.daemon.pid}`,
|
|
245
269
|
` port ${layers.daemon.port ?? "?"}`,
|
|
246
270
|
...(rss === undefined ? [] : [` rss ${formatRss(rss)}`]),
|
|
@@ -319,6 +343,31 @@ function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
|
|
|
319
343
|
];
|
|
320
344
|
}
|
|
321
345
|
|
|
346
|
+
/** The reporting row: which categories interrupt the operator's phone, and
|
|
347
|
+
* where everything else goes. It reads the snapshot's effective summary —
|
|
348
|
+
* the legacy preset name is a label, never the source of truth: an explicit
|
|
349
|
+
* policy without `scopePreset` renders truthfully from `interruptOn` and the
|
|
350
|
+
* digest cadence, so a routine outcome under `decisions` explains itself as
|
|
351
|
+
* digest-only instead of reading like a broken Telegram (#633). */
|
|
352
|
+
function formatReportingStatus(s: StatusSnapshot): string[] {
|
|
353
|
+
const r = s.reporting;
|
|
354
|
+
if (r === undefined) return [];
|
|
355
|
+
const interrupt =
|
|
356
|
+
r.interruptOn.length === 0 ? "nothing interrupts" : `${r.interruptOn.join(", ")} interrupt`;
|
|
357
|
+
const label = r.scopePreset === undefined ? "reporting —" : `reporting ${r.scopePreset} —`;
|
|
358
|
+
let consequence: string;
|
|
359
|
+
if (r.interruptOn.includes("material")) {
|
|
360
|
+
consequence = "no outcome waits for the digest";
|
|
361
|
+
} else if (r.digest.cadence === "daily") {
|
|
362
|
+
consequence = "material outcomes wait for the daily digest";
|
|
363
|
+
} else if (r.digest.cadence === "none") {
|
|
364
|
+
consequence = "material outcomes are held; no digest is configured";
|
|
365
|
+
} else {
|
|
366
|
+
consequence = "material outcomes wait for the per-tick digest";
|
|
367
|
+
}
|
|
368
|
+
return [`${label} ${interrupt}; ${consequence}`];
|
|
369
|
+
}
|
|
370
|
+
|
|
322
371
|
/**
|
|
323
372
|
* The shared-daemon visibility row (#545). When the daemon also serves sibling
|
|
324
373
|
* projects, tell the reader how many runs each has live, so they can tell "my
|
|
@@ -342,6 +391,14 @@ function formatProjectBody(
|
|
|
342
391
|
const lines = [
|
|
343
392
|
`project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
|
|
344
393
|
...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]),
|
|
394
|
+
// Structured-exposure half of the drain surface (#484): a plain row for
|
|
395
|
+
// the renderer, so the active drain is visible in `status` before the
|
|
396
|
+
// human-wording work of the later #484 child lands.
|
|
397
|
+
...(s.drain === undefined
|
|
398
|
+
? []
|
|
399
|
+
: [
|
|
400
|
+
`drain active — expires ${new Date(s.drain.expiresAt).toISOString()}, ${s.drain.remainingRuns} run(s) remaining`,
|
|
401
|
+
]),
|
|
345
402
|
`config ${s.configPath}`,
|
|
346
403
|
`state ${s.stateDir}`,
|
|
347
404
|
...(siblingLine === undefined ? [] : [siblingLine]),
|
|
@@ -350,6 +407,7 @@ function formatProjectBody(
|
|
|
350
407
|
...(s.orchestratorDown === undefined
|
|
351
408
|
? []
|
|
352
409
|
: formatOrchestratorDown(s.orchestratorDown, now)),
|
|
410
|
+
...formatReportingStatus(s),
|
|
353
411
|
...formatAvailabilityStatus(s),
|
|
354
412
|
...formatDigestScheduleStatus(s),
|
|
355
413
|
"",
|
|
@@ -405,6 +463,13 @@ function formatProjectBody(
|
|
|
405
463
|
` continuations ${s.caps.maxContinuationsPerIssue}`,
|
|
406
464
|
"",
|
|
407
465
|
...formatReleaseGrants(s.releaseGrants),
|
|
466
|
+
// The effective review policy (#678), read from the same snapshot surface
|
|
467
|
+
// as the grants: the level is what Duty 1 judges findings against, and the
|
|
468
|
+
// ceiling is its hard bound — an orchestrator reading `review-revision N`
|
|
469
|
+
// below has to see both without opening the config.
|
|
470
|
+
"review",
|
|
471
|
+
` strictness ${s.review.strictness}`,
|
|
472
|
+
` max rounds/PR ${s.review.maxRounds}`,
|
|
408
473
|
"",
|
|
409
474
|
formatDispatchSummary(s.dispatch),
|
|
410
475
|
"",
|
|
@@ -419,16 +484,39 @@ function formatProjectBody(
|
|
|
419
484
|
// a failure and from an ordinary continuation, with the round number
|
|
420
485
|
// from the durable revision row (#692). The live pause overlay still
|
|
421
486
|
// wins: an operator pause is the newer fact about the same session.
|
|
487
|
+
const paused = phase === "pausing" || phase === "paused";
|
|
422
488
|
const round = s.reviewRounds?.[r.id];
|
|
423
|
-
const state =
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
489
|
+
const state = paused
|
|
490
|
+
? phase
|
|
491
|
+
: round !== undefined
|
|
492
|
+
? `review-revision ${round}`
|
|
493
|
+
: r.state;
|
|
494
|
+
// Turn rate and cap projection (#730/#767): a stalled run and a fast one
|
|
495
|
+
// used to render identically as a bare turn count. The rate is the
|
|
496
|
+
// lifetime average over the elapsed shown — the snapshot carries no
|
|
497
|
+
// checkpoint from which a recent-interval rate could be derived — so it
|
|
498
|
+
// is labelled `avg`, and `elapsed` is elapsed since claim, never
|
|
499
|
+
// "remaining wall clock": a paused worker banks its budget, which the
|
|
500
|
+
// snapshot cannot know. #767 adds the projection #518 asked for: the
|
|
501
|
+
// same claim-elapsed rate held forward to the wall-clock budget
|
|
502
|
+
// (`caps.workerWallClockMs`, rendered in the caps block) as the turn
|
|
503
|
+
// count the run would reach at that cap — trajectory math over
|
|
504
|
+
// elapsed-so-far, so reading it against the run's `maxTurns` tells
|
|
505
|
+
// which cap fires first. A paused run gets neither number, because its
|
|
506
|
+
// elapsed includes banked pause time and its state already says
|
|
507
|
+
// `paused`.
|
|
508
|
+
const elapsedMs = Math.max(0, now - r.startedAt);
|
|
509
|
+
const progress = paused
|
|
510
|
+
? ""
|
|
511
|
+
: ` ${formatDownDuration(elapsedMs)} elapsed ${(
|
|
512
|
+
(r.turns * 60_000) / Math.max(elapsedMs, 1_000)
|
|
513
|
+
).toFixed(1)} turns/min avg` +
|
|
514
|
+
` projects ${Math.round(
|
|
515
|
+
(r.turns * s.caps.workerWallClockMs) / Math.max(elapsedMs, 1_000),
|
|
516
|
+
)} turns at cap`;
|
|
429
517
|
lines.push(
|
|
430
518
|
` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
|
|
431
|
-
`${r.turns}/${r.maxTurns} turns ${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
519
|
+
`${r.turns}/${r.maxTurns} turns${progress} ${r.spendUsd.toFixed(2)} ${r.branch}` +
|
|
432
520
|
(r.prUrl ? ` ${r.prUrl}` : ""),
|
|
433
521
|
);
|
|
434
522
|
// The orchestrator's Duty 1 reads this command, and a flagged run's
|
|
@@ -441,6 +529,7 @@ function formatProjectBody(
|
|
|
441
529
|
lines.push(...formatBaseHealth(s.baseHealth));
|
|
442
530
|
lines.push(...formatFreezes(s.freezes));
|
|
443
531
|
lines.push(...formatSalvagedRuns(s.salvagedRuns));
|
|
532
|
+
lines.push(...formatQuarantinedRuns(s.quarantinedRuns));
|
|
444
533
|
lines.push(...formatOpenReports(s.openReports));
|
|
445
534
|
lines.push(...formatDigestBacklog(s.digestBacklog));
|
|
446
535
|
if (s.liveWorkers > 0) {
|