omp-conductor 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +12 -8
- package/package.json +1 -1
- package/schema/config.schema.json +40 -1
- package/src/admission.ts +263 -44
- package/src/ask.ts +39 -3
- package/src/availability.ts +27 -1
- package/src/backups.ts +2 -2
- package/src/briefs/orchestrator.md +1 -0
- package/src/briefs/worker.md +38 -19
- package/src/command-help.ts +8 -1
- package/src/command-manifest.ts +5 -2
- package/src/commands/arm.ts +6 -3
- package/src/commands/message.ts +32 -4
- package/src/commands/watch.ts +62 -3
- package/src/config-schema.ts +53 -0
- package/src/config.ts +97 -1
- package/src/daemon.ts +1479 -1483
- package/src/decisions.ts +51 -6
- package/src/depends-on.ts +261 -1
- package/src/diff-flags.ts +350 -0
- package/src/digest-schedule.ts +37 -0
- package/src/doctor.ts +310 -22
- package/src/escalate.ts +560 -57
- package/src/failure-class.ts +71 -15
- package/src/fleet.ts +189 -34
- package/src/gitops.ts +103 -24
- package/src/graph-health.ts +20 -7
- package/src/graph.ts +313 -68
- package/src/lifecycle.ts +43 -7
- package/src/omp.ts +42 -0
- package/src/orchestrator-tick.ts +430 -162
- package/src/release-policy.ts +177 -5
- package/src/routing.ts +11 -3
- package/src/session-host.ts +16 -0
- package/src/settlement.ts +1728 -0
- package/src/setup-host.ts +193 -4
- package/src/setup-install.ts +91 -30
- package/src/setup-wizard.ts +1257 -78
- package/src/setup.ts +153 -6
- package/src/status-render.ts +36 -4
- package/src/store.ts +411 -17
- package/src/tracker/github.ts +607 -12
- package/src/types.ts +331 -5
- package/src/upgrade.ts +50 -19
- package/src/verbs/actions.ts +66 -18
- package/src/verbs/protocol.ts +45 -0
- package/src/verbs/server.ts +270 -13
- package/src/worker.ts +239 -6
- package/src/worktree.ts +115 -8
- package/systemd/omp-conductor-recover.sh +73 -0
- package/systemd/recover-unit-test.sh +61 -0
package/src/types.ts
CHANGED
|
@@ -95,7 +95,8 @@ export interface RepoTarget {
|
|
|
95
95
|
/**
|
|
96
96
|
* Absolute path of the **conductor-owned, index-only clone** of this repo
|
|
97
97
|
* whose code-graph index workers query. Optional: absent means this repo has
|
|
98
|
-
* no graph, and the worker brief says
|
|
98
|
+
* no graph, and the worker brief says so explicitly rather than staying
|
|
99
|
+
* silent — a worker that knows there is no graph stops looking for one.
|
|
99
100
|
*
|
|
100
101
|
* Two things it deliberately is not, and both were paid for:
|
|
101
102
|
*
|
|
@@ -412,8 +413,11 @@ export type BehindBaseAction = (typeof BEHIND_BASE_ACTIONS)[number];
|
|
|
412
413
|
* the entry price for being in this list rather than in POLICY.md.
|
|
413
414
|
*/
|
|
414
415
|
export const RELEASE_REQUIREMENTS = [
|
|
415
|
-
/** Every run
|
|
416
|
+
/** Every active run of the released repo reached `merged`, not merely `pushed-green`. */
|
|
416
417
|
"runs-settled",
|
|
418
|
+
/** Every active run in the project reached `merged` — the suite-wide form of
|
|
419
|
+
* `runs-settled`, for shapes that consume several repos at once. */
|
|
420
|
+
"fleet-runs-settled",
|
|
417
421
|
/** No pull request is still open against the branch being released. */
|
|
418
422
|
"no-open-prs",
|
|
419
423
|
/** Nothing still carries the queue label: the batch is finished, not paused. */
|
|
@@ -581,6 +585,111 @@ export const LABEL_REASONS = [
|
|
|
581
585
|
|
|
582
586
|
export type LabelReason = (typeof LABEL_REASONS)[number];
|
|
583
587
|
|
|
588
|
+
/**
|
|
589
|
+
* Why a green pull request is being returned to its worker for revision.
|
|
590
|
+
* Closed for the reason {@link MERGE_REASONS} is: a revision request is a
|
|
591
|
+
* decision the ledger has to be able to name, and the same request repeated
|
|
592
|
+
* must read the same both times.
|
|
593
|
+
*/
|
|
594
|
+
export const REVIEW_REASONS = [
|
|
595
|
+
/** The orchestrator read the green PR and found issues that block merging. */
|
|
596
|
+
"blocking-findings",
|
|
597
|
+
] as const;
|
|
598
|
+
|
|
599
|
+
export type ReviewReason = (typeof REVIEW_REASONS)[number];
|
|
600
|
+
|
|
601
|
+
/**
|
|
602
|
+
* How a queued review revision ended, written by the daemon side of the
|
|
603
|
+
* transport. `revised` is a revision worker that again reached a verified
|
|
604
|
+
* green head; `pending` is a still-unverified push; `failed` is a revision
|
|
605
|
+
* worker that settled terminally; `skipped` is a queued round the daemon could
|
|
606
|
+
* not dispatch (its run moved, its transcript vanished, or the store already
|
|
607
|
+
* claimed the slot).
|
|
608
|
+
*/
|
|
609
|
+
export const REVIEW_REVISION_OUTCOMES = ["revised", "pending", "failed", "skipped"] as const;
|
|
610
|
+
|
|
611
|
+
export type ReviewRevisionOutcome = (typeof REVIEW_REVISION_OUTCOMES)[number];
|
|
612
|
+
|
|
613
|
+
/**
|
|
614
|
+
* How strictly this project's green PRs are reviewed before they merge
|
|
615
|
+
* (#678). One of three typed levels; each level's bar is documented once in
|
|
616
|
+
* {@link REVIEW_STRICTNESS_CHOICES} (setup.ts), which is the same single text
|
|
617
|
+
* the rendered orchestrator brief carries, so the level a config declares and
|
|
618
|
+
* the threshold a session actually enforces cannot disagree.
|
|
619
|
+
*
|
|
620
|
+
* `low` blocks only correctness, security, data-loss or explicit
|
|
621
|
+
* acceptance-criteria failures. `medium` (the recommended default for a new
|
|
622
|
+
* project) adds material maintainability or reliability defects likely to
|
|
623
|
+
* become incidents within six months. `high` adds concrete quality defects —
|
|
624
|
+
* never subjective style churn or unbounded refactoring.
|
|
625
|
+
*/
|
|
626
|
+
export const REVIEW_STRICTNESS = ["low", "medium", "high"] as const;
|
|
627
|
+
|
|
628
|
+
export type ReviewStrictness = (typeof REVIEW_STRICTNESS)[number];
|
|
629
|
+
|
|
630
|
+
/** The recommended strictness for a project that has never answered. */
|
|
631
|
+
export const DEFAULT_REVIEW_STRICTNESS: ReviewStrictness = "medium";
|
|
632
|
+
|
|
633
|
+
/** The small validated range a project's own review-round ceiling may sit in. */
|
|
634
|
+
export const REVIEW_MAX_ROUNDS_MIN = 1;
|
|
635
|
+
export const REVIEW_MAX_ROUNDS_MAX = 6;
|
|
636
|
+
|
|
637
|
+
/** The typed default ceiling: three corrected heads per PR lifecycle — enough
|
|
638
|
+
* for real correction cycles, and a hard stop against endless polishing. */
|
|
639
|
+
export const DEFAULT_REVIEW_MAX_ROUNDS = 3;
|
|
640
|
+
|
|
641
|
+
/**
|
|
642
|
+
* One project's review policy (#678): the strictness its orchestrator applies
|
|
643
|
+
* when deciding whether a green PR is returned to its worker, and the hard
|
|
644
|
+
* ceiling on how many such rounds one PR lifecycle may consume. The ceiling is
|
|
645
|
+
* enforced by the dispatch side, never left to the session's reading of the
|
|
646
|
+
* brief: at the ceiling `conductor_pr_review` refuses, the orchestrator leaves
|
|
647
|
+
* the PR open, records the unresolved findings and escalates once.
|
|
648
|
+
*/
|
|
649
|
+
export interface ReviewPolicy {
|
|
650
|
+
strictness: ReviewStrictness;
|
|
651
|
+
maxRounds: number;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
/** The documented migration default: what a config written before the key
|
|
655
|
+
* existed loads as, deterministically, until the operator answers setup. */
|
|
656
|
+
export const DEFAULT_REVIEW_POLICY: ReviewPolicy = {
|
|
657
|
+
strictness: DEFAULT_REVIEW_STRICTNESS,
|
|
658
|
+
maxRounds: DEFAULT_REVIEW_MAX_ROUNDS,
|
|
659
|
+
};
|
|
660
|
+
|
|
661
|
+
/**
|
|
662
|
+
* One durable review-revision request (#677): the orchestrator returned a
|
|
663
|
+
* green, run-owned pull request to its worker with blocking findings, and
|
|
664
|
+
* everything the daemon needs to resume the *same* OMP session is recorded
|
|
665
|
+
* here before any worker is woken. The round numbers successive revisions of
|
|
666
|
+
* one run (1, 2, …); the findings text is delivered to the resumed session
|
|
667
|
+
* verbatim.
|
|
668
|
+
*/
|
|
669
|
+
export interface ReviewRevisionRecord {
|
|
670
|
+
id: string;
|
|
671
|
+
project: string;
|
|
672
|
+
/** The run whose pushed-green row this revises. The row is reused, never
|
|
673
|
+
* cloned — no new attempt, and no failed-attempt or continuation charge. */
|
|
674
|
+
runId: string;
|
|
675
|
+
issue: number;
|
|
676
|
+
prUrl: string;
|
|
677
|
+
/** The exact reviewed head; re-verified against the live PR before persisting. */
|
|
678
|
+
headSha: string;
|
|
679
|
+
/** Structured blocking findings, delivered verbatim to the resumed session. */
|
|
680
|
+
findings: string;
|
|
681
|
+
round: number;
|
|
682
|
+
reason: ReviewReason;
|
|
683
|
+
/** The transcript the revision must resume, as the run recorded it — the
|
|
684
|
+
* durable "same OMP session" identity, recorded before any wake. */
|
|
685
|
+
sessionFile?: string;
|
|
686
|
+
requestedAt: number;
|
|
687
|
+
/** Set when the daemon handed the revision to a worker. */
|
|
688
|
+
dispatchedAt?: number;
|
|
689
|
+
settledAt?: number;
|
|
690
|
+
outcome?: ReviewRevisionOutcome;
|
|
691
|
+
}
|
|
692
|
+
|
|
584
693
|
/**
|
|
585
694
|
* A model-supplied justification: one value out of a closed set, plus prose that
|
|
586
695
|
* is written down and read by nothing that decides.
|
|
@@ -612,6 +721,33 @@ export const ORCHESTRATOR_MODES = ["embedded", "external"] as const;
|
|
|
612
721
|
|
|
613
722
|
export type OrchestratorMode = (typeof ORCHESTRATOR_MODES)[number];
|
|
614
723
|
|
|
724
|
+
/**
|
|
725
|
+
* What `arm` accepts as proof that a human just approved arming (conductor
|
|
726
|
+
* #613). Declared as data for the reason {@link BASE_FRESHNESS} is: the schema,
|
|
727
|
+
* the setup wizard and the arm gate must all read one vocabulary, and a new
|
|
728
|
+
* proof must be a compile-time change everywhere at once.
|
|
729
|
+
*
|
|
730
|
+
* `challenge` is the pre-#613 behaviour, byte for byte: a Telegram challenge
|
|
731
|
+
* goes out and the armed marker is written only after the reply appears in the
|
|
732
|
+
* orchestrator session. `claim-only` arms on the shared live-plumbing verdict
|
|
733
|
+
* alone — no Telegram send and no wait — which is what makes an unattended
|
|
734
|
+
* recovery re-arm possible, at the cost spelled out in the wizard question:
|
|
735
|
+
* anything that can invoke the already-privileged `omp-conductor arm` command
|
|
736
|
+
* can start dispatch once the live claim and poller pass.
|
|
737
|
+
*/
|
|
738
|
+
export const ARM_PROOFS = ["challenge", "claim-only"] as const;
|
|
739
|
+
|
|
740
|
+
export type ArmProof = (typeof ARM_PROOFS)[number];
|
|
741
|
+
|
|
742
|
+
/**
|
|
743
|
+
* The proof a config that never answered the question gets: today's
|
|
744
|
+
* authenticated round-trip, so no existing install changes behaviour on
|
|
745
|
+
* upgrade. Read through `resolveArmProof` (config.ts), never off a raw
|
|
746
|
+
* `ProjectConfig` — the loader materialises `arm` complete, but a hand-built
|
|
747
|
+
* config may omit the key.
|
|
748
|
+
*/
|
|
749
|
+
export const DEFAULT_ARM_PROOF: ArmProof = "challenge";
|
|
750
|
+
|
|
615
751
|
/**
|
|
616
752
|
* Everything the dispatcher needs to service one product: where work comes
|
|
617
753
|
* from, where code goes, and what it may spend doing it. Config is per project
|
|
@@ -629,8 +765,11 @@ export interface ProjectConfig {
|
|
|
629
765
|
* orchestrator-tick.ts. */
|
|
630
766
|
groomBelow?: number;
|
|
631
767
|
/** Labels the dispatcher writes back so the tracker alone shows live state
|
|
632
|
-
* to a human who never opens the daemon's logs.
|
|
633
|
-
|
|
768
|
+
* to a human who never opens the daemon's logs. `backlog` is the
|
|
769
|
+
* operator's own park gesture (#507) — not dispatcher-written, but read by
|
|
770
|
+
* eligibility so a parked issue can never be claimed while its queue label
|
|
771
|
+
* is still physically present. */
|
|
772
|
+
stateLabels: { inProgress: string; blocked: string; failed: string; backlog: string };
|
|
634
773
|
/** Maps a `${labelPrefix}${name}` label on an issue to the checkout it
|
|
635
774
|
* belongs in, so routing is declared by humans, not guessed. */
|
|
636
775
|
routing: { labelPrefix: string; repos: Record<string, RepoTarget> };
|
|
@@ -682,6 +821,16 @@ export interface ProjectConfig {
|
|
|
682
821
|
* the session that triages it. See {@link ORCHESTRATOR_MODES}.
|
|
683
822
|
*/
|
|
684
823
|
escalation: { telegramChatId?: string; telegramTopicId?: number; fallbackToIssueComment: boolean; orchestrator: OrchestratorMode };
|
|
824
|
+
/**
|
|
825
|
+
* How `arm` proves a human just approved arming (conductor #613). Optional
|
|
826
|
+
* only for configs written before the key existed and for hand-built test
|
|
827
|
+
* configs: the loader always materialises a complete value, so read it
|
|
828
|
+
* through `resolveArmProof` rather than reaching for `.proof` on a project
|
|
829
|
+
* some test hand-built. Absent resolves to {@link DEFAULT_ARM_PROOF}
|
|
830
|
+
* (`challenge`), preserving today's authenticated round-trip — the one value
|
|
831
|
+
* that changes nothing on upgrade.
|
|
832
|
+
*/
|
|
833
|
+
arm?: { proof: ArmProof };
|
|
685
834
|
/**
|
|
686
835
|
* Who lands green PRs, who cuts releases and who promotes. The daemon never
|
|
687
836
|
* acts on this itself — it words the orchestrator's standing orders and the
|
|
@@ -714,6 +863,16 @@ export interface ProjectConfig {
|
|
|
714
863
|
* reaching for `.merge` on a project some test hand-built.
|
|
715
864
|
*/
|
|
716
865
|
policy?: ProjectPolicy;
|
|
866
|
+
/**
|
|
867
|
+
* How green PRs are reviewed and returned (#678): the strictness level and
|
|
868
|
+
* the hard ceiling on review rounds per PR lifecycle. Optional only for
|
|
869
|
+
* configs written before the key existed — the loader always materialises a
|
|
870
|
+
* complete value, so read it through `resolveReview` rather than reaching
|
|
871
|
+
* for `.review` on a project some test hand-built. A config that never
|
|
872
|
+
* answered migrates to {@link DEFAULT_REVIEW_POLICY} deterministically, and
|
|
873
|
+
* the schema's bounds agree with the runtime ones.
|
|
874
|
+
*/
|
|
875
|
+
review?: ReviewPolicy;
|
|
717
876
|
/**
|
|
718
877
|
* Exact operator-authored exceptions for recovery PRs that no run recorded.
|
|
719
878
|
* Optional and hand-edited: setup preserves entries but never invents them.
|
|
@@ -742,7 +901,11 @@ export interface ProjectConfig {
|
|
|
742
901
|
* the production daemon. With a marker configured, the dispatcher refuses to
|
|
743
902
|
* reattach any preserved branch missing it, holds the issue as `stale-base`
|
|
744
903
|
* (distinct from capacity or dependency holds), and never silently merges
|
|
745
|
-
* base into the user's work.
|
|
904
|
+
* base into the user's work. A branch the marker cannot be verified against
|
|
905
|
+
* at all — a failed mirror fetch, an unresolvable marker, or no probe wired
|
|
906
|
+
* — fails closed too, but as `critical-base-verify-error`: a verification
|
|
907
|
+
* failure, never a claim that the branch is stale, re-admitted automatically
|
|
908
|
+
* once a later probe succeeds. Absent or empty, ordinary stale continuations
|
|
746
909
|
* keep today's behaviour. Optional and hand-edited, like
|
|
747
910
|
* {@link recoveryMerges}.
|
|
748
911
|
*/
|
|
@@ -862,6 +1025,12 @@ export const SETTLEMENT_FLAG_KINDS = [
|
|
|
862
1025
|
"assertions-removed",
|
|
863
1026
|
/** A named timeout in a test file went up. */
|
|
864
1027
|
"test-timeout-raised",
|
|
1028
|
+
/** A command the PR body claims as proof, with no matching command in the
|
|
1029
|
+
* run's transcript — the claim has no attempt behind it at all. */
|
|
1030
|
+
"claimed-proof-missing",
|
|
1031
|
+
/** A claimed proof command was attempted, but the shared-host guard refused
|
|
1032
|
+
* it — the honest blocked shape, told apart from a claim with no attempt. */
|
|
1033
|
+
"claimed-proof-blocked",
|
|
865
1034
|
/** A terminal run recovered an OPEN PR by exact issue, repo and branch identity. */
|
|
866
1035
|
"pr-adopted",
|
|
867
1036
|
/** A workflow on the merge commit's base branch failed after settlement. */
|
|
@@ -962,6 +1131,14 @@ export interface Tracker {
|
|
|
962
1131
|
* read exists to prevent (#517).
|
|
963
1132
|
*/
|
|
964
1133
|
listComments(issue: number): Promise<IssueComment[]>;
|
|
1134
|
+
/**
|
|
1135
|
+
* One issue's body, or undefined when the tracker could not produce it.
|
|
1136
|
+
* The promotion echo (#724) renders the parsed file lane exactly where the
|
|
1137
|
+
* author asserts the claim; the queue label is being added to an issue the
|
|
1138
|
+
* `listReady` filter would have excluded by definition, so the verb needs
|
|
1139
|
+
* this read rather than a list the issue is not on.
|
|
1140
|
+
*/
|
|
1141
|
+
getIssue(issue: number): Promise<ReadyIssue | undefined>;
|
|
965
1142
|
addLabel(issue: number, label: string): Promise<void>;
|
|
966
1143
|
removeLabel(issue: number, label: string): Promise<void>;
|
|
967
1144
|
comment(issue: number, body: string): Promise<void>;
|
|
@@ -1008,6 +1185,12 @@ export interface Tracker {
|
|
|
1008
1185
|
* when either fact cannot be read safely.
|
|
1009
1186
|
*/
|
|
1010
1187
|
issueSnapshot(issue: number): Promise<IssueSnapshot | undefined>;
|
|
1188
|
+
/**
|
|
1189
|
+
* An issue's raw body, or undefined when it cannot be read. Used only by the
|
|
1190
|
+
* dependency-graph cycle pass (#421) to follow a reachable prerequisite's
|
|
1191
|
+
* own `Depends-on:` declarations; never a "no body" claim.
|
|
1192
|
+
*/
|
|
1193
|
+
issueBody(issue: number): Promise<string | undefined>;
|
|
1011
1194
|
/**
|
|
1012
1195
|
* The state of one specific pull request, or undefined when this adapter
|
|
1013
1196
|
* could not tell — a network failure, a deleted PR, a URL it cannot parse.
|
|
@@ -1028,6 +1211,20 @@ export interface Tracker {
|
|
|
1028
1211
|
sha: string,
|
|
1029
1212
|
opts?: { event?: string; branch?: string },
|
|
1030
1213
|
): Promise<WorkflowRun[] | undefined>;
|
|
1214
|
+
/** Full failed-job logs of one workflow run across its attempts, oldest
|
|
1215
|
+
* attempt first, one chunk per failed job — never one aggregate per attempt,
|
|
1216
|
+
* so a mixed failure (setup 429 + sibling compile error) stays visible to
|
|
1217
|
+
* the caller's all-infra guard. `undefined` when the evidence is undecided:
|
|
1218
|
+
* the run register, any attempt's job register, or any failed job's log
|
|
1219
|
+
* could not be read, or the run has more attempts than `maxAttempts` (a
|
|
1220
|
+
* prefix is never presented as the complete attempt history); a no-mutation
|
|
1221
|
+
* refusal the next pass asks again. `[]` when the run is determinately
|
|
1222
|
+
* green across its attempts. Logs are full and untruncated — a truncated
|
|
1223
|
+
* last-400-lines view can discard contrary failure evidence. Read through
|
|
1224
|
+
* the tracker's own guarded runner — call/refusal accounting and the
|
|
1225
|
+
* rate-limit breaker apply exactly as they do to a live tick's reads
|
|
1226
|
+
* (#638). */
|
|
1227
|
+
runFailedAttemptLogs(repo: string, runUrl: string, maxAttempts: number): Promise<string[] | undefined>;
|
|
1031
1228
|
/** The live commit at one repository branch, or undefined when unreadable. */
|
|
1032
1229
|
branchHead(repo: string, branch: string): Promise<string | undefined>;
|
|
1033
1230
|
/** The newest run of this workflow on the base branch before `before`.
|
|
@@ -1060,6 +1257,13 @@ export interface Tracker {
|
|
|
1060
1257
|
* better answer on a later tick.
|
|
1061
1258
|
*/
|
|
1062
1259
|
prDiff(url: string): Promise<PrDiff | undefined>;
|
|
1260
|
+
/**
|
|
1261
|
+
* The pull request's body, or undefined when this adapter could not produce
|
|
1262
|
+
* one — the same advisory contract as {@link Tracker.prDiff}. Its only
|
|
1263
|
+
* consumer is the settlement audit's claimed-proof check (#582): a claim
|
|
1264
|
+
* extracted from a body this port could not read is no claim at all.
|
|
1265
|
+
*/
|
|
1266
|
+
prBody(url: string): Promise<string | undefined>;
|
|
1063
1267
|
/**
|
|
1064
1268
|
* Every check on a PR with the state the tracker reports for it (#132).
|
|
1065
1269
|
*
|
|
@@ -1282,6 +1486,12 @@ export interface RunRecord {
|
|
|
1282
1486
|
autoCompactionCount?: number;
|
|
1283
1487
|
/** omp session transcript, so a human can read what the worker actually did. */
|
|
1284
1488
|
sessionFile?: string;
|
|
1489
|
+
/** The run id of the orphan-clean attempt whose session this row resumes
|
|
1490
|
+
* (#536, #567). Set only when the dispatch-time resume verdict fired;
|
|
1491
|
+
* absent means a fresh dispatch, never "resumed from unknown". Readable
|
|
1492
|
+
* without opening the transcript, so "how often did resume fire and did it
|
|
1493
|
+
* save turns" is a query, not a file walk. */
|
|
1494
|
+
resumedFromRunId?: string;
|
|
1285
1495
|
prUrl?: string;
|
|
1286
1496
|
/** Pull request head the worker observed after its deterministic CI watcher exited. */
|
|
1287
1497
|
headSha?: string;
|
|
@@ -1312,6 +1522,14 @@ export interface RunRecord {
|
|
|
1312
1522
|
/** When an operator accepted the loss or recovered the tree by hand
|
|
1313
1523
|
* (`unblock --force`). Clears the hold without erasing what happened. */
|
|
1314
1524
|
salvageAckAt?: number;
|
|
1525
|
+
/** Why the retained tree is under quarantine (#737): the cleanup pass could
|
|
1526
|
+
* not make its object store sound (`repairAlternates`), so fetching into it
|
|
1527
|
+
* was refused and the tree's commits cannot be verified against any remote.
|
|
1528
|
+
* Present means the tree is quarantined right now; the pass clears it the
|
|
1529
|
+
* moment it reaps the tree or re-verifies the store on a later pass.
|
|
1530
|
+
* Absent means never quarantined, or no longer — never "quarantine not
|
|
1531
|
+
* checked", because every retained-tree pass checks. */
|
|
1532
|
+
quarantineDetail?: string;
|
|
1315
1533
|
startedAt: number;
|
|
1316
1534
|
endedAt?: number;
|
|
1317
1535
|
/** Last failure text, surfaced verbatim in escalations. */
|
|
@@ -1364,6 +1582,10 @@ export type AdmissionHoldReason =
|
|
|
1364
1582
|
| "issue-closed"
|
|
1365
1583
|
| "issue-state-lookup-error"
|
|
1366
1584
|
| "issue-dequeued"
|
|
1585
|
+
/** An operator-parked issue: the park label beat the queue label at claim
|
|
1586
|
+
* time (#734). Unlike `issue-dequeued` it is an operator gesture, not a
|
|
1587
|
+
* queue edit — the queue label may still be physically present. */
|
|
1588
|
+
| "issue-parked"
|
|
1367
1589
|
| "parent-lookup-error"
|
|
1368
1590
|
| "sibling-active"
|
|
1369
1591
|
| "repo-active"
|
|
@@ -1374,8 +1596,14 @@ export type AdmissionHoldReason =
|
|
|
1374
1596
|
| "plan-usage-cap"
|
|
1375
1597
|
| "shutting-down"
|
|
1376
1598
|
| "stale-base"
|
|
1599
|
+
/** A ready issue whose lifecycle state label is residual: its newest run is
|
|
1600
|
+
* terminal (or absent), so nobody owns it — Duty 1 reconciliation work, not
|
|
1601
|
+
* occupied capacity (#611). */
|
|
1602
|
+
| "stale-lifecycle"
|
|
1603
|
+
| "critical-base-verify-error"
|
|
1377
1604
|
| "file-lane"
|
|
1378
1605
|
| "depends-on"
|
|
1606
|
+
| "dependency-cycle"
|
|
1379
1607
|
| "unroutable:no-repo-label"
|
|
1380
1608
|
| "unroutable:multiple-repo-labels"
|
|
1381
1609
|
| "unroutable:unknown-repo";
|
|
@@ -1425,6 +1653,7 @@ export type FrictionAdmissionReason =
|
|
|
1425
1653
|
| "issue-state-lookup-error"
|
|
1426
1654
|
| "open-pr-lookup-error"
|
|
1427
1655
|
| "stale-base"
|
|
1656
|
+
| "critical-base-verify-error"
|
|
1428
1657
|
| "unroutable:no-repo-label"
|
|
1429
1658
|
| "unroutable:multiple-repo-labels"
|
|
1430
1659
|
| "unroutable:unknown-repo";
|
|
@@ -1744,6 +1973,14 @@ export interface LabelOp {
|
|
|
1744
1973
|
* carry, so the operator reconciles by hand, which is the only safe move once
|
|
1745
1974
|
* the two disagree.
|
|
1746
1975
|
*/
|
|
1976
|
+
|
|
1977
|
+
/**
|
|
1978
|
+
* A settled `ci-deterministic` row offered to the historical infrastructure
|
|
1979
|
+
* reconciliation, with the internal `rowid` that breaks start-time ties so the
|
|
1980
|
+
* persisted review cursor can resume exactly below the last decided row (#638).
|
|
1981
|
+
*/
|
|
1982
|
+
export type HistoricalInfraCandidate = RunRecord & { rowid: number };
|
|
1983
|
+
|
|
1747
1984
|
export interface Store {
|
|
1748
1985
|
/** A claimed run atomically consumes and applies its issue's pending turn override. */
|
|
1749
1986
|
createRun(r: Omit<RunRecord, "id">): RunRecord;
|
|
@@ -1764,6 +2001,35 @@ export interface Store {
|
|
|
1764
2001
|
* per-issue view — a requeued continuation must never hide the PR its
|
|
1765
2002
|
* predecessor opened. Merged rows keep the same recent-history bound. */
|
|
1766
2003
|
runsForProjectPr(project: string, prUrl: string, mergedSinceEpochMs: number): RunRecord[];
|
|
2004
|
+
/** Persist one review-revision request (#677), durably, before anything is
|
|
2005
|
+
* woken. Returns `undefined` when a revision for the same run is already
|
|
2006
|
+
* pending undelivered — the atomic half of the duplicate in-flight guard
|
|
2007
|
+
* (the other half is the run row's state once the revision is dispatched). */
|
|
2008
|
+
createReviewRevision(draft: Omit<ReviewRevisionRecord, "id">): ReviewRevisionRecord | undefined;
|
|
2009
|
+
/** Review revisions not yet handed to a worker, oldest first — what the
|
|
2010
|
+
* daemon's dispatch pass wakes on its next tick. */
|
|
2011
|
+
pendingReviewRevisions(project: string): ReviewRevisionRecord[];
|
|
2012
|
+
/** Round number of the newest revision ever requested for a run (0 = never). */
|
|
2013
|
+
latestReviewRound(project: string, runId: string): number;
|
|
2014
|
+
/** Record that a revision was handed to a worker. */
|
|
2015
|
+
markReviewRevisionDispatched(id: string, at: number): void;
|
|
2016
|
+
/** Record how a revision ended. */
|
|
2017
|
+
settleReviewRevision(id: string, outcome: ReviewRevisionOutcome, at: number): void;
|
|
2018
|
+
/** Every revision row not yet settled — queued (`dispatchedAt` unset) and
|
|
2019
|
+
* dispatched-but-unfinished — oldest first. This is what a restart reads to
|
|
2020
|
+
* find review rounds a previous daemon left in flight (#692): the row is
|
|
2021
|
+
* the durable home of the findings, reviewed head, round and target
|
|
2022
|
+
* session, so re-queuing the row is how the exact session resumes. */
|
|
2023
|
+
unsettledReviewRevisions(project: string): ReviewRevisionRecord[];
|
|
2024
|
+
/** Return a dispatched revision to the pending set (clear its dispatched
|
|
2025
|
+
* marker) so the next dispatch pass wakes it again — the reopen half of
|
|
2026
|
+
* restart recovery, paired with restoring its run to `pushed-green`. */
|
|
2027
|
+
requeueReviewRevision(id: string): void;
|
|
2028
|
+
/** Atomically take a settled green run out of reviewable state:
|
|
2029
|
+
* `pushed-green` → `running`, and only from `pushed-green`. False when the
|
|
2030
|
+
* row already moved (a prior revision was dispatched, the PR settled), so
|
|
2031
|
+
* two concurrent dispatchers cannot both wake one run. */
|
|
2032
|
+
claimRunForReview(runId: string): boolean;
|
|
1767
2033
|
/** Merged rows whose post-merge workflow verdict is still pending, oldest first. */
|
|
1768
2034
|
runsNeedingBaseCheck(project: string, limit?: number): RunRecord[];
|
|
1769
2035
|
/** Replace the current live-head health row for one routed repository. */
|
|
@@ -1808,6 +2074,11 @@ export interface Store {
|
|
|
1808
2074
|
* can name every WIP tip a re-claim would build on and every tree that is
|
|
1809
2075
|
* still the only copy. */
|
|
1810
2076
|
salvagedRuns(project: string): RunRecord[];
|
|
2077
|
+
/** Newest attempt per issue whose retained tree is under quarantine: the
|
|
2078
|
+
* object store could not be made sound, so the tree's commits cannot be
|
|
2079
|
+
* verified against any remote and the daemon refused to fetch into it
|
|
2080
|
+
* (#737). Newest-attempt-only for the same reason the board is. */
|
|
2081
|
+
quarantinedRuns(project: string): RunRecord[];
|
|
1811
2082
|
/** Total run segments, used only for the monotonically increasing run number. */
|
|
1812
2083
|
attemptsFor(project: string, issue: number): number;
|
|
1813
2084
|
/** Terminal implementation failures that consume `maxAttemptsPerIssue`. */
|
|
@@ -2014,6 +2285,43 @@ export interface Store {
|
|
|
2014
2285
|
* than spending one tick on all of it.
|
|
2015
2286
|
*/
|
|
2016
2287
|
runsNeedingClassification(project: string, limit?: number): RunRecord[];
|
|
2288
|
+
/**
|
|
2289
|
+
* Settled rows currently reading `ci-deterministic` — the only rows the
|
|
2290
|
+
* historical infrastructure reconciliation may repair (#638): the verdict
|
|
2291
|
+
* the forward classifier no longer gives, and nothing else. Newest first
|
|
2292
|
+
* and bounded, because each candidate costs tracker calls to re-fetch the
|
|
2293
|
+
* evidence its log carries.
|
|
2294
|
+
*
|
|
2295
|
+
* `after` is the persisted review cursor (the last row a prior pass decided):
|
|
2296
|
+
* candidates are then only rows strictly *older* than it, so a bounded pass
|
|
2297
|
+
* progresses through a long history instead of rescanning the newest
|
|
2298
|
+
* non-matches forever. `rowid` on each candidate breaks the start-time tie,
|
|
2299
|
+
* giving the cursor an exact resume boundary. */
|
|
2300
|
+
historicalInfraCandidates(
|
|
2301
|
+
project: string,
|
|
2302
|
+
limit?: number,
|
|
2303
|
+
after?: { startedAt: number; rowid: number },
|
|
2304
|
+
): HistoricalInfraCandidate[];
|
|
2305
|
+
/** The persisted historical-review cursor for a project, or undefined before
|
|
2306
|
+
* any pass decided a row. The `classifierVersion` stamp tells the pass
|
|
2307
|
+
* whether the cursor is still current against the signature list. */
|
|
2308
|
+
historicalInfraCursor(
|
|
2309
|
+
project: string,
|
|
2310
|
+
): { startedAt: number; rowid: number; classifierVersion: string } | undefined;
|
|
2311
|
+
/** Persist (or move) the historical-review cursor after a pass decided rows. */
|
|
2312
|
+
setHistoricalInfraCursor(
|
|
2313
|
+
project: string,
|
|
2314
|
+
startedAt: number,
|
|
2315
|
+
rowid: number,
|
|
2316
|
+
classifierVersion: string,
|
|
2317
|
+
): void;
|
|
2318
|
+
/** Reclassify one settled `ci-deterministic` row to `ci-infra` (budget
|
|
2319
|
+
* exempt) after its re-fetched check log matched the closed infrastructure
|
|
2320
|
+
* signature. Changes `failureClass` only — `recoveryAction`/`recoveredAt`
|
|
2321
|
+
* stay byte-for-byte unchanged, so a months-old run is never re-animated
|
|
2322
|
+
* into recovery. False when the row was no longer `ci-deterministic`: a
|
|
2323
|
+
* no-op, exactly what a second repair pass must be. */
|
|
2324
|
+
reclassifyInfra(id: string): boolean;
|
|
2017
2325
|
/** Unrecovered rows per class, for `status`. Empty when nothing is carrying one. */
|
|
2018
2326
|
failureClassCounts(project: string): { cls: FailureClass; n: number }[];
|
|
2019
2327
|
/** Rows whose recovery ran at or after `since`, newest first — the tick's
|
|
@@ -2275,6 +2583,11 @@ export const VERB_NAMES = [
|
|
|
2275
2583
|
* install shape gates it like a release shape; execution detaches from this
|
|
2276
2584
|
* session and the daemon, and the first tick after the restart verifies. */
|
|
2277
2585
|
"conductor_install",
|
|
2586
|
+
/** Return one green, run-owned pull request to its worker with blocking
|
|
2587
|
+
* findings: durably record the revision, and the daemon resumes the SAME
|
|
2588
|
+
* OMP session on the existing branch and PR. No close, no reopen, no
|
|
2589
|
+
* redispatch. */
|
|
2590
|
+
"conductor_pr_review",
|
|
2278
2591
|
/** The one read verb. It answers with the merge gate's own verdict, so
|
|
2279
2592
|
* "pushed-green" is the dispatcher's reading of the PR rather than a claim the
|
|
2280
2593
|
* worker makes about itself from whatever it happened to run. */
|
|
@@ -2357,6 +2670,19 @@ export const VERB_REFUSALS = [
|
|
|
2357
2670
|
"checks-not-green",
|
|
2358
2671
|
/** Another merge is in flight for this project. */
|
|
2359
2672
|
"merge-in-flight",
|
|
2673
|
+
/**
|
|
2674
|
+
* The run backing a PR is not in a revisable settled-green state: a review
|
|
2675
|
+
* revision is already in flight (queued or dispatched), a worker is still
|
|
2676
|
+
* running on it, or the run settled otherwise. The detail names the state.
|
|
2677
|
+
*/
|
|
2678
|
+
"review-in-flight",
|
|
2679
|
+
/**
|
|
2680
|
+
* The PR has already been through the configured review-round ceiling
|
|
2681
|
+
* (#678): the policy's hard bound against endless polishing. Refused rather
|
|
2682
|
+
* than recorded — the orchestrator leaves the PR open, records the
|
|
2683
|
+
* unresolved findings and escalates once instead of spending a worker round.
|
|
2684
|
+
*/
|
|
2685
|
+
"review-round-ceiling",
|
|
2360
2686
|
/**
|
|
2361
2687
|
* The routed repository's base branch is frozen because a watched merge (or
|
|
2362
2688
|
* live base observation) turned it red — a base-red-freeze. Further merges to
|
package/src/upgrade.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import { backupTimestamp, copyToUniqueBackup } from "./backups.ts";
|
|
3
3
|
import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
|
|
4
|
-
import { pauseInstance, setPaused, statusSnapshot } from "./daemon.ts";
|
|
4
|
+
import { pauseInstance, setPaused, statusSnapshot, type AdmissionAckRecord } from "./daemon.ts";
|
|
5
5
|
import {
|
|
6
6
|
DEFAULT_HERDR_SESSION,
|
|
7
7
|
fleetLayers,
|
|
@@ -335,6 +335,22 @@ export interface UpgradeScope {
|
|
|
335
335
|
pauseKey: string | undefined;
|
|
336
336
|
}
|
|
337
337
|
|
|
338
|
+
/**
|
|
339
|
+
* The subset of {@link UpgradeDeps} the pause/drain fence needs. Factored out
|
|
340
|
+
* so a caller that only wants the fence — `setup host` — can provide just
|
|
341
|
+
* these members instead of a full install-and-verify deps object. Every
|
|
342
|
+
* {@link UpgradeDeps} is a `DrainDeps`.
|
|
343
|
+
*/
|
|
344
|
+
/**
|
|
345
|
+
* The daemon a transaction targets: running or not, its project when it
|
|
346
|
+
* recorded one, and a `generation` identifying the exact instance (#377).
|
|
347
|
+
*/
|
|
348
|
+
export interface DaemonIdentity {
|
|
349
|
+
running: boolean;
|
|
350
|
+
project?: string;
|
|
351
|
+
generation?: string;
|
|
352
|
+
}
|
|
353
|
+
|
|
338
354
|
/**
|
|
339
355
|
* The subset of {@link UpgradeDeps} the pause/drain fence needs. Factored out
|
|
340
356
|
* so a caller that only wants the fence — `setup host` — can provide just
|
|
@@ -352,11 +368,20 @@ export interface DrainDeps {
|
|
|
352
368
|
* The daemon the transaction targets: running or not, its project when it
|
|
353
369
|
* recorded one, and a `generation` identifying the exact instance (#377).
|
|
354
370
|
*/
|
|
355
|
-
daemonIdentity():
|
|
371
|
+
daemonIdentity(): DaemonIdentity;
|
|
356
372
|
/** The active pause sentinel as an instance, when one is readable (#377). */
|
|
357
373
|
pauseState(project?: string): { source: string; reason?: string; since: number } | undefined;
|
|
358
374
|
/** Write or clear the durable pause sentinel for the scope's project. */
|
|
359
375
|
setPaused(value: boolean, project?: string): void;
|
|
376
|
+
/**
|
|
377
|
+
* The daemon's durable admission acknowledgement for the current fence, when
|
|
378
|
+
* one is readable: the daemon attests it has observed the pause sentinel at
|
|
379
|
+
* an admission boundary and will claim nothing while it stands (#651 review
|
|
380
|
+
* #3). Absent when no daemon is running, when the file cannot be read, or
|
|
381
|
+
* when the daemon has not (yet) observed the fence. The setup barrier
|
|
382
|
+
* requires this before any mutation when a daemon is running.
|
|
383
|
+
*/
|
|
384
|
+
admissionAck?(project?: string): AdmissionAckRecord | undefined;
|
|
360
385
|
sleep(ms: number): Promise<void>;
|
|
361
386
|
log(message: string): void;
|
|
362
387
|
}
|
|
@@ -455,31 +480,37 @@ export async function waitForDrain(
|
|
|
455
480
|
* live state against this snapshot, so the destructive restart call only ever
|
|
456
481
|
* acts on the world the request began in (#377).
|
|
457
482
|
*/
|
|
458
|
-
interface RestartBegun {
|
|
483
|
+
export interface RestartBegun {
|
|
459
484
|
/** The durable pause sentinel this request relies on, proved readable as an instance. */
|
|
460
485
|
pauseToken: { source: string; reason?: string; since: number };
|
|
461
486
|
/** The daemon instance the restart targets, with its generation identity. */
|
|
462
|
-
daemon:
|
|
487
|
+
daemon: DaemonIdentity;
|
|
463
488
|
}
|
|
464
489
|
|
|
465
490
|
/**
|
|
466
|
-
* Why a
|
|
467
|
-
* the pause and the daemon generation it began with. The pause
|
|
468
|
-
* *
|
|
469
|
-
* front, so a token that cannot be re-read (sentinel removed,
|
|
470
|
-
* re-created — even under the same source and reason) means
|
|
471
|
-
* pause no longer exists and it must cancel. The
|
|
472
|
-
* later-daemon fence: a daemon created after
|
|
473
|
-
* `restart --now`, a crash-restart, or an operator
|
|
474
|
-
* stopped by it.
|
|
491
|
+
* Why a pause-and-drain transaction can no longer act, or `undefined` when it
|
|
492
|
+
* still owns the pause and the daemon generation it began with. The pause
|
|
493
|
+
* check is the *transaction-owned sentinel instance* — the caller proves
|
|
494
|
+
* ownership up front, so a token that cannot be re-read (sentinel removed,
|
|
495
|
+
* unreadable, or re-created — even under the same source and reason) means
|
|
496
|
+
* this request's own pause no longer exists and it must cancel. The
|
|
497
|
+
* generation check is the later-daemon fence: a daemon created after the
|
|
498
|
+
* transaction began — by `restart --now`, a crash-restart, or an operator
|
|
499
|
+
* start — must never be stopped by it.
|
|
500
|
+
*
|
|
501
|
+
* Exported so the setup apply's acknowledged quiescence barrier runs the same
|
|
502
|
+
* fence instead of inventing a second counter-only one (#618): it freezes
|
|
503
|
+
* admission host-wide through {@link pauseAndDrain}'s machinery, and this
|
|
504
|
+
* check is what proves the freeze is still the one it acknowledged.
|
|
475
505
|
*/
|
|
476
|
-
function restartFenceProblem(
|
|
506
|
+
export function restartFenceProblem(
|
|
477
507
|
deps: DrainDeps,
|
|
478
508
|
scope: UpgradeScope,
|
|
479
509
|
begun: RestartBegun,
|
|
510
|
+
actor: string,
|
|
480
511
|
): string | undefined {
|
|
481
512
|
if (!deps.layers(scope.pauseKey).paused) {
|
|
482
|
-
return
|
|
513
|
+
return `${actor} cancelled: dispatch was resumed while the drain was in progress — nothing was restarted`;
|
|
483
514
|
}
|
|
484
515
|
const owned = deps.pauseState(scope.pauseKey);
|
|
485
516
|
if (
|
|
@@ -488,11 +519,11 @@ function restartFenceProblem(
|
|
|
488
519
|
owned.source !== begun.pauseToken.source ||
|
|
489
520
|
owned.reason !== begun.pauseToken.reason
|
|
490
521
|
) {
|
|
491
|
-
return
|
|
522
|
+
return `${actor} cancelled: the ${actor}-owned pause was lifted and replaced while the drain was in progress — nothing was restarted`;
|
|
492
523
|
}
|
|
493
524
|
const now = deps.daemonIdentity();
|
|
494
525
|
if (now.running !== begun.daemon.running || (begun.daemon.running && now.generation !== begun.daemon.generation)) {
|
|
495
|
-
return
|
|
526
|
+
return `${actor} cancelled: the daemon instance was replaced while the drain was in progress — the newer daemon was left running`;
|
|
496
527
|
}
|
|
497
528
|
return undefined;
|
|
498
529
|
}
|
|
@@ -504,7 +535,7 @@ function restartFenceProblem(
|
|
|
504
535
|
* {@link pauseSourceToken}; a verb with a space) fails the suite instead of
|
|
505
536
|
* shipping an unprovable pause (#552).
|
|
506
537
|
*/
|
|
507
|
-
export const DRAIN_VERBS = ["upgrade", "restart", "setup host"] as const;
|
|
538
|
+
export const DRAIN_VERBS = ["upgrade", "restart", "setup host", "setup"] as const;
|
|
508
539
|
export type DrainVerb = (typeof DRAIN_VERBS)[number];
|
|
509
540
|
|
|
510
541
|
/**
|
|
@@ -556,7 +587,7 @@ export async function pauseAndDrain(
|
|
|
556
587
|
pauseToken,
|
|
557
588
|
daemon: deps.daemonIdentity(),
|
|
558
589
|
};
|
|
559
|
-
const stale = () => restartFenceProblem(deps, scope, begun);
|
|
590
|
+
const stale = () => restartFenceProblem(deps, scope, begun, verb);
|
|
560
591
|
await waitForDrain(
|
|
561
592
|
deps,
|
|
562
593
|
scope,
|