omp-conductor 0.16.2 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -4
- package/REFERENCE.md +18 -12
- package/package.json +2 -1
- package/schema/config.schema.json +16 -0
- package/src/admission.ts +159 -43
- package/src/availability.ts +27 -1
- package/src/briefs/worker.md +2 -0
- package/src/clack-ui.ts +83 -0
- package/src/command-manifest.ts +16 -7
- package/src/commands/arm.ts +11 -3
- package/src/commands/decision.ts +17 -7
- package/src/commands/doctor.ts +18 -1
- package/src/commands/hold.ts +9 -7
- package/src/commands/ledger.ts +25 -4
- package/src/commands/message.ts +32 -4
- package/src/commands/setup.ts +61 -10
- package/src/commands/stats.ts +9 -5
- package/src/commands/status.ts +32 -5
- package/src/commands/tail.ts +13 -1
- package/src/commands/watch.ts +16 -7
- package/src/config-schema.ts +20 -0
- package/src/config.ts +37 -0
- package/src/daemon.ts +1240 -18
- package/src/doctor.ts +310 -22
- package/src/escalate.ts +560 -57
- package/src/failure-class.ts +56 -13
- package/src/fleet.ts +224 -47
- package/src/gitops.ts +103 -24
- package/src/lifecycle.ts +7 -2
- package/src/orchestrator-tick.ts +372 -157
- package/src/privileged.ts +3 -0
- package/src/release-policy.ts +177 -5
- package/src/setup-answers.ts +135 -0
- package/src/setup-host.ts +193 -4
- package/src/setup-install.ts +2 -0
- package/src/setup-probe.ts +1 -0
- package/src/setup-wizard.ts +1296 -101
- package/src/setup.ts +60 -3
- package/src/status-render.ts +11 -1
- package/src/store.ts +333 -12
- package/src/tracker/github.ts +562 -13
- package/src/types.ts +204 -2
- package/src/ui/progress.ts +32 -0
- package/src/ui/style.ts +11 -0
- package/src/upgrade.ts +50 -19
- package/src/verbs/actions.ts +66 -18
- package/src/verbs/protocol.ts +45 -0
- package/src/verbs/server.ts +212 -11
- package/src/wizard-ui.ts +14 -5
- package/src/worker.ts +26 -0
- package/systemd/omp-conductor-recover.sh +73 -0
- package/systemd/recover-unit-test.sh +61 -0
package/src/types.ts
CHANGED
|
@@ -412,8 +412,11 @@ export type BehindBaseAction = (typeof BEHIND_BASE_ACTIONS)[number];
|
|
|
412
412
|
* the entry price for being in this list rather than in POLICY.md.
|
|
413
413
|
*/
|
|
414
414
|
export const RELEASE_REQUIREMENTS = [
|
|
415
|
-
/** Every run
|
|
415
|
+
/** Every active run of the released repo reached `merged`, not merely `pushed-green`. */
|
|
416
416
|
"runs-settled",
|
|
417
|
+
/** Every active run in the project reached `merged` — the suite-wide form of
|
|
418
|
+
* `runs-settled`, for shapes that consume several repos at once. */
|
|
419
|
+
"fleet-runs-settled",
|
|
417
420
|
/** No pull request is still open against the branch being released. */
|
|
418
421
|
"no-open-prs",
|
|
419
422
|
/** Nothing still carries the queue label: the batch is finished, not paused. */
|
|
@@ -581,6 +584,63 @@ export const LABEL_REASONS = [
|
|
|
581
584
|
|
|
582
585
|
export type LabelReason = (typeof LABEL_REASONS)[number];
|
|
583
586
|
|
|
587
|
+
/**
|
|
588
|
+
* Why a green pull request is being returned to its worker for revision.
|
|
589
|
+
* Closed for the reason {@link MERGE_REASONS} is: a revision request is a
|
|
590
|
+
* decision the ledger has to be able to name, and the same request repeated
|
|
591
|
+
* must read the same both times.
|
|
592
|
+
*/
|
|
593
|
+
export const REVIEW_REASONS = [
|
|
594
|
+
/** The orchestrator read the green PR and found issues that block merging. */
|
|
595
|
+
"blocking-findings",
|
|
596
|
+
] as const;
|
|
597
|
+
|
|
598
|
+
export type ReviewReason = (typeof REVIEW_REASONS)[number];
|
|
599
|
+
|
|
600
|
+
/**
|
|
601
|
+
* How a queued review revision ended, written by the daemon side of the
|
|
602
|
+
* transport. `revised` is a revision worker that again reached a verified
|
|
603
|
+
* green head; `pending` is a still-unverified push; `failed` is a revision
|
|
604
|
+
* worker that settled terminally; `skipped` is a queued round the daemon could
|
|
605
|
+
* not dispatch (its run moved, its transcript vanished, or the store already
|
|
606
|
+
* claimed the slot).
|
|
607
|
+
*/
|
|
608
|
+
export const REVIEW_REVISION_OUTCOMES = ["revised", "pending", "failed", "skipped"] as const;
|
|
609
|
+
|
|
610
|
+
export type ReviewRevisionOutcome = (typeof REVIEW_REVISION_OUTCOMES)[number];
|
|
611
|
+
|
|
612
|
+
/**
|
|
613
|
+
* One durable review-revision request (#677): the orchestrator returned a
|
|
614
|
+
* green, run-owned pull request to its worker with blocking findings, and
|
|
615
|
+
* everything the daemon needs to resume the *same* OMP session is recorded
|
|
616
|
+
* here before any worker is woken. The round numbers successive revisions of
|
|
617
|
+
* one run (1, 2, …); the findings text is delivered to the resumed session
|
|
618
|
+
* verbatim.
|
|
619
|
+
*/
|
|
620
|
+
export interface ReviewRevisionRecord {
|
|
621
|
+
id: string;
|
|
622
|
+
project: string;
|
|
623
|
+
/** The run whose pushed-green row this revises. The row is reused, never
|
|
624
|
+
* cloned — no new attempt, and no failed-attempt or continuation charge. */
|
|
625
|
+
runId: string;
|
|
626
|
+
issue: number;
|
|
627
|
+
prUrl: string;
|
|
628
|
+
/** The exact reviewed head; re-verified against the live PR before persisting. */
|
|
629
|
+
headSha: string;
|
|
630
|
+
/** Structured blocking findings, delivered verbatim to the resumed session. */
|
|
631
|
+
findings: string;
|
|
632
|
+
round: number;
|
|
633
|
+
reason: ReviewReason;
|
|
634
|
+
/** The transcript the revision must resume, as the run recorded it — the
|
|
635
|
+
* durable "same OMP session" identity, recorded before any wake. */
|
|
636
|
+
sessionFile?: string;
|
|
637
|
+
requestedAt: number;
|
|
638
|
+
/** Set when the daemon handed the revision to a worker. */
|
|
639
|
+
dispatchedAt?: number;
|
|
640
|
+
settledAt?: number;
|
|
641
|
+
outcome?: ReviewRevisionOutcome;
|
|
642
|
+
}
|
|
643
|
+
|
|
584
644
|
/**
|
|
585
645
|
* A model-supplied justification: one value out of a closed set, plus prose that
|
|
586
646
|
* is written down and read by nothing that decides.
|
|
@@ -612,6 +672,33 @@ export const ORCHESTRATOR_MODES = ["embedded", "external"] as const;
|
|
|
612
672
|
|
|
613
673
|
export type OrchestratorMode = (typeof ORCHESTRATOR_MODES)[number];
|
|
614
674
|
|
|
675
|
+
/**
|
|
676
|
+
* What `arm` accepts as proof that a human just approved arming (conductor
|
|
677
|
+
* #613). Declared as data for the reason {@link BASE_FRESHNESS} is: the schema,
|
|
678
|
+
* the setup wizard and the arm gate must all read one vocabulary, and a new
|
|
679
|
+
* proof must be a compile-time change everywhere at once.
|
|
680
|
+
*
|
|
681
|
+
* `challenge` is the pre-#613 behaviour, byte for byte: a Telegram challenge
|
|
682
|
+
* goes out and the armed marker is written only after the reply appears in the
|
|
683
|
+
* orchestrator session. `claim-only` arms on the shared live-plumbing verdict
|
|
684
|
+
* alone — no Telegram send and no wait — which is what makes an unattended
|
|
685
|
+
* recovery re-arm possible, at the cost spelled out in the wizard question:
|
|
686
|
+
* anything that can invoke the already-privileged `omp-conductor arm` command
|
|
687
|
+
* can start dispatch once the live claim and poller pass.
|
|
688
|
+
*/
|
|
689
|
+
export const ARM_PROOFS = ["challenge", "claim-only"] as const;
|
|
690
|
+
|
|
691
|
+
export type ArmProof = (typeof ARM_PROOFS)[number];
|
|
692
|
+
|
|
693
|
+
/**
|
|
694
|
+
* The proof a config that never answered the question gets: today's
|
|
695
|
+
* authenticated round-trip, so no existing install changes behaviour on
|
|
696
|
+
* upgrade. Read through `resolveArmProof` (config.ts), never off a raw
|
|
697
|
+
* `ProjectConfig` — the loader materialises `arm` complete, but a hand-built
|
|
698
|
+
* config may omit the key.
|
|
699
|
+
*/
|
|
700
|
+
export const DEFAULT_ARM_PROOF: ArmProof = "challenge";
|
|
701
|
+
|
|
615
702
|
/**
|
|
616
703
|
* Everything the dispatcher needs to service one product: where work comes
|
|
617
704
|
* from, where code goes, and what it may spend doing it. Config is per project
|
|
@@ -682,6 +769,16 @@ export interface ProjectConfig {
|
|
|
682
769
|
* the session that triages it. See {@link ORCHESTRATOR_MODES}.
|
|
683
770
|
*/
|
|
684
771
|
escalation: { telegramChatId?: string; telegramTopicId?: number; fallbackToIssueComment: boolean; orchestrator: OrchestratorMode };
|
|
772
|
+
/**
|
|
773
|
+
* How `arm` proves a human just approved arming (conductor #613). Optional
|
|
774
|
+
* only for configs written before the key existed and for hand-built test
|
|
775
|
+
* configs: the loader always materialises a complete value, so read it
|
|
776
|
+
* through `resolveArmProof` rather than reaching for `.proof` on a project
|
|
777
|
+
* some test hand-built. Absent resolves to {@link DEFAULT_ARM_PROOF}
|
|
778
|
+
* (`challenge`), preserving today's authenticated round-trip — the one value
|
|
779
|
+
* that changes nothing on upgrade.
|
|
780
|
+
*/
|
|
781
|
+
arm?: { proof: ArmProof };
|
|
685
782
|
/**
|
|
686
783
|
* Who lands green PRs, who cuts releases and who promotes. The daemon never
|
|
687
784
|
* acts on this itself — it words the orchestrator's standing orders and the
|
|
@@ -742,7 +839,11 @@ export interface ProjectConfig {
|
|
|
742
839
|
* the production daemon. With a marker configured, the dispatcher refuses to
|
|
743
840
|
* reattach any preserved branch missing it, holds the issue as `stale-base`
|
|
744
841
|
* (distinct from capacity or dependency holds), and never silently merges
|
|
745
|
-
* base into the user's work.
|
|
842
|
+
* base into the user's work. A branch the marker cannot be verified against
|
|
843
|
+
* at all — a failed mirror fetch, an unresolvable marker, or no probe wired
|
|
844
|
+
* — fails closed too, but as `critical-base-verify-error`: a verification
|
|
845
|
+
* failure, never a claim that the branch is stale, re-admitted automatically
|
|
846
|
+
* once a later probe succeeds. Absent or empty, ordinary stale continuations
|
|
746
847
|
* keep today's behaviour. Optional and hand-edited, like
|
|
747
848
|
* {@link recoveryMerges}.
|
|
748
849
|
*/
|
|
@@ -1028,6 +1129,20 @@ export interface Tracker {
|
|
|
1028
1129
|
sha: string,
|
|
1029
1130
|
opts?: { event?: string; branch?: string },
|
|
1030
1131
|
): Promise<WorkflowRun[] | undefined>;
|
|
1132
|
+
/** Full failed-job logs of one workflow run across its attempts, oldest
|
|
1133
|
+
* attempt first, one chunk per failed job — never one aggregate per attempt,
|
|
1134
|
+
* so a mixed failure (setup 429 + sibling compile error) stays visible to
|
|
1135
|
+
* the caller's all-infra guard. `undefined` when the evidence is undecided:
|
|
1136
|
+
* the run register, any attempt's job register, or any failed job's log
|
|
1137
|
+
* could not be read, or the run has more attempts than `maxAttempts` (a
|
|
1138
|
+
* prefix is never presented as the complete attempt history); a no-mutation
|
|
1139
|
+
* refusal the next pass asks again. `[]` when the run is determinately
|
|
1140
|
+
* green across its attempts. Logs are full and untruncated — a truncated
|
|
1141
|
+
* last-400-lines view can discard contrary failure evidence. Read through
|
|
1142
|
+
* the tracker's own guarded runner — call/refusal accounting and the
|
|
1143
|
+
* rate-limit breaker apply exactly as they do to a live tick's reads
|
|
1144
|
+
* (#638). */
|
|
1145
|
+
runFailedAttemptLogs(repo: string, runUrl: string, maxAttempts: number): Promise<string[] | undefined>;
|
|
1031
1146
|
/** The live commit at one repository branch, or undefined when unreadable. */
|
|
1032
1147
|
branchHead(repo: string, branch: string): Promise<string | undefined>;
|
|
1033
1148
|
/** The newest run of this workflow on the base branch before `before`.
|
|
@@ -1374,6 +1489,7 @@ export type AdmissionHoldReason =
|
|
|
1374
1489
|
| "plan-usage-cap"
|
|
1375
1490
|
| "shutting-down"
|
|
1376
1491
|
| "stale-base"
|
|
1492
|
+
| "critical-base-verify-error"
|
|
1377
1493
|
| "file-lane"
|
|
1378
1494
|
| "depends-on"
|
|
1379
1495
|
| "unroutable:no-repo-label"
|
|
@@ -1425,6 +1541,7 @@ export type FrictionAdmissionReason =
|
|
|
1425
1541
|
| "issue-state-lookup-error"
|
|
1426
1542
|
| "open-pr-lookup-error"
|
|
1427
1543
|
| "stale-base"
|
|
1544
|
+
| "critical-base-verify-error"
|
|
1428
1545
|
| "unroutable:no-repo-label"
|
|
1429
1546
|
| "unroutable:multiple-repo-labels"
|
|
1430
1547
|
| "unroutable:unknown-repo";
|
|
@@ -1744,6 +1861,14 @@ export interface LabelOp {
|
|
|
1744
1861
|
* carry, so the operator reconciles by hand, which is the only safe move once
|
|
1745
1862
|
* the two disagree.
|
|
1746
1863
|
*/
|
|
1864
|
+
|
|
1865
|
+
/**
|
|
1866
|
+
* A settled `ci-deterministic` row offered to the historical infrastructure
|
|
1867
|
+
* reconciliation, with the internal `rowid` that breaks start-time ties so the
|
|
1868
|
+
* persisted review cursor can resume exactly below the last decided row (#638).
|
|
1869
|
+
*/
|
|
1870
|
+
export type HistoricalInfraCandidate = RunRecord & { rowid: number };
|
|
1871
|
+
|
|
1747
1872
|
export interface Store {
|
|
1748
1873
|
/** A claimed run atomically consumes and applies its issue's pending turn override. */
|
|
1749
1874
|
createRun(r: Omit<RunRecord, "id">): RunRecord;
|
|
@@ -1764,6 +1889,35 @@ export interface Store {
|
|
|
1764
1889
|
* per-issue view — a requeued continuation must never hide the PR its
|
|
1765
1890
|
* predecessor opened. Merged rows keep the same recent-history bound. */
|
|
1766
1891
|
runsForProjectPr(project: string, prUrl: string, mergedSinceEpochMs: number): RunRecord[];
|
|
1892
|
+
/** Persist one review-revision request (#677), durably, before anything is
|
|
1893
|
+
* woken. Returns `undefined` when a revision for the same run is already
|
|
1894
|
+
* pending undelivered — the atomic half of the duplicate in-flight guard
|
|
1895
|
+
* (the other half is the run row's state once the revision is dispatched). */
|
|
1896
|
+
createReviewRevision(draft: Omit<ReviewRevisionRecord, "id">): ReviewRevisionRecord | undefined;
|
|
1897
|
+
/** Review revisions not yet handed to a worker, oldest first — what the
|
|
1898
|
+
* daemon's dispatch pass wakes on its next tick. */
|
|
1899
|
+
pendingReviewRevisions(project: string): ReviewRevisionRecord[];
|
|
1900
|
+
/** Round number of the newest revision ever requested for a run (0 = never). */
|
|
1901
|
+
latestReviewRound(project: string, runId: string): number;
|
|
1902
|
+
/** Record that a revision was handed to a worker. */
|
|
1903
|
+
markReviewRevisionDispatched(id: string, at: number): void;
|
|
1904
|
+
/** Record how a revision ended. */
|
|
1905
|
+
settleReviewRevision(id: string, outcome: ReviewRevisionOutcome, at: number): void;
|
|
1906
|
+
/** Every revision row not yet settled — queued (`dispatchedAt` unset) and
|
|
1907
|
+
* dispatched-but-unfinished — oldest first. This is what a restart reads to
|
|
1908
|
+
* find review rounds a previous daemon left in flight (#692): the row is
|
|
1909
|
+
* the durable home of the findings, reviewed head, round and target
|
|
1910
|
+
* session, so re-queuing the row is how the exact session resumes. */
|
|
1911
|
+
unsettledReviewRevisions(project: string): ReviewRevisionRecord[];
|
|
1912
|
+
/** Return a dispatched revision to the pending set (clear its dispatched
|
|
1913
|
+
* marker) so the next dispatch pass wakes it again — the reopen half of
|
|
1914
|
+
* restart recovery, paired with restoring its run to `pushed-green`. */
|
|
1915
|
+
requeueReviewRevision(id: string): void;
|
|
1916
|
+
/** Atomically take a settled green run out of reviewable state:
|
|
1917
|
+
* `pushed-green` → `running`, and only from `pushed-green`. False when the
|
|
1918
|
+
* row already moved (a prior revision was dispatched, the PR settled), so
|
|
1919
|
+
* two concurrent dispatchers cannot both wake one run. */
|
|
1920
|
+
claimRunForReview(runId: string): boolean;
|
|
1767
1921
|
/** Merged rows whose post-merge workflow verdict is still pending, oldest first. */
|
|
1768
1922
|
runsNeedingBaseCheck(project: string, limit?: number): RunRecord[];
|
|
1769
1923
|
/** Replace the current live-head health row for one routed repository. */
|
|
@@ -2014,6 +2168,43 @@ export interface Store {
|
|
|
2014
2168
|
* than spending one tick on all of it.
|
|
2015
2169
|
*/
|
|
2016
2170
|
runsNeedingClassification(project: string, limit?: number): RunRecord[];
|
|
2171
|
+
/**
|
|
2172
|
+
* Settled rows currently reading `ci-deterministic` — the only rows the
|
|
2173
|
+
* historical infrastructure reconciliation may repair (#638): the verdict
|
|
2174
|
+
* the forward classifier no longer gives, and nothing else. Newest first
|
|
2175
|
+
* and bounded, because each candidate costs tracker calls to re-fetch the
|
|
2176
|
+
* evidence its log carries.
|
|
2177
|
+
*
|
|
2178
|
+
* `after` is the persisted review cursor (the last row a prior pass decided):
|
|
2179
|
+
* candidates are then only rows strictly *older* than it, so a bounded pass
|
|
2180
|
+
* progresses through a long history instead of rescanning the newest
|
|
2181
|
+
* non-matches forever. `rowid` on each candidate breaks the start-time tie,
|
|
2182
|
+
* giving the cursor an exact resume boundary. */
|
|
2183
|
+
historicalInfraCandidates(
|
|
2184
|
+
project: string,
|
|
2185
|
+
limit?: number,
|
|
2186
|
+
after?: { startedAt: number; rowid: number },
|
|
2187
|
+
): HistoricalInfraCandidate[];
|
|
2188
|
+
/** The persisted historical-review cursor for a project, or undefined before
|
|
2189
|
+
* any pass decided a row. The `classifierVersion` stamp tells the pass
|
|
2190
|
+
* whether the cursor is still current against the signature list. */
|
|
2191
|
+
historicalInfraCursor(
|
|
2192
|
+
project: string,
|
|
2193
|
+
): { startedAt: number; rowid: number; classifierVersion: string } | undefined;
|
|
2194
|
+
/** Persist (or move) the historical-review cursor after a pass decided rows. */
|
|
2195
|
+
setHistoricalInfraCursor(
|
|
2196
|
+
project: string,
|
|
2197
|
+
startedAt: number,
|
|
2198
|
+
rowid: number,
|
|
2199
|
+
classifierVersion: string,
|
|
2200
|
+
): void;
|
|
2201
|
+
/** Reclassify one settled `ci-deterministic` row to `ci-infra` (budget
|
|
2202
|
+
* exempt) after its re-fetched check log matched the closed infrastructure
|
|
2203
|
+
* signature. Changes `failureClass` only — `recoveryAction`/`recoveredAt`
|
|
2204
|
+
* stay byte-for-byte unchanged, so a months-old run is never re-animated
|
|
2205
|
+
* into recovery. False when the row was no longer `ci-deterministic`: a
|
|
2206
|
+
* no-op, exactly what a second repair pass must be. */
|
|
2207
|
+
reclassifyInfra(id: string): boolean;
|
|
2017
2208
|
/** Unrecovered rows per class, for `status`. Empty when nothing is carrying one. */
|
|
2018
2209
|
failureClassCounts(project: string): { cls: FailureClass; n: number }[];
|
|
2019
2210
|
/** Rows whose recovery ran at or after `since`, newest first — the tick's
|
|
@@ -2275,6 +2466,11 @@ export const VERB_NAMES = [
|
|
|
2275
2466
|
* install shape gates it like a release shape; execution detaches from this
|
|
2276
2467
|
* session and the daemon, and the first tick after the restart verifies. */
|
|
2277
2468
|
"conductor_install",
|
|
2469
|
+
/** Return one green, run-owned pull request to its worker with blocking
|
|
2470
|
+
* findings: durably record the revision, and the daemon resumes the SAME
|
|
2471
|
+
* OMP session on the existing branch and PR. No close, no reopen, no
|
|
2472
|
+
* redispatch. */
|
|
2473
|
+
"conductor_pr_review",
|
|
2278
2474
|
/** The one read verb. It answers with the merge gate's own verdict, so
|
|
2279
2475
|
* "pushed-green" is the dispatcher's reading of the PR rather than a claim the
|
|
2280
2476
|
* worker makes about itself from whatever it happened to run. */
|
|
@@ -2357,6 +2553,12 @@ export const VERB_REFUSALS = [
|
|
|
2357
2553
|
"checks-not-green",
|
|
2358
2554
|
/** Another merge is in flight for this project. */
|
|
2359
2555
|
"merge-in-flight",
|
|
2556
|
+
/**
|
|
2557
|
+
* The run backing a PR is not in a revisable settled-green state: a review
|
|
2558
|
+
* revision is already in flight (queued or dispatched), a worker is still
|
|
2559
|
+
* running on it, or the run settled otherwise. The detail names the state.
|
|
2560
|
+
*/
|
|
2561
|
+
"review-in-flight",
|
|
2360
2562
|
/**
|
|
2361
2563
|
* The routed repository's base branch is frozen because a watched merge (or
|
|
2362
2564
|
* live base observation) turned it red — a base-red-freeze. Further merges to
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/** Whether the CLI owns an interactive terminal rather than a pipe or plain-UI session. */
|
|
2
|
+
export function interactiveUi(): boolean {
|
|
3
|
+
return Boolean(
|
|
4
|
+
process.stdin.isTTY &&
|
|
5
|
+
process.stdout.isTTY &&
|
|
6
|
+
!process.env.OMP_CONDUCTOR_PLAIN_UI,
|
|
7
|
+
);
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/** Run one long, silent operation with safe Clack progress on an interactive TTY. */
|
|
11
|
+
export async function withProgress<T>(
|
|
12
|
+
message: string,
|
|
13
|
+
success: string,
|
|
14
|
+
operation: () => Promise<T>,
|
|
15
|
+
options: { plainMessage?: boolean } = {},
|
|
16
|
+
): Promise<T> {
|
|
17
|
+
if (!interactiveUi()) {
|
|
18
|
+
if (options.plainMessage) process.stdout.write(`${message}\n`);
|
|
19
|
+
return operation();
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const { log } = await import("@clack/prompts");
|
|
23
|
+
log.step(message);
|
|
24
|
+
try {
|
|
25
|
+
const result = await operation();
|
|
26
|
+
log.success(success);
|
|
27
|
+
return result;
|
|
28
|
+
} catch (err) {
|
|
29
|
+
log.error(`${message} failed`);
|
|
30
|
+
throw err;
|
|
31
|
+
}
|
|
32
|
+
}
|
package/src/ui/style.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { styleText } from "node:util";
|
|
2
|
+
|
|
3
|
+
function styled(format: Parameters<typeof styleText>[0], text: string): string {
|
|
4
|
+
return process.stdout.isTTY ? styleText(format, text) : text;
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export const heading = (text: string): string => styled(["bold", "cyan"], text);
|
|
8
|
+
export const ok = (text: string): string => styled("green", text);
|
|
9
|
+
export const warn = (text: string): string => styled("yellow", text);
|
|
10
|
+
export const fail = (text: string): string => styled("red", text);
|
|
11
|
+
export const dim = (text: string): string => styled("dim", text);
|
package/src/upgrade.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import { backupTimestamp, copyToUniqueBackup } from "./backups.ts";
|
|
3
3
|
import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
|
|
4
|
-
import { pauseInstance, setPaused, statusSnapshot } from "./daemon.ts";
|
|
4
|
+
import { pauseInstance, setPaused, statusSnapshot, type AdmissionAckRecord } from "./daemon.ts";
|
|
5
5
|
import {
|
|
6
6
|
DEFAULT_HERDR_SESSION,
|
|
7
7
|
fleetLayers,
|
|
@@ -335,6 +335,22 @@ export interface UpgradeScope {
|
|
|
335
335
|
pauseKey: string | undefined;
|
|
336
336
|
}
|
|
337
337
|
|
|
338
|
+
/**
|
|
339
|
+
* The subset of {@link UpgradeDeps} the pause/drain fence needs. Factored out
|
|
340
|
+
* so a caller that only wants the fence — `setup host` — can provide just
|
|
341
|
+
* these members instead of a full install-and-verify deps object. Every
|
|
342
|
+
* {@link UpgradeDeps} is a `DrainDeps`.
|
|
343
|
+
*/
|
|
344
|
+
/**
|
|
345
|
+
* The daemon a transaction targets: running or not, its project when it
|
|
346
|
+
* recorded one, and a `generation` identifying the exact instance (#377).
|
|
347
|
+
*/
|
|
348
|
+
export interface DaemonIdentity {
|
|
349
|
+
running: boolean;
|
|
350
|
+
project?: string;
|
|
351
|
+
generation?: string;
|
|
352
|
+
}
|
|
353
|
+
|
|
338
354
|
/**
|
|
339
355
|
* The subset of {@link UpgradeDeps} the pause/drain fence needs. Factored out
|
|
340
356
|
* so a caller that only wants the fence — `setup host` — can provide just
|
|
@@ -352,11 +368,20 @@ export interface DrainDeps {
|
|
|
352
368
|
* The daemon the transaction targets: running or not, its project when it
|
|
353
369
|
* recorded one, and a `generation` identifying the exact instance (#377).
|
|
354
370
|
*/
|
|
355
|
-
daemonIdentity():
|
|
371
|
+
daemonIdentity(): DaemonIdentity;
|
|
356
372
|
/** The active pause sentinel as an instance, when one is readable (#377). */
|
|
357
373
|
pauseState(project?: string): { source: string; reason?: string; since: number } | undefined;
|
|
358
374
|
/** Write or clear the durable pause sentinel for the scope's project. */
|
|
359
375
|
setPaused(value: boolean, project?: string): void;
|
|
376
|
+
/**
|
|
377
|
+
* The daemon's durable admission acknowledgement for the current fence, when
|
|
378
|
+
* one is readable: the daemon attests it has observed the pause sentinel at
|
|
379
|
+
* an admission boundary and will claim nothing while it stands (#651 review
|
|
380
|
+
* #3). Absent when no daemon is running, when the file cannot be read, or
|
|
381
|
+
* when the daemon has not (yet) observed the fence. The setup barrier
|
|
382
|
+
* requires this before any mutation when a daemon is running.
|
|
383
|
+
*/
|
|
384
|
+
admissionAck?(project?: string): AdmissionAckRecord | undefined;
|
|
360
385
|
sleep(ms: number): Promise<void>;
|
|
361
386
|
log(message: string): void;
|
|
362
387
|
}
|
|
@@ -455,31 +480,37 @@ export async function waitForDrain(
|
|
|
455
480
|
* live state against this snapshot, so the destructive restart call only ever
|
|
456
481
|
* acts on the world the request began in (#377).
|
|
457
482
|
*/
|
|
458
|
-
interface RestartBegun {
|
|
483
|
+
export interface RestartBegun {
|
|
459
484
|
/** The durable pause sentinel this request relies on, proved readable as an instance. */
|
|
460
485
|
pauseToken: { source: string; reason?: string; since: number };
|
|
461
486
|
/** The daemon instance the restart targets, with its generation identity. */
|
|
462
|
-
daemon:
|
|
487
|
+
daemon: DaemonIdentity;
|
|
463
488
|
}
|
|
464
489
|
|
|
465
490
|
/**
|
|
466
|
-
* Why a
|
|
467
|
-
* the pause and the daemon generation it began with. The pause
|
|
468
|
-
* *
|
|
469
|
-
* front, so a token that cannot be re-read (sentinel removed,
|
|
470
|
-
* re-created — even under the same source and reason) means
|
|
471
|
-
* pause no longer exists and it must cancel. The
|
|
472
|
-
* later-daemon fence: a daemon created after
|
|
473
|
-
* `restart --now`, a crash-restart, or an operator
|
|
474
|
-
* stopped by it.
|
|
491
|
+
* Why a pause-and-drain transaction can no longer act, or `undefined` when it
|
|
492
|
+
* still owns the pause and the daemon generation it began with. The pause
|
|
493
|
+
* check is the *transaction-owned sentinel instance* — the caller proves
|
|
494
|
+
* ownership up front, so a token that cannot be re-read (sentinel removed,
|
|
495
|
+
* unreadable, or re-created — even under the same source and reason) means
|
|
496
|
+
* this request's own pause no longer exists and it must cancel. The
|
|
497
|
+
* generation check is the later-daemon fence: a daemon created after the
|
|
498
|
+
* transaction began — by `restart --now`, a crash-restart, or an operator
|
|
499
|
+
* start — must never be stopped by it.
|
|
500
|
+
*
|
|
501
|
+
* Exported so the setup apply's acknowledged quiescence barrier runs the same
|
|
502
|
+
* fence instead of inventing a second counter-only one (#618): it freezes
|
|
503
|
+
* admission host-wide through {@link pauseAndDrain}'s machinery, and this
|
|
504
|
+
* check is what proves the freeze is still the one it acknowledged.
|
|
475
505
|
*/
|
|
476
|
-
function restartFenceProblem(
|
|
506
|
+
export function restartFenceProblem(
|
|
477
507
|
deps: DrainDeps,
|
|
478
508
|
scope: UpgradeScope,
|
|
479
509
|
begun: RestartBegun,
|
|
510
|
+
actor: string,
|
|
480
511
|
): string | undefined {
|
|
481
512
|
if (!deps.layers(scope.pauseKey).paused) {
|
|
482
|
-
return
|
|
513
|
+
return `${actor} cancelled: dispatch was resumed while the drain was in progress — nothing was restarted`;
|
|
483
514
|
}
|
|
484
515
|
const owned = deps.pauseState(scope.pauseKey);
|
|
485
516
|
if (
|
|
@@ -488,11 +519,11 @@ function restartFenceProblem(
|
|
|
488
519
|
owned.source !== begun.pauseToken.source ||
|
|
489
520
|
owned.reason !== begun.pauseToken.reason
|
|
490
521
|
) {
|
|
491
|
-
return
|
|
522
|
+
return `${actor} cancelled: the ${actor}-owned pause was lifted and replaced while the drain was in progress — nothing was restarted`;
|
|
492
523
|
}
|
|
493
524
|
const now = deps.daemonIdentity();
|
|
494
525
|
if (now.running !== begun.daemon.running || (begun.daemon.running && now.generation !== begun.daemon.generation)) {
|
|
495
|
-
return
|
|
526
|
+
return `${actor} cancelled: the daemon instance was replaced while the drain was in progress — the newer daemon was left running`;
|
|
496
527
|
}
|
|
497
528
|
return undefined;
|
|
498
529
|
}
|
|
@@ -504,7 +535,7 @@ function restartFenceProblem(
|
|
|
504
535
|
* {@link pauseSourceToken}; a verb with a space) fails the suite instead of
|
|
505
536
|
* shipping an unprovable pause (#552).
|
|
506
537
|
*/
|
|
507
|
-
export const DRAIN_VERBS = ["upgrade", "restart", "setup host"] as const;
|
|
538
|
+
export const DRAIN_VERBS = ["upgrade", "restart", "setup host", "setup"] as const;
|
|
508
539
|
export type DrainVerb = (typeof DRAIN_VERBS)[number];
|
|
509
540
|
|
|
510
541
|
/**
|
|
@@ -556,7 +587,7 @@ export async function pauseAndDrain(
|
|
|
556
587
|
pauseToken,
|
|
557
588
|
daemon: deps.daemonIdentity(),
|
|
558
589
|
};
|
|
559
|
-
const stale = () => restartFenceProblem(deps, scope, begun);
|
|
590
|
+
const stale = () => restartFenceProblem(deps, scope, begun, verb);
|
|
560
591
|
await waitForDrain(
|
|
561
592
|
deps,
|
|
562
593
|
scope,
|
package/src/verbs/actions.ts
CHANGED
|
@@ -20,6 +20,13 @@ import { stateDir } from "../config.ts";
|
|
|
20
20
|
import { ensureMirror } from "../worktree.ts";
|
|
21
21
|
import { launchTransientUnit, readUpgradeJournal } from "../upgrade-journal.ts";
|
|
22
22
|
import { classifyUpgrade } from "../upgrade-verify.ts";
|
|
23
|
+
import {
|
|
24
|
+
GraphqlBreaker,
|
|
25
|
+
graphqlBreakerRefusal,
|
|
26
|
+
isGraphqlSurface,
|
|
27
|
+
isTransientServer5xx,
|
|
28
|
+
prUrlParts,
|
|
29
|
+
} from "../tracker/github.ts";
|
|
23
30
|
import type { ProjectConfig, ReleaseShape, RepoTarget } from "../types.ts";
|
|
24
31
|
import type { ActionOutcome, InstallExecution, ReleaseExecution, VerbActions } from "./server.ts";
|
|
25
32
|
|
|
@@ -204,15 +211,32 @@ export function githubVerbActions(
|
|
|
204
211
|
project: ProjectConfig,
|
|
205
212
|
run: CommandRunner = spawnCommand,
|
|
206
213
|
prepareMirror: MirrorPreparer = ensureMirror,
|
|
214
|
+
graphqlBreaker?: GraphqlBreaker,
|
|
207
215
|
): VerbActions {
|
|
208
216
|
const env = (): Record<string, string> => credentialedEnv();
|
|
209
217
|
|
|
210
218
|
const runGh = async (argv: string[], cwd?: string): Promise<{ argv: string[]; result: CommandRun }> => {
|
|
211
219
|
const full = ["gh", ...argv];
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
220
|
+
// The mutation surface rides the same GraphQL endpoint the tracker's
|
|
221
|
+
// parent/closer checks ride in the pinned gh (2.86): `pr merge`,
|
|
222
|
+
// `pr update-branch`, `pr edit`, `pr create`. When a transient 5xx has
|
|
223
|
+
// already proven that surface unavailable — observed by admission or by a
|
|
224
|
+
// sibling mutation this pass — a mutation must fast-fail WITHOUT spawning
|
|
225
|
+
// `gh`, exactly like the tracker's candidate checks (#642 comment 4):
|
|
226
|
+
// attempting a mutation into a known-broken surface is how one provider
|
|
227
|
+
// outage defeats queue locking. REST-backed commands (`gh api repos/...`,
|
|
228
|
+
// `gh release create`) keep running: the outage is GraphQL-only.
|
|
229
|
+
if (graphqlBreaker !== undefined && isGraphqlSurface(argv) && graphqlBreaker.refused()) {
|
|
230
|
+
return {
|
|
231
|
+
argv: full,
|
|
232
|
+
result: { ok: false, stdout: "", stderr: graphqlBreakerRefusal(graphqlBreaker.until) },
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
const result = await run(full, { env: env(), ...(cwd === undefined ? {} : { cwd }) });
|
|
236
|
+
if (!result.ok && graphqlBreaker !== undefined && isGraphqlSurface(argv) && isTransientServer5xx(result.stderr)) {
|
|
237
|
+
graphqlBreaker.open();
|
|
238
|
+
}
|
|
239
|
+
return { argv: full, result };
|
|
216
240
|
};
|
|
217
241
|
|
|
218
242
|
const gh = async (argv: string[], cwd?: string): Promise<ActionOutcome> => {
|
|
@@ -573,7 +597,44 @@ export function githubVerbActions(
|
|
|
573
597
|
|
|
574
598
|
createPr: (target, opts) => openRunPr(project, target, opts),
|
|
575
599
|
|
|
576
|
-
updatePrBranch: (prUrl) =>
|
|
600
|
+
updatePrBranch: async (prUrl) => {
|
|
601
|
+
// The branch refresh rides the REST update-branch endpoint, not `gh pr
|
|
602
|
+
// update-branch` (#655): `gh pr update-branch` is a GraphQL mutation, so
|
|
603
|
+
// a selective GraphQL outage made an otherwise mergeable PR unrefreshable
|
|
604
|
+
// even while REST answered. Both calls here are core REST, so they keep
|
|
605
|
+
// running while the GraphQL breaker is open.
|
|
606
|
+
const parts = prUrlParts(prUrl);
|
|
607
|
+
if (parts === undefined) {
|
|
608
|
+
return { ok: false, stderr: `could not parse ${prUrl} as a pull request URL` };
|
|
609
|
+
}
|
|
610
|
+
const pulls = `repos/${parts.owner}/${parts.repo}/pulls/${parts.number}`;
|
|
611
|
+
// The stale-head guard needs the exact head the update is bound to. It is
|
|
612
|
+
// the last observed head of the branch refresh, so a concurrent push is
|
|
613
|
+
// refused (422) rather than updating an unreviewed head.
|
|
614
|
+
const head = await runGh(["api", pulls, "--jq", ".head.sha"]);
|
|
615
|
+
if (!head.result.ok) return failed(head.result, head.argv);
|
|
616
|
+
const headSha = commitFrom(head.result.stdout);
|
|
617
|
+
if (headSha === undefined) {
|
|
618
|
+
return { ok: false, stderr: `${prUrl} returned no full head commit SHA` };
|
|
619
|
+
}
|
|
620
|
+
const updated = await gh([
|
|
621
|
+
"api",
|
|
622
|
+
"--method",
|
|
623
|
+
"PUT",
|
|
624
|
+
`${pulls}/update-branch`,
|
|
625
|
+
"-f",
|
|
626
|
+
`expected_head_sha=${headSha}`,
|
|
627
|
+
]);
|
|
628
|
+
if (!updated.ok) return updated;
|
|
629
|
+
// The endpoint answers 202 Accepted and refreshes the branch
|
|
630
|
+
// asynchronously, so the head observed for `expected_head_sha` is the
|
|
631
|
+
// pre-update head. Producing it as `sha` would report a merge at a head
|
|
632
|
+
// nobody observed; the caller re-reads PR and check state instead.
|
|
633
|
+
return {
|
|
634
|
+
ok: true,
|
|
635
|
+
detail: `requested a base-branch update of ${prUrl} from its observed head ${headSha}; re-read PR and check state before merging`,
|
|
636
|
+
};
|
|
637
|
+
},
|
|
577
638
|
|
|
578
639
|
updatePr: async (prUrl, fields) => {
|
|
579
640
|
const argv = ["pr", "edit", prUrl];
|
|
@@ -589,19 +650,6 @@ export function githubVerbActions(
|
|
|
589
650
|
// and braces on the one operation that cannot be undone.
|
|
590
651
|
mergePr: (prUrl, headSha) => gh(["pr", "merge", prUrl, "--squash", "--match-head-commit", headSha]),
|
|
591
652
|
|
|
592
|
-
setLabel: async (issue, label, action) => {
|
|
593
|
-
const slug = project.tracker.repo;
|
|
594
|
-
return gh([
|
|
595
|
-
"issue",
|
|
596
|
-
"edit",
|
|
597
|
-
String(issue),
|
|
598
|
-
"--repo",
|
|
599
|
-
slug,
|
|
600
|
-
action === "add" ? "--add-label" : "--remove-label",
|
|
601
|
-
label,
|
|
602
|
-
]);
|
|
603
|
-
},
|
|
604
|
-
|
|
605
653
|
release: async (execution: ReleaseExecution): Promise<ActionOutcome> => {
|
|
606
654
|
const slug = repoSlugFor(execution.repo);
|
|
607
655
|
const tag = execution.tag;
|
package/src/verbs/protocol.ts
CHANGED
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
RELEASE_REASONS,
|
|
27
27
|
RELEASE_SHAPES,
|
|
28
28
|
RESERVED_VERB_FIELDS,
|
|
29
|
+
REVIEW_REASONS,
|
|
29
30
|
VERB_NAMES,
|
|
30
31
|
} from "../types.ts";
|
|
31
32
|
import type { ReservedVerbField, SessionRole, VerbName, VerbRefusal } from "../types.ts";
|
|
@@ -326,6 +327,50 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
|
|
|
326
327
|
`install authority is the orchestrator's, and this is a ${role} session. ` +
|
|
327
328
|
"A worker session holds no install grant whatever the config says.",
|
|
328
329
|
},
|
|
330
|
+
conductor_pr_review: {
|
|
331
|
+
name: "conductor_pr_review",
|
|
332
|
+
mutating: true,
|
|
333
|
+
allowedRoles: ["orchestrator"],
|
|
334
|
+
description:
|
|
335
|
+
"Return one green, run-owned pull request to its worker with blocking findings (#677). " +
|
|
336
|
+
"The daemon records the revision durably — findings, exact reviewed head, round number, " +
|
|
337
|
+
"target run and session — then resumes the SAME OMP session on the existing branch and PR: " +
|
|
338
|
+
"no close, no reopen, no issue redispatch, no fresh brief. The worker edits and pushes the " +
|
|
339
|
+
"same branch; a new green head returns to the normal orchestrator review surface. Refused " +
|
|
340
|
+
"for a stale head, a PR no run of this project owns, a run that is not settled green, or a " +
|
|
341
|
+
"revision already in flight for the same PR.",
|
|
342
|
+
args: {
|
|
343
|
+
prUrl: {
|
|
344
|
+
type: "string",
|
|
345
|
+
required: true,
|
|
346
|
+
description:
|
|
347
|
+
"Full pull request URL — one a run of this project opened and pushed green. Must be in " +
|
|
348
|
+
"a routed repository.",
|
|
349
|
+
},
|
|
350
|
+
headSha: {
|
|
351
|
+
type: "string",
|
|
352
|
+
required: true,
|
|
353
|
+
description:
|
|
354
|
+
"The exact reviewed head. The live head must match it and the checks at it must be green.",
|
|
355
|
+
},
|
|
356
|
+
findings: {
|
|
357
|
+
type: "string",
|
|
358
|
+
required: true,
|
|
359
|
+
description:
|
|
360
|
+
"Structured blocking findings. Delivered verbatim to the resumed session; the worker is " +
|
|
361
|
+
"told to address exactly these and push the same branch.",
|
|
362
|
+
},
|
|
363
|
+
reason: {
|
|
364
|
+
type: "string",
|
|
365
|
+
required: true,
|
|
366
|
+
description: `Why this revision, from the closed set: ${REVIEW_REASONS.join(", ")}.`,
|
|
367
|
+
oneOf: REVIEW_REASONS,
|
|
368
|
+
},
|
|
369
|
+
rationale: RATIONALE_ARG,
|
|
370
|
+
},
|
|
371
|
+
roleRefusalText: (role) =>
|
|
372
|
+
`review authority is the orchestrator's; this is a ${role} session, and a worker never returns its own PR for revision.`,
|
|
373
|
+
},
|
|
329
374
|
conductor_pr_status: {
|
|
330
375
|
name: "conductor_pr_status",
|
|
331
376
|
mutating: false,
|