@coreplane/switchboard 1.260.0 → 1.260.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/config/config.example.yaml +1 -0
- package/dist/assets/deploy/cloudflare/worker.ts +17 -56
- package/dist/assets/deploy/cloudflare/wrangler.template.jsonc +6 -0
- package/dist/assets/deploy/profile.example.json +2 -1
- package/dist/assets/package-lock.json +3 -3
- package/dist/assets/package.json +1 -1
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/core/budgets.ts +12 -6
- package/dist/assets/src/core/coordinator/driver.ts +9 -4
- package/dist/assets/src/core/pipelineStanding.ts +2 -0
- package/dist/assets/src/core/runEvents.ts +9 -1
- package/dist/assets/src/core/runLedger/types.ts +3 -3
- package/dist/assets/src/core/runRecord.ts +28 -8
- package/dist/assets/src/core/ship/coordinator.ts +145 -36
- package/dist/assets/src/core/ship/renewal.ts +2 -0
- package/dist/assets/src/deploy/liveGate.ts +8 -0
- package/dist/assets/src/deploy/profile.ts +8 -0
- package/dist/assets/src/deploy/restart.ts +49 -35
- package/dist/assets/web/dist/.vite/manifest.json +67 -67
- package/dist/assets/web/dist/assets/{DeliveryPage-DvMrWUg7.js → DeliveryPage-62qCSLvk.js} +1 -1
- package/dist/assets/web/dist/assets/{HomePage-CcGEJ4w0.js → HomePage-Dk3HRBc3.js} +1 -1
- package/dist/assets/web/dist/assets/{PendingTurnRow-BzGVxDYs.js → PendingTurnRow-CeTGOXUy.js} +1 -1
- package/dist/assets/web/dist/assets/{PlanePage-Arj9cyd5.js → PlanePage-RD0M6O6W.js} +1 -1
- package/dist/assets/web/dist/assets/{ResidentDetailPage-CBPPe4Sj.js → ResidentDetailPage-Dl2KId95.js} +1 -1
- package/dist/assets/web/dist/assets/{ResidentsIndexPage-CEpXgGo7.js → ResidentsIndexPage-DBy43I1R.js} +1 -1
- package/dist/assets/web/dist/assets/{RunFoldRow-BJq7tnLr.js → RunFoldRow-CwHDL-Mo.js} +1 -1
- package/dist/assets/web/dist/assets/{RunRoutePage-DWiny7LN.js → RunRoutePage-Du1HqLZs.js} +3 -3
- package/dist/assets/web/dist/assets/{RunsIndexPage-Cxufw5sV.js → RunsIndexPage-BZphy3Hr.js} +1 -1
- package/dist/assets/web/dist/assets/{ScheduledPage-SBcuDdVg.js → ScheduledPage-B7jh898t.js} +1 -1
- package/dist/assets/web/dist/assets/{SettingsPage-DrD_vWdp.js → SettingsPage-3uLEsiiE.js} +1 -1
- package/dist/assets/web/dist/assets/{SilentTurn-Bjl7EPEN.js → SilentTurn-CKxSxnQC.js} +1 -1
- package/dist/assets/web/dist/assets/{StatusDot-BqteQTav.js → StatusDot-x7vcK0iE.js} +1 -1
- package/dist/assets/web/dist/assets/{Tooltip-KRqJXEPN.js → Tooltip-wVXCWhMp.js} +1 -1
- package/dist/assets/web/dist/assets/{UnitRoutePage-Cv28FV4I.js → UnitRoutePage-V5BXK7FH.js} +1 -1
- package/dist/assets/web/dist/assets/budgets-KNNkT4PZ.js +1 -0
- package/dist/assets/web/dist/assets/{dist-CTASUSno.js → dist-C-XGnvM4.js} +1 -1
- package/dist/assets/web/dist/assets/{indexRow-B1YBy_IL.js → indexRow-Dxg7C8s9.js} +1 -1
- package/dist/assets/web/dist/assets/{main-eTAe9-hk.js → main-DAL--hyj.js} +2 -2
- package/dist/assets/web/dist/assets/sseReplay-BqgOggyQ.js +11 -0
- package/dist/cli.js +857 -288
- package/package.json +1 -1
- package/dist/assets/web/dist/assets/budgets-DQXVllsm.js +0 -1
- package/dist/assets/web/dist/assets/sseReplay-BcsNbn2j.js +0 -11
|
@@ -551,10 +551,10 @@ export type CoordinatorAction =
|
|
|
551
551
|
| { type: "merge"; step: string; prNumber: number; headSha: string; queued?: true }
|
|
552
552
|
/** Read the check runs at the reviewed head (record 0055, the round verdict):
|
|
553
553
|
* the merge door's own reading, folded into the round. `retry` names the
|
|
554
|
-
* failed checks whose one flake re-run the machine is spending
|
|
555
|
-
*
|
|
556
|
-
* of
|
|
557
|
-
| { type: "checks"; step: string; prNumber: number; headSha: string; retry?: string[] }
|
|
554
|
+
* failed checks whose one flake re-run the machine is spending; `refire`
|
|
555
|
+
* asks for the one close/reopen recovery when no required check launched.
|
|
556
|
+
* Either effect runs instead of a read. */
|
|
557
|
+
| { type: "checks"; step: string; prNumber: number; headSha: string; retry?: string[]; refire?: true }
|
|
558
558
|
/** Wait for the intake's checks-settled event at the approved head, bounded as the fallback. */
|
|
559
559
|
| { type: "wait-checks"; step: string; headSha: string; timeoutMs: number }
|
|
560
560
|
| { type: "sleep"; step: string; ms: number }
|
|
@@ -680,10 +680,18 @@ export type StepReturn =
|
|
|
680
680
|
* queue removed it — the reason is the queue's own (issue 2011). */
|
|
681
681
|
| { type: "merge"; step: string; outcome: "pending" | "refused" | "enqueued" | "removed"; reason: string; at: number }
|
|
682
682
|
/** The checks read at the reviewed head; `checks` absent means GitHub could
|
|
683
|
-
* not be read, which the machine treats as pending (record 0055).
|
|
684
|
-
*
|
|
685
|
-
*
|
|
686
|
-
| {
|
|
683
|
+
* not be read, which the machine treats as pending (record 0055). Effect
|
|
684
|
+
* asks answer `retried` or `refired` instead; false still spends that one
|
|
685
|
+
* recovery so an unchanged head cannot loop on the write. */
|
|
686
|
+
| {
|
|
687
|
+
type: "checks";
|
|
688
|
+
step: string;
|
|
689
|
+
checks?: RoundChecks;
|
|
690
|
+
draft?: boolean;
|
|
691
|
+
retried?: boolean;
|
|
692
|
+
refired?: boolean;
|
|
693
|
+
at: number;
|
|
694
|
+
}
|
|
687
695
|
| { type: "wait-checks"; step: string; outcome: "event" | "timeout" }
|
|
688
696
|
| { type: "sleep"; step: string };
|
|
689
697
|
|
|
@@ -699,14 +707,18 @@ export interface CheckFailure {
|
|
|
699
707
|
}
|
|
700
708
|
|
|
701
709
|
/** The check runs at the reviewed head as the round's checks step reads them.
|
|
702
|
-
* `
|
|
703
|
-
*
|
|
704
|
-
*
|
|
705
|
-
*
|
|
710
|
+
* `required` carries the base's whole required-context set and `expected`
|
|
711
|
+
* its unreported subset. The whole set lets the table distinguish no required
|
|
712
|
+
* check launching from an unrelated run that did report; an ordinary missing
|
|
713
|
+
* subset reads exactly as pending (issue 2063). */
|
|
706
714
|
export interface RoundChecks {
|
|
707
715
|
total: number;
|
|
708
716
|
pending: string[];
|
|
709
717
|
failed: CheckFailure[];
|
|
718
|
+
/** Every check context required by the base. Together with `expected`, this
|
|
719
|
+
* distinguishes an empty required-check launch from an unrelated check
|
|
720
|
+
* that did report (for example the title workflow). */
|
|
721
|
+
required?: string[];
|
|
710
722
|
expected?: string[];
|
|
711
723
|
}
|
|
712
724
|
|
|
@@ -831,6 +843,8 @@ export type UnitEnding =
|
|
|
831
843
|
finalReply?: string;
|
|
832
844
|
/** A changes-requested review posted this round before the stop. */
|
|
833
845
|
postedReview?: boolean;
|
|
846
|
+
/** The mechanical WIP checkpoint a stopped coding child pushed before teardown. */
|
|
847
|
+
checkpoint?: { branch: string; sha: string };
|
|
834
848
|
}
|
|
835
849
|
| {
|
|
836
850
|
kind: "aborted";
|
|
@@ -874,7 +888,14 @@ export type UnitEnding =
|
|
|
874
888
|
* `transient` so it reads as a condition beside `checks_failed` and `held`
|
|
875
889
|
* in the plane's table, never as the child failing on its task. */
|
|
876
890
|
| { kind: "transient"; round: RoundRef; runId: string; reviewRounds: number }
|
|
877
|
-
| {
|
|
891
|
+
| {
|
|
892
|
+
kind: "interrupted";
|
|
893
|
+
round: RoundRef;
|
|
894
|
+
runId: string;
|
|
895
|
+
reviewRounds: number;
|
|
896
|
+
cause?: InterruptionCause;
|
|
897
|
+
checkpoint?: { branch: string; sha: string };
|
|
898
|
+
}
|
|
878
899
|
| { kind: "refused"; refusal: string; message?: string; round: RoundRef; reviewRounds: number }
|
|
879
900
|
/** The unit idles instead of ending (record 0051): with the resolved
|
|
880
901
|
* `ship.idleDays` above zero, `end()` wraps an idling kind — every kind but
|
|
@@ -1072,9 +1093,10 @@ type Phase =
|
|
|
1072
1093
|
| { at: "merge-wait"; pr: PrRef; headSha: string; n: number; since: number; waitMs: number; queued?: true }
|
|
1073
1094
|
/** The round's checks step (record 0055): the check runs at the reviewed head
|
|
1074
1095
|
* are read after an approve settles, before merge_ready or the merge door.
|
|
1075
|
-
* `graced`: a head with no check reported has had its one-chunk
|
|
1076
|
-
* `
|
|
1077
|
-
* the
|
|
1096
|
+
* `graced`: a head with no required check reported has had its one-chunk
|
|
1097
|
+
* grace; `refired`: the one pull_request close/reopen recovery is spent;
|
|
1098
|
+
* `retried`: the one flake re-run is spent. `retry` and `refire` make the
|
|
1099
|
+
* next ask perform that effect instead of reading. */
|
|
1078
1100
|
| {
|
|
1079
1101
|
at: "checks";
|
|
1080
1102
|
round: RoundRef;
|
|
@@ -1085,7 +1107,9 @@ type Phase =
|
|
|
1085
1107
|
waitMs: number;
|
|
1086
1108
|
graced: boolean;
|
|
1087
1109
|
retried: boolean;
|
|
1110
|
+
refired: boolean;
|
|
1088
1111
|
retry?: string[];
|
|
1112
|
+
refire?: true;
|
|
1089
1113
|
}
|
|
1090
1114
|
/** Waiting on `checks-settled-<head>` between two checks reads, in the merge
|
|
1091
1115
|
* wait's own chunks — the intake wakes the machine when the head settles. */
|
|
@@ -1099,6 +1123,7 @@ type Phase =
|
|
|
1099
1123
|
waitMs: number;
|
|
1100
1124
|
graced: boolean;
|
|
1101
1125
|
retried: boolean;
|
|
1126
|
+
refired: boolean;
|
|
1102
1127
|
}
|
|
1103
1128
|
| { at: "ended" };
|
|
1104
1129
|
|
|
@@ -1338,6 +1363,7 @@ export function nextAction(s: UnitPipelineState): CoordinatorAction {
|
|
|
1338
1363
|
prNumber: p.prNumber,
|
|
1339
1364
|
headSha: p.headSha,
|
|
1340
1365
|
...(p.retry !== undefined ? { retry: p.retry } : {}),
|
|
1366
|
+
...(p.refire === true ? { refire: true as const } : {}),
|
|
1341
1367
|
};
|
|
1342
1368
|
case "checks-wait":
|
|
1343
1369
|
return {
|
|
@@ -1518,6 +1544,16 @@ function nextReview(s: UnitPipelineState, notes: CoordinatorNote[] = []): Transi
|
|
|
1518
1544
|
const stopMode = (status: RunStatus): "soft" | "hard" | undefined =>
|
|
1519
1545
|
status === "stopped_soft" ? "soft" : status === "stopped_hard" ? "hard" : undefined;
|
|
1520
1546
|
|
|
1547
|
+
/** The last mechanical WIP push to this unit branch. Unlike an ordinary push,
|
|
1548
|
+
* it says the child ended before its work was ready for review. */
|
|
1549
|
+
function interruptedCheckpoint(
|
|
1550
|
+
s: UnitPipelineState,
|
|
1551
|
+
pushed: readonly PushedHeadFact[] | undefined,
|
|
1552
|
+
): { branch: string; sha: string } | undefined {
|
|
1553
|
+
const found = pushed?.filter((p) => p.ref === s.input.unit.branch && p.by === "salvage").at(-1);
|
|
1554
|
+
return found ? { branch: found.ref, sha: found.sha } : undefined;
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1521
1557
|
/** A coding run's confirmed end: round 0's child, or the run a findings step dispatched. */
|
|
1522
1558
|
function settleCoding(
|
|
1523
1559
|
s: UnitPipelineState,
|
|
@@ -1545,6 +1581,7 @@ function settleCoding(
|
|
|
1545
1581
|
}
|
|
1546
1582
|
: {}),
|
|
1547
1583
|
};
|
|
1584
|
+
const checkpoint = interruptedCheckpoint(next, facts.pushed);
|
|
1548
1585
|
const mode = stopMode(facts.status);
|
|
1549
1586
|
if (mode !== undefined)
|
|
1550
1587
|
return end(
|
|
@@ -1555,13 +1592,33 @@ function settleCoding(
|
|
|
1555
1592
|
round,
|
|
1556
1593
|
reviewRounds: next.reviewRounds,
|
|
1557
1594
|
...(facts.finalReply !== undefined ? { finalReply: facts.finalReply } : {}),
|
|
1595
|
+
...(checkpoint !== undefined ? { checkpoint } : {}),
|
|
1558
1596
|
},
|
|
1559
1597
|
[roundNote(round, "stopped")],
|
|
1560
1598
|
);
|
|
1599
|
+
// A mechanical WIP push means the child did not declare the work ready for
|
|
1600
|
+
// review. Keep the unit branch as the resumption point and abort this
|
|
1601
|
+
// attempt; an ordinary push still takes the established recover-PR path.
|
|
1602
|
+
if (facts.status === "failed" && checkpoint !== undefined) {
|
|
1603
|
+
const cause =
|
|
1604
|
+
facts.failure?.kind === "provider_transient"
|
|
1605
|
+
? "the model provider's transport retry budget was spent"
|
|
1606
|
+
: "it failed before finishing";
|
|
1607
|
+
return end(
|
|
1608
|
+
next,
|
|
1609
|
+
{
|
|
1610
|
+
kind: "aborted",
|
|
1611
|
+
reason: `⚠️ The coding child ended because ${cause}; the branch carries the interrupted work at \`${checkpoint.sha.slice(0, 7)}\` on \`${checkpoint.branch}\`. Re-issue the request to resume from it instead of starting over.`,
|
|
1612
|
+
round,
|
|
1613
|
+
reviewRounds: next.reviewRounds,
|
|
1614
|
+
},
|
|
1615
|
+
[roundNote(round, "aborted")],
|
|
1616
|
+
);
|
|
1617
|
+
}
|
|
1561
1618
|
// A failed coding child no longer aborts outright: the pr-check looks at the
|
|
1562
|
-
// branch first —
|
|
1563
|
-
//
|
|
1564
|
-
//
|
|
1619
|
+
// branch first — an ordinary push before the death is recovered as the
|
|
1620
|
+
// round's pull request, and only a branch with nothing on it ends the unit
|
|
1621
|
+
// with the child's own reason.
|
|
1565
1622
|
if (facts.status === "failed")
|
|
1566
1623
|
return {
|
|
1567
1624
|
state: {
|
|
@@ -1745,6 +1802,7 @@ function enterChecks(s: UnitPipelineState, round: RoundRef, notes: CoordinatorNo
|
|
|
1745
1802
|
waitMs,
|
|
1746
1803
|
graced: false,
|
|
1747
1804
|
retried: false,
|
|
1805
|
+
refired: false,
|
|
1748
1806
|
},
|
|
1749
1807
|
},
|
|
1750
1808
|
notes,
|
|
@@ -1764,14 +1822,16 @@ function enterChecks(s: UnitPipelineState, round: RoundRef, notes: CoordinatorNo
|
|
|
1764
1822
|
* (`expected`: a required check, or the repository's approve workflow
|
|
1765
1823
|
* whose run does not exist yet), or GitHub unreadable: the head waits on
|
|
1766
1824
|
* the settled event and is read again;
|
|
1767
|
-
* (—) no check reported
|
|
1825
|
+
* (—) no required check reported: one chunk of grace, then the pull_request
|
|
1826
|
+
* event is re-fired once by closing and reopening the pull request;
|
|
1768
1827
|
* (c) every expected check reported green: the round proceeds with no wait. */
|
|
1769
1828
|
function checksVerdict(
|
|
1770
1829
|
checks: RoundChecks | undefined,
|
|
1771
|
-
p: { retried: boolean; graced: boolean },
|
|
1830
|
+
p: { retried: boolean; graced: boolean; refired: boolean },
|
|
1772
1831
|
draft?: boolean,
|
|
1773
1832
|
):
|
|
1774
1833
|
| { kind: "retry"; names: string[] }
|
|
1834
|
+
| { kind: "refire" }
|
|
1775
1835
|
| { kind: "failed"; failed: CheckFailure[] }
|
|
1776
1836
|
| { kind: "draft" }
|
|
1777
1837
|
| { kind: "pending" }
|
|
@@ -1791,14 +1851,18 @@ function checksVerdict(
|
|
|
1791
1851
|
// person's "ready" changes anything — a red check above still gets its fix
|
|
1792
1852
|
// round, since that work stands whether or not the pull request is a draft.
|
|
1793
1853
|
if (draft === true) return { kind: "draft" };
|
|
1854
|
+
const required = checks?.required ?? [];
|
|
1855
|
+
const missing = checks?.expected ?? [];
|
|
1856
|
+
const noRequiredReported = required.length > 0 && missing.length === required.length;
|
|
1857
|
+
// A check launcher may report an unrelated workflow while every required
|
|
1858
|
+
// context is still absent. Give it one normal chunk; if it remains empty,
|
|
1859
|
+
// re-fire pull_request once. This is the same recovery a person performs for
|
|
1860
|
+
// a CI run that exists but started no workflows.
|
|
1861
|
+
if ((checks?.total === 0 || noRequiredReported) && !p.graced) return { kind: "grace" };
|
|
1862
|
+
if (noRequiredReported && !p.refired) return { kind: "refire" };
|
|
1794
1863
|
// GitHub unreadable answers as pending and is re-read at the chunk's end;
|
|
1795
1864
|
// an expected check not yet reported (issue 2063) is pending the same way.
|
|
1796
|
-
if (checks === undefined || checks.pending.length > 0 ||
|
|
1797
|
-
return { kind: "pending" };
|
|
1798
|
-
// No check reported at the head: one chunk of grace — the first check starts
|
|
1799
|
-
// within minutes where CI exists — then the round proceeds, so a repository
|
|
1800
|
-
// without CI costs one chunk per round and never idles (record 0055).
|
|
1801
|
-
if (checks.total === 0 && !p.graced) return { kind: "grace" };
|
|
1865
|
+
if (checks === undefined || checks.pending.length > 0 || missing.length > 0) return { kind: "pending" };
|
|
1802
1866
|
return { kind: "green" };
|
|
1803
1867
|
}
|
|
1804
1868
|
|
|
@@ -1810,14 +1874,15 @@ function checksVerdict(
|
|
|
1810
1874
|
* is still pending (issue 1991); only a head with no failed check waits a
|
|
1811
1875
|
* chunk inside the step's ask on what is unreadable or pending and then
|
|
1812
1876
|
* proceeds — the ending's facts read names what is still pending; a head with
|
|
1813
|
-
* no check reported
|
|
1814
|
-
* proceeds with no wait added. */
|
|
1877
|
+
* no required check reported gets one grace chunk and one event re-fire; a
|
|
1878
|
+
* green head proceeds with no wait added. */
|
|
1815
1879
|
function settleChecks(
|
|
1816
1880
|
s: UnitPipelineState,
|
|
1817
1881
|
p: Extract<Phase, { at: "checks" }>,
|
|
1818
1882
|
checks: RoundChecks | undefined,
|
|
1819
1883
|
retried?: boolean,
|
|
1820
1884
|
draft?: boolean,
|
|
1885
|
+
refired?: boolean,
|
|
1821
1886
|
): Transition {
|
|
1822
1887
|
const { round } = p;
|
|
1823
1888
|
const wait = (over: Partial<Extract<Phase, { at: "checks-wait" }>>): Transition => ({
|
|
@@ -1833,11 +1898,23 @@ function settleChecks(
|
|
|
1833
1898
|
waitMs: p.waitMs,
|
|
1834
1899
|
graced: p.graced,
|
|
1835
1900
|
retried: p.retried,
|
|
1901
|
+
refired: p.refired,
|
|
1836
1902
|
...over,
|
|
1837
1903
|
},
|
|
1838
1904
|
},
|
|
1839
1905
|
notes: [],
|
|
1840
1906
|
});
|
|
1907
|
+
// The pull_request re-fire is an effect ask after a whole grace chunk with
|
|
1908
|
+
// no required check. Whether GitHub accepted it or not, spend the one attempt
|
|
1909
|
+
// and read immediately: that read registers the ordinary bounded wait. A
|
|
1910
|
+
// successful effect adds the round boundary the card renders.
|
|
1911
|
+
if (p.refire === true) {
|
|
1912
|
+
const { refire: _refire, ...read } = p;
|
|
1913
|
+
return {
|
|
1914
|
+
state: { ...s, phase: { ...read, n: p.n + 1, refired: true } },
|
|
1915
|
+
notes: refired === true ? [roundNote(p.round, "checks_restarted")] : [],
|
|
1916
|
+
};
|
|
1917
|
+
}
|
|
1841
1918
|
// The retry ask. Dispatched: wait for the head to settle, then read again;
|
|
1842
1919
|
// a second failure is the finding. NOT dispatched (`retried: false` — no
|
|
1843
1920
|
// re-runnable run behind the checks, or GitHub refused): the one re-run is
|
|
@@ -1858,6 +1935,11 @@ function settleChecks(
|
|
|
1858
1935
|
state: { ...s, phase: { ...p, n: p.n + 1, retry: verdict.names } },
|
|
1859
1936
|
notes: [],
|
|
1860
1937
|
};
|
|
1938
|
+
case "refire":
|
|
1939
|
+
return {
|
|
1940
|
+
state: { ...s, phase: { ...p, n: p.n + 1, refire: true } },
|
|
1941
|
+
notes: [],
|
|
1942
|
+
};
|
|
1861
1943
|
case "failed": {
|
|
1862
1944
|
const findings = [...(s.findingsByRound[round.index] ?? []), ...verdict.failed.map(checkFinding)];
|
|
1863
1945
|
const next: UnitPipelineState = {
|
|
@@ -2387,7 +2469,8 @@ export function applyReturn(s: UnitPipelineState, ret: StepReturn): Transition {
|
|
|
2387
2469
|
// The hosted parent's hard stop landed while this child ran (record 0060;
|
|
2388
2470
|
// issue 1924): the unit ends stopped as the child ends, whatever the
|
|
2389
2471
|
// child's own status — the runner runs nothing more of it.
|
|
2390
|
-
if (r.stopped === true)
|
|
2472
|
+
if (r.stopped === true) {
|
|
2473
|
+
const checkpoint = p.round.kind === "review" ? undefined : interruptedCheckpoint(clocked, r.run.pushed);
|
|
2391
2474
|
return end(
|
|
2392
2475
|
clocked,
|
|
2393
2476
|
{
|
|
@@ -2396,14 +2479,33 @@ export function applyReturn(s: UnitPipelineState, ret: StepReturn): Transition {
|
|
|
2396
2479
|
round: p.round,
|
|
2397
2480
|
reviewRounds: s.reviewRounds,
|
|
2398
2481
|
...(r.run.finalReply !== undefined ? { finalReply: r.run.finalReply } : {}),
|
|
2482
|
+
...(checkpoint !== undefined ? { checkpoint } : {}),
|
|
2399
2483
|
},
|
|
2400
2484
|
[roundNote(p.round, "stopped")],
|
|
2401
2485
|
);
|
|
2486
|
+
}
|
|
2402
2487
|
if (r.run.status === "interrupted") {
|
|
2403
2488
|
const cause = r.run.interruption;
|
|
2404
|
-
|
|
2405
|
-
//
|
|
2406
|
-
//
|
|
2489
|
+
const checkpoint = interruptedCheckpoint(clocked, r.run.pushed);
|
|
2490
|
+
// A mechanical WIP checkpoint is deliberately not opened for review:
|
|
2491
|
+
// the interrupted child never declared it finished. The next attempt
|
|
2492
|
+
// starts from the unit branch at that head.
|
|
2493
|
+
if (p.round.kind !== "review" && checkpoint !== undefined)
|
|
2494
|
+
return end(
|
|
2495
|
+
clocked,
|
|
2496
|
+
{
|
|
2497
|
+
kind: "interrupted",
|
|
2498
|
+
round: p.round,
|
|
2499
|
+
runId: p.runId,
|
|
2500
|
+
reviewRounds: s.reviewRounds,
|
|
2501
|
+
...(cause !== undefined ? { cause } : {}),
|
|
2502
|
+
checkpoint,
|
|
2503
|
+
},
|
|
2504
|
+
[roundNote(p.round, "aborted")],
|
|
2505
|
+
);
|
|
2506
|
+
// A dead CODING child may have pushed normally before the ledger closed
|
|
2507
|
+
// it: the pr-check recovers that branch. A review child has nothing on
|
|
2508
|
+
// the branch to recover, so its interruption still ends the unit at once.
|
|
2407
2509
|
if (p.round.kind !== "review")
|
|
2408
2510
|
return {
|
|
2409
2511
|
state: {
|
|
@@ -2438,7 +2540,7 @@ export function applyReturn(s: UnitPipelineState, ret: StepReturn): Transition {
|
|
|
2438
2540
|
return settlePrCheck(clocked, p, (ret as Extract<StepReturn, { type: "pr-check" }>).pr);
|
|
2439
2541
|
case "checks": {
|
|
2440
2542
|
const r = ret as Extract<StepReturn, { type: "checks" }>;
|
|
2441
|
-
return settleChecks(clocked, p, r.checks, r.retried, r.draft);
|
|
2543
|
+
return settleChecks(clocked, p, r.checks, r.retried, r.draft, r.refired);
|
|
2442
2544
|
}
|
|
2443
2545
|
case "checks-wait":
|
|
2444
2546
|
// The event fired or the chunk elapsed either way the head is read again.
|
|
@@ -2455,6 +2557,7 @@ export function applyReturn(s: UnitPipelineState, ret: StepReturn): Transition {
|
|
|
2455
2557
|
waitMs: p.waitMs,
|
|
2456
2558
|
graced: p.graced,
|
|
2457
2559
|
retried: p.retried,
|
|
2560
|
+
refired: p.refired,
|
|
2458
2561
|
},
|
|
2459
2562
|
},
|
|
2460
2563
|
notes: [],
|
|
@@ -2725,6 +2828,10 @@ export function renderUnitReport(
|
|
|
2725
2828
|
? `Findings below ${level}, left as-is: ${skipped.map((f) => `${f.id} (${f.severity}) — ${f.title}`).join("; ")}`
|
|
2726
2829
|
: undefined;
|
|
2727
2830
|
const join = (parts: Array<string | undefined>) => parts.filter(Boolean).join("\n\n");
|
|
2831
|
+
const checkpointLine = (checkpoint: { branch: string; sha: string } | undefined) =>
|
|
2832
|
+
checkpoint === undefined
|
|
2833
|
+
? undefined
|
|
2834
|
+
: `The branch carries the interrupted work at \`${checkpoint.sha.slice(0, 7)}\` on \`${checkpoint.branch}\`; re-issue to resume from it instead of starting over.`;
|
|
2728
2835
|
switch (e.kind) {
|
|
2729
2836
|
case "merged":
|
|
2730
2837
|
if (e.by === "other")
|
|
@@ -2873,9 +2980,11 @@ export function renderUnitReport(
|
|
|
2873
2980
|
aside(reissue),
|
|
2874
2981
|
]);
|
|
2875
2982
|
case "stopped":
|
|
2876
|
-
if (!shows(verbosity, "verbose"))
|
|
2983
|
+
if (!shows(verbosity, "verbose"))
|
|
2984
|
+
return join([`${e.mode === "hard" ? "⛔" : "⏹"} Stopped.${prLine}`, checkpointLine(e.checkpoint)]);
|
|
2877
2985
|
return join([
|
|
2878
2986
|
`${e.mode === "hard" ? "⛔" : "⏹"} Ship stopped by operator (${e.mode} stop) after ${rounds}.${prLine}`,
|
|
2987
|
+
checkpointLine(e.checkpoint),
|
|
2879
2988
|
writeUpPointer(
|
|
2880
2989
|
s,
|
|
2881
2990
|
e.round.kind,
|
|
@@ -2930,7 +3039,7 @@ export function renderUnitReport(
|
|
|
2930
3039
|
reissue,
|
|
2931
3040
|
]);
|
|
2932
3041
|
case "interrupted":
|
|
2933
|
-
return shipInterruptedNote(prUrl, e.cause, (s.input.idleDays ?? 0) > 0);
|
|
3042
|
+
return join([shipInterruptedNote(prUrl, e.cause, (s.input.idleDays ?? 0) > 0), checkpointLine(e.checkpoint)]);
|
|
2934
3043
|
case "idle_expired":
|
|
2935
3044
|
return "⌛ Idle expired: no reply continued this unit before its idle window closed.";
|
|
2936
3045
|
case "refused":
|
|
@@ -22,6 +22,9 @@ export interface HealthzBody {
|
|
|
22
22
|
build?: { commit?: unknown; builtAt?: unknown } | unknown;
|
|
23
23
|
/** ISO process start — `deploy restart`'s identity (the image, hence `build.commit`, is unchanged). */
|
|
24
24
|
startedAt?: unknown;
|
|
25
|
+
/** A refusal-only production boot's public reason. Such a generation owns a
|
|
26
|
+
* port so the operator can diagnose it, but is never live. */
|
|
27
|
+
config?: unknown;
|
|
25
28
|
}
|
|
26
29
|
|
|
27
30
|
/** Parse a `/healthz` response body; undefined when it is not a JSON object
|
|
@@ -83,6 +86,11 @@ function decideReady<Identity>(
|
|
|
83
86
|
let reason: string;
|
|
84
87
|
if (!body) {
|
|
85
88
|
reason = "/healthz not answering with JSON (container restarting, or unreachable)";
|
|
89
|
+
} else if (body.ok !== true) {
|
|
90
|
+
reason =
|
|
91
|
+
typeof body.config === "string" && body.config !== ""
|
|
92
|
+
? `config: ${body.config}`
|
|
93
|
+
: "/healthz reports this container is not ready";
|
|
86
94
|
} else {
|
|
87
95
|
const verdict = identify(body);
|
|
88
96
|
if (verdict.live) return { kind: "live", ...verdict.identity };
|
|
@@ -79,6 +79,14 @@ export const profileSchema = z.object({
|
|
|
79
79
|
/** Where the bot's runtime config comes from at deploy time; `deploy all`
|
|
80
80
|
* materializes it into the image's build context. */
|
|
81
81
|
configSource: source,
|
|
82
|
+
/** The ingress-token subject allowed to stop the bot container. Rendered as
|
|
83
|
+
* a Worker var, so config recovery never depends on the runtime document it
|
|
84
|
+
* may be repairing. Absent disables the restart route. */
|
|
85
|
+
restart: z
|
|
86
|
+
.object({
|
|
87
|
+
deployer: z.string().regex(/^[A-Za-z0-9_.:@/-]+$/, "an ingress identity subject, without whitespace"),
|
|
88
|
+
})
|
|
89
|
+
.optional(),
|
|
82
90
|
/** Where `secrets put` reads values from: a directory of `<NAME>` files, or
|
|
83
91
|
* `op://Vault/Item` with the secret's name as the field. Optional: the
|
|
84
92
|
* secrets tooling has its own default directory. */
|
|
@@ -18,34 +18,25 @@ import { profileUrls, type DeploymentProfile } from "./profile.js";
|
|
|
18
18
|
// (src/deploy/run.ts `runBotRestart`), so both sides are unit-tested here and
|
|
19
19
|
// nothing in this file imports node:*.
|
|
20
20
|
//
|
|
21
|
-
// Authorization: the bearer
|
|
22
|
-
// `SWITCHBOARD_INGRESS_TOKENS
|
|
23
|
-
// `
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
// bearer is — `authenticateRestart`, 401/503 without touching the container —
|
|
29
|
-
// but the grants live in the container's config, so it asks the bot
|
|
30
|
-
// (`POST /admin/restart/authorize`, src/channels/adminRestartAuthorize.ts) whether
|
|
31
|
-
// that subject holds the action and relays the answer (`parseRestartAuthorization`).
|
|
32
|
-
// The Worker names the subject it authenticated in `RESTART_SUBJECT_HEADER` and
|
|
33
|
-
// the bot decides the grant for THAT subject without re-authenticating the
|
|
34
|
-
// bearer: the two sides can hold different generations of the token map (a
|
|
35
|
-
// `wrangler secret put` reaches the Worker's env at once and the container's
|
|
36
|
-
// only after a restart), so re-authenticating in the container would refuse the
|
|
37
|
-
// very rotation the restart exists to finish. The header is trustworthy because
|
|
38
|
-
// the container is reachable only through the Worker, which strips it from every
|
|
39
|
-
// proxied request (`stripRestartSubject`) and sets it only on its own internal
|
|
40
|
-
// call. The bot's own `/admin/crash` runs the whole check in one place
|
|
41
|
-
// (`authorizeRestart`).
|
|
21
|
+
// Authorization stays entirely on the Worker side: the bearer is an entry of
|
|
22
|
+
// `SWITCHBOARD_INGRESS_TOKENS`, and its subject must equal
|
|
23
|
+
// `SWITCHBOARD_RESTART_DEPLOYER`, rendered from the deployment profile (or set
|
|
24
|
+
// directly as a Worker var). The route never asks the container or its runtime
|
|
25
|
+
// grants: a missing/invalid config is exactly when recovery must still work.
|
|
26
|
+
// The bot's separate `/admin/crash` check still uses `authorizeRestart`, the
|
|
27
|
+
// ordinary runtime-grant path for a route served by the bot itself.
|
|
42
28
|
//
|
|
43
29
|
// The route's URL is the installation's: the deployment profile names the
|
|
44
30
|
// bot's hostname, `planRestart` derives `https://<bot>/admin/restart` from it.
|
|
45
31
|
|
|
46
32
|
/** The operator's env var holding a `SWITCHBOARD_INGRESS_TOKENS` bearer with `deploy:write`. */
|
|
47
33
|
export const RESTART_TOKEN_ENV = "SWITCHBOARD_DEPLOY_TOKEN";
|
|
48
|
-
/**
|
|
34
|
+
/** Worker var naming the authenticated ingress subject allowed to restart. It
|
|
35
|
+
* is rendered from the deployment profile and never forwarded into the
|
|
36
|
+
* container, so a broken runtime config cannot lock out its own recovery. */
|
|
37
|
+
export const RESTART_DEPLOYER_ENV = "SWITCHBOARD_RESTART_DEPLOYER";
|
|
38
|
+
/** The scope the command declares. The Worker-side restart grant above is the
|
|
39
|
+
* independent recovery authorization for the route. */
|
|
49
40
|
export const RESTART_SCOPE = "deploy:write";
|
|
50
41
|
|
|
51
42
|
// ---- refusal decision (Worker side; the CLI relies on the 409) -----------------------------
|
|
@@ -122,6 +113,25 @@ export function constantTimeEqual(a: string, b: string): boolean {
|
|
|
122
113
|
return diff === 0;
|
|
123
114
|
}
|
|
124
115
|
|
|
116
|
+
/** Authorize the already-authenticated ingress subject against the deployment
|
|
117
|
+
* profile's Worker var. This check deliberately has no ConfigStore input: the
|
|
118
|
+
* route exists to recover a missing or invalid runtime config. */
|
|
119
|
+
export function authorizeRestartDeployer(subject: string, configured: string | undefined): RestartAuth {
|
|
120
|
+
if (!configured)
|
|
121
|
+
return {
|
|
122
|
+
ok: false,
|
|
123
|
+
status: 503,
|
|
124
|
+
reason: `restart disabled: ${RESTART_DEPLOYER_ENV} is not set by the deployment profile or Worker env`,
|
|
125
|
+
};
|
|
126
|
+
if (!constantTimeEqual(subject, configured))
|
|
127
|
+
return {
|
|
128
|
+
ok: false,
|
|
129
|
+
status: 403,
|
|
130
|
+
reason: `forbidden: identity "${subject}" is not the restart deployer named by ${RESTART_DEPLOYER_ENV}`,
|
|
131
|
+
};
|
|
132
|
+
return { ok: true, subject };
|
|
133
|
+
}
|
|
134
|
+
|
|
125
135
|
/** Find the presented bearer among the configured tokens by comparing against
|
|
126
136
|
* EVERY entry (fixed work; a plain object-key lookup would let a probe learn
|
|
127
137
|
* which prefixes exist from timing). Returns the matched identity, if any. */
|
|
@@ -134,14 +144,14 @@ export function lookupConstantTime<T>(tokens: Record<string, T>, presented: stri
|
|
|
134
144
|
export type RestartAuth = { ok: true; subject: string } | { ok: false; status: 401 | 403 | 503; reason: string };
|
|
135
145
|
export type RestartAuthn = { ok: true; identity: IngressIdentity } | { ok: false; status: 401 | 503; reason: string };
|
|
136
146
|
|
|
137
|
-
/**
|
|
138
|
-
*
|
|
147
|
+
/** Compatibility contract for a bot-served authorization probe. The external
|
|
148
|
+
* restart route no longer calls it; keeping the pure shape avoids breaking an
|
|
149
|
+
* older shim while a fleet rolls between versions. */
|
|
139
150
|
export const RESTART_AUTHORIZE_PATH = "/admin/restart/authorize";
|
|
140
|
-
/** The header the Worker sets on its internal authorize call, naming the subject it
|
|
141
|
-
* authenticated from its own token map; stripped from every proxied request. */
|
|
142
151
|
export const RESTART_SUBJECT_HEADER = "x-switchboard-restart-subject";
|
|
143
152
|
|
|
144
|
-
/** WHETHER
|
|
153
|
+
/** WHETHER for the bot's own runtime-granted admin routes (`/admin/crash`).
|
|
154
|
+
* The Worker's restart route deliberately does not call this path. */
|
|
145
155
|
export function authorizeRestartSubject(subject: string, grantsFor: GrantsLookup): RestartAuth {
|
|
146
156
|
if (subject.trim() === "") return { ok: false, status: 401, reason: "unauthorized: an empty subject" };
|
|
147
157
|
if (!hasAction(grantsFor(`http:${subject}`).actions, RESTART_SCOPE))
|
|
@@ -153,8 +163,7 @@ export function authorizeRestartSubject(subject: string, grantsFor: GrantsLookup
|
|
|
153
163
|
return { ok: true, subject };
|
|
154
164
|
}
|
|
155
165
|
|
|
156
|
-
/**
|
|
157
|
-
* every route it does not answer itself, so a caller can never assert a subject. */
|
|
166
|
+
/** Strip the compatibility probe's trusted header from ordinary proxying. */
|
|
158
167
|
export function stripRestartSubject(request: Request): Request {
|
|
159
168
|
if (!request.headers.has(RESTART_SUBJECT_HEADER)) return request;
|
|
160
169
|
const headers = new Headers(request.headers);
|
|
@@ -232,11 +241,8 @@ export function authorizeIngressBearer(
|
|
|
232
241
|
return { ok: true, subject: identity.subject };
|
|
233
242
|
}
|
|
234
243
|
|
|
235
|
-
/**
|
|
236
|
-
*
|
|
237
|
-
* relayed as they are; anything else (a bot without the route, a non-JSON body,
|
|
238
|
-
* an unexpected status) → 503, fail-closed — the Worker never restarts on an
|
|
239
|
-
* answer it cannot read. */
|
|
244
|
+
/** Parse the compatibility probe's answer. New restart routes authorize on the
|
|
245
|
+
* Worker without making this request. */
|
|
240
246
|
export function parseRestartAuthorization(status: number, text: string): RestartAuth {
|
|
241
247
|
let parsed: unknown;
|
|
242
248
|
try {
|
|
@@ -358,6 +364,9 @@ export interface RestartPlan {
|
|
|
358
364
|
target: "bot";
|
|
359
365
|
adminUrl: string;
|
|
360
366
|
healthUrl: string;
|
|
367
|
+
/** The source and durable base document this restart must prove equal before
|
|
368
|
+
* it stops anything. `generation` is the document version read at runtime. */
|
|
369
|
+
config: { source: string; document: "base"; stateWorkerUrl?: string };
|
|
361
370
|
tokenEnv: string;
|
|
362
371
|
force: boolean;
|
|
363
372
|
/** Budget for waiting out a 409 (runs in flight) before giving up. */
|
|
@@ -373,6 +382,11 @@ export function planRestart(opts: RestartOptions, profile: DeploymentProfile): R
|
|
|
373
382
|
target: opts.only,
|
|
374
383
|
adminUrl: urls.botAdminRestartUrl,
|
|
375
384
|
healthUrl: `${urls.publicBaseUrl}/healthz`,
|
|
385
|
+
config: {
|
|
386
|
+
source: profile.configSource,
|
|
387
|
+
document: "base",
|
|
388
|
+
...(urls.stateWorkerUrl !== undefined ? { stateWorkerUrl: urls.stateWorkerUrl } : {}),
|
|
389
|
+
},
|
|
376
390
|
tokenEnv: RESTART_TOKEN_ENV,
|
|
377
391
|
force: opts.force,
|
|
378
392
|
waitMaxMs: opts.waitMaxMinutes * 60_000,
|
|
@@ -386,7 +400,7 @@ export function formatRestartPlan(plan: RestartPlan): string {
|
|
|
386
400
|
? `preflight FORCED — in-flight runs are drained (SIGTERM), killed only at the drain deadline`
|
|
387
401
|
: `refused while runs are in flight or draining (409) — retry every ${plan.pollMs / 1000}s up to ${plan.waitMaxMs / 60_000} min`;
|
|
388
402
|
return [
|
|
389
|
-
`Restart ${plan.target}: POST ${plan.adminUrl} (bearer from $${plan.tokenEnv}, needs ${RESTART_SCOPE}) — ${gate}`,
|
|
403
|
+
`Restart ${plan.target}: prove ${plan.config.source} matches document "${plan.config.document}" on ${plan.config.stateWorkerUrl ?? "the missing state Worker"}, then POST ${plan.adminUrl} (bearer from $${plan.tokenEnv}, needs ${RESTART_SCOPE}) — ${gate}`,
|
|
390
404
|
` then wait until ${plan.healthUrl} answers not draining with a later startedAt (up to ${Math.round(plan.liveDeadlineMs / 60_000)} min: drain + cold start)`,
|
|
391
405
|
` no image build: the container restarts on the same build with the Worker's CURRENT secrets`,
|
|
392
406
|
].join("\n");
|