@coreplane/switchboard 1.256.0 → 1.258.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/config/config.example.yaml +14 -0
- package/dist/assets/deploy/cloudflare-memory/worker.ts +340 -7
- package/dist/assets/deploy/cloudflare-resident/drain.ts +49 -4
- package/dist/assets/deploy/cloudflare-resident/worker.ts +52 -4
- package/dist/assets/package-lock.json +3 -3
- package/dist/assets/package.json +5 -3
- package/dist/assets/project.json +17 -9
- package/dist/assets/source.json +3 -3
- package/dist/assets/src/agents/registry.ts +65 -2
- package/dist/assets/src/core/authz/grants.ts +12 -0
- package/dist/assets/src/core/authz/policy.ts +15 -0
- package/dist/assets/src/core/budgets.ts +66 -18
- package/dist/assets/src/core/coordinator/contract.ts +45 -5
- package/dist/assets/src/core/coordinator/driver.ts +6 -3
- package/dist/assets/src/core/delivery.ts +8 -5
- package/dist/assets/src/core/pipelineStanding.ts +2 -0
- package/dist/assets/src/core/plane/decide.ts +477 -12
- package/dist/assets/src/core/plane/findings.ts +120 -0
- package/dist/assets/src/core/runEvents.ts +54 -14
- package/dist/assets/src/core/runFriction.ts +16 -0
- package/dist/assets/src/core/runLedger/types.ts +16 -3
- package/dist/assets/src/core/runRecord.ts +12 -0
- package/dist/assets/src/core/ship/contract.ts +14 -2
- package/dist/assets/src/core/ship/coordinator.ts +179 -51
- package/dist/assets/src/core/ship/renewal.ts +23 -20
- package/dist/assets/src/core/trace/attrs.ts +1 -1
- package/dist/assets/src/core/untrusted.ts +35 -0
- package/dist/assets/src/execution/residentCredentials.ts +7 -4
- package/dist/assets/web/dist/.vite/manifest.json +97 -72
- package/dist/assets/web/dist/assets/{DeliveryPage-BBAyLwPq.js → DeliveryPage-DadntjSp.js} +1 -1
- package/dist/assets/web/dist/assets/HomePage-BEEpTnVT.js +1 -0
- package/dist/assets/web/dist/assets/{PendingTurnRow-DGINv9XT.js → PendingTurnRow-DBPMLVsm.js} +1 -1
- package/dist/assets/web/dist/assets/PlanePage-DLusMGQ0.js +1 -0
- package/dist/assets/web/dist/assets/{ResidentDetailPage-jYhEeeyu.js → ResidentDetailPage-CPnHnrxy.js} +1 -1
- package/dist/assets/web/dist/assets/{ResidentsIndexPage-DvFmlV4U.js → ResidentsIndexPage-C9hASwHL.js} +1 -1
- package/dist/assets/web/dist/assets/RunFoldRow-OPe7Z0Cz.js +1 -0
- package/dist/assets/web/dist/assets/{RunRoutePage-CvZ-TOT3.js → RunRoutePage-o1xzkTGZ.js} +4 -4
- package/dist/assets/web/dist/assets/RunsIndexPage-0YAClYQh.js +1 -0
- package/dist/assets/web/dist/assets/{ScheduledPage-Viim-bus.js → ScheduledPage-BIdfC0V3.js} +1 -1
- package/dist/assets/web/dist/assets/{SettingsPage-CYUy8McC.js → SettingsPage-CnqWAjl3.js} +1 -1
- package/dist/assets/web/dist/assets/SilentTurn-DGNg0sIZ.js +2 -0
- package/dist/assets/web/dist/assets/SlackMark-CtiHrNjr.js +1 -0
- package/dist/assets/web/dist/assets/{StatusDot-Dv6UMaPy.js → StatusDot-D_D_iJS8.js} +1 -1
- package/dist/assets/web/dist/assets/{Tooltip-Bd-5Rypv.js → Tooltip-DVFmM7on.js} +1 -1
- package/dist/assets/web/dist/assets/UnitRoutePage-7fAXFoB_.js +1 -0
- package/dist/assets/web/dist/assets/budgets-BRLm6oID.js +1 -0
- package/dist/assets/web/dist/assets/{dist-Nl3uaxrP.js → dist-C1VX-_mm.js} +1 -1
- package/dist/assets/web/dist/assets/durationTone-CVpX_yIk.js +1 -0
- package/dist/assets/web/dist/assets/{indexRow-DborJPFp.js → indexRow-BO_GIdF6.js} +1 -1
- package/dist/assets/web/dist/assets/main-Cl7xHG34.js +28 -0
- package/dist/assets/web/dist/assets/main-DGC6WFqS.css +1 -0
- package/dist/assets/web/dist/assets/{sseReplay-DE6wv1Ua.js → sseReplay-BjlKmQsY.js} +1 -1
- package/dist/cli.js +3096 -1136
- package/package.json +1 -1
- package/dist/assets/web/dist/assets/HomePage-BEiStFwW.js +0 -3
- package/dist/assets/web/dist/assets/PlanePage-JEj-lqgz.js +0 -1
- package/dist/assets/web/dist/assets/RunFoldRow-DvWzR1JQ.js +0 -1
- package/dist/assets/web/dist/assets/RunsIndexPage-JqVSDOok.js +0 -1
- package/dist/assets/web/dist/assets/UnitRoutePage-B81wEnKN.js +0 -1
- package/dist/assets/web/dist/assets/budgets-c1eumrqD.js +0 -1
- package/dist/assets/web/dist/assets/durationTone-U_rOqo6r.js +0 -1
- package/dist/assets/web/dist/assets/main-C4QfwybO.css +0 -1
- package/dist/assets/web/dist/assets/main-DRxWlffc.js +0 -28
|
@@ -132,6 +132,20 @@ defaults:
|
|
|
132
132
|
# maxIdentity: write
|
|
133
133
|
# machines: [none, blank, repo-cold, repo-resident]
|
|
134
134
|
# confirm: write
|
|
135
|
+
# Watch until merge (docs/reference/specs/routing-and-config.md item 32;
|
|
136
|
+
# agent-ship.md item 21): keep a merge-ready pipeline unit on its pull
|
|
137
|
+
# request until the pull request merges — each push to its base that leaves
|
|
138
|
+
# it conflicting buys one rebase of the sweep's resolver. OFF by default.
|
|
139
|
+
# `rebaseInFlight` caps the watch's rebases per repository (default 1);
|
|
140
|
+
# `spendLimitUsd` caps one pull request's model rounds — read in whole
|
|
141
|
+
# rounds today, so the default buys one rebase round per pull request and
|
|
142
|
+
# a finer value has no extra effect yet. This block is the
|
|
143
|
+
# org tier; `config set org --pulls.…` layers over it at run time, and
|
|
144
|
+
# `config set repo --repo <owner/name> --pulls.…` overrides per repository.
|
|
145
|
+
# pulls:
|
|
146
|
+
# watch: false
|
|
147
|
+
# rebaseInFlight: 1
|
|
148
|
+
# spendLimitUsd: 5
|
|
135
149
|
|
|
136
150
|
# PR reviews and the reading diff (docs/reference/specs/reading-diff.md). Every
|
|
137
151
|
# review records its full `git diff`; the ABRIDGED version (meat.dev, run on
|
|
@@ -79,11 +79,16 @@ import {
|
|
|
79
79
|
} from "../../src/core/runLedger/decisions.ts";
|
|
80
80
|
import { intakeReceiptRetentionMs, minutesToMs, PLANE } from "../../src/core/budgets.ts";
|
|
81
81
|
import {
|
|
82
|
+
causeOfClose,
|
|
83
|
+
causeOfReclaim,
|
|
82
84
|
decide,
|
|
83
85
|
effectCapRefusal,
|
|
84
86
|
planeAskAnswerOf,
|
|
85
87
|
planeAskWordOf,
|
|
86
88
|
RESIDENT_DRAIN_WINDOW,
|
|
89
|
+
type PlaneEndingCause,
|
|
90
|
+
type PlaneReclaimWord,
|
|
91
|
+
type HeartbeatFacts,
|
|
87
92
|
type PlaneAskAnswer,
|
|
88
93
|
type PlaneLevelRow,
|
|
89
94
|
type PlaneAckOutcome,
|
|
@@ -96,6 +101,7 @@ import {
|
|
|
96
101
|
type PlaneState,
|
|
97
102
|
type PlaneWrite,
|
|
98
103
|
} from "../../src/core/plane/decide.ts";
|
|
104
|
+
import { mergePlaneFindings, planeFindingKey, type PlaneFinding } from "../../src/core/plane/findings.ts";
|
|
99
105
|
import {
|
|
100
106
|
IDEMPOTENCY_KEY_PATTERN,
|
|
101
107
|
capThreadEvent,
|
|
@@ -103,6 +109,7 @@ import {
|
|
|
103
109
|
isCoordinatorInstance,
|
|
104
110
|
isCoordinatorUnit,
|
|
105
111
|
isThreadEvent,
|
|
112
|
+
sendChildSignal,
|
|
106
113
|
sendRunFinished,
|
|
107
114
|
UNIT_PATTERN,
|
|
108
115
|
type CoordinatorInstance,
|
|
@@ -1777,6 +1784,12 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
1777
1784
|
generation TEXT NOT NULL,
|
|
1778
1785
|
PRIMARY KEY (resident, name)
|
|
1779
1786
|
);
|
|
1787
|
+
CREATE TABLE IF NOT EXISTS plane_endings (
|
|
1788
|
+
run_id TEXT PRIMARY KEY,
|
|
1789
|
+
kind TEXT NOT NULL,
|
|
1790
|
+
cause TEXT NOT NULL,
|
|
1791
|
+
at INTEGER NOT NULL
|
|
1792
|
+
);
|
|
1780
1793
|
`);
|
|
1781
1794
|
}
|
|
1782
1795
|
|
|
@@ -1793,13 +1806,20 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
1793
1806
|
// a dispatch that died between the admission answer and its claim must not
|
|
1794
1807
|
// hold the thread forever. Promotion deletes the row (the live row holds
|
|
1795
1808
|
// the thread from there), so age alone is the test.
|
|
1809
|
+
// `steer` and `park` rows (record 0064) have no expiry window: a steer's
|
|
1810
|
+
// dedupe row and a parked run's wait live until the run's seal deletes them.
|
|
1796
1811
|
const reservations = this.sql
|
|
1797
1812
|
.exec<{ kind: string; key: string; run_id: string; at: number }>(
|
|
1798
|
-
`SELECT * FROM plane_reservations WHERE kind
|
|
1813
|
+
`SELECT * FROM plane_reservations WHERE kind != 'thread' OR at > ?`,
|
|
1799
1814
|
now - minutesToMs(PLANE.reservationMinutes),
|
|
1800
1815
|
)
|
|
1801
1816
|
.toArray()
|
|
1802
|
-
.map((r): PlaneReservation => ({
|
|
1817
|
+
.map((r): PlaneReservation => ({
|
|
1818
|
+
kind: r.kind as PlaneReservation["kind"],
|
|
1819
|
+
key: r.key,
|
|
1820
|
+
runId: r.run_id,
|
|
1821
|
+
at: r.at,
|
|
1822
|
+
}));
|
|
1803
1823
|
const openWindows = this.sql
|
|
1804
1824
|
.exec<{ kind: string }>(`SELECT kind FROM plane_windows WHERE phase = 'open'`)
|
|
1805
1825
|
.toArray()
|
|
@@ -1877,7 +1897,22 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
1877
1897
|
w.row.at,
|
|
1878
1898
|
);
|
|
1879
1899
|
} else if (w.table === "plane_reservations" && w.op === "del") {
|
|
1880
|
-
this.sql.exec(`DELETE FROM plane_reservations WHERE kind =
|
|
1900
|
+
this.sql.exec(`DELETE FROM plane_reservations WHERE kind = ? AND key = ?`, w.kind ?? "thread", w.key);
|
|
1901
|
+
} else if (w.table === "run_inbox" && w.op === "push") {
|
|
1902
|
+
// The plane's steer into a live run's durable inbox (record 0064;
|
|
1903
|
+
// run-history item 40), in the decider's own transaction: the run reads
|
|
1904
|
+
// it at its next boundary like any follow-up; a run with no live row
|
|
1905
|
+
// reads nothing and the row would be an orphan, so it is skipped.
|
|
1906
|
+
if (!this.liveRow(w.runId)) continue;
|
|
1907
|
+
const last = this.sql
|
|
1908
|
+
.exec<{ m: number | null }>(`SELECT MAX(seq) AS m FROM run_inbox WHERE run_id = ?`, w.runId)
|
|
1909
|
+
.one().m;
|
|
1910
|
+
this.sql.exec(
|
|
1911
|
+
`INSERT INTO run_inbox (run_id, seq, json) VALUES (?, ?, ?)`,
|
|
1912
|
+
w.runId,
|
|
1913
|
+
(last ?? 0) + 1,
|
|
1914
|
+
JSON.stringify(w.message),
|
|
1915
|
+
);
|
|
1881
1916
|
} else if (w.table === "plane_windows" && w.op === "put") {
|
|
1882
1917
|
this.sql.exec(
|
|
1883
1918
|
`INSERT OR REPLACE INTO plane_windows (kind, key, phase, opened_at, reason_json) VALUES (?, ?, 'open', ?, '{}')`,
|
|
@@ -1896,6 +1931,38 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
1896
1931
|
w.row.reportedAt,
|
|
1897
1932
|
w.row.generation,
|
|
1898
1933
|
);
|
|
1934
|
+
} else if (w.table === "plane_findings") {
|
|
1935
|
+
// A finding (record 0064, "Endings and the watches"): keyed by watch
|
|
1936
|
+
// and subject, so two findings on one subject are ONE row — an
|
|
1937
|
+
// existing row absorbs the timeline through the same merge the
|
|
1938
|
+
// decider's module defines, its first-seen stamp kept.
|
|
1939
|
+
const key = planeFindingKey(w.finding);
|
|
1940
|
+
const prior = this.sql
|
|
1941
|
+
.exec<{ timeline_json: string; filed_at: number }>(
|
|
1942
|
+
`SELECT timeline_json, filed_at FROM plane_findings WHERE id = ?`,
|
|
1943
|
+
key,
|
|
1944
|
+
)
|
|
1945
|
+
.toArray()[0];
|
|
1946
|
+
const existing = prior
|
|
1947
|
+
? [
|
|
1948
|
+
{
|
|
1949
|
+
watch: w.finding.watch,
|
|
1950
|
+
subject: w.finding.subject,
|
|
1951
|
+
timeline: JSON.parse(prior.timeline_json) as PlaneFinding["timeline"],
|
|
1952
|
+
firstAt: Number(prior.filed_at),
|
|
1953
|
+
lastAt: w.finding.lastAt,
|
|
1954
|
+
},
|
|
1955
|
+
]
|
|
1956
|
+
: [];
|
|
1957
|
+
const merged = mergePlaneFindings(existing, w.finding)[0]!;
|
|
1958
|
+
this.sql.exec(
|
|
1959
|
+
`INSERT OR REPLACE INTO plane_findings (id, watch, subject, timeline_json, filed_at) VALUES (?, ?, ?, ?, ?)`,
|
|
1960
|
+
key,
|
|
1961
|
+
merged.watch,
|
|
1962
|
+
merged.subject,
|
|
1963
|
+
JSON.stringify(merged.timeline),
|
|
1964
|
+
merged.firstAt,
|
|
1965
|
+
);
|
|
1899
1966
|
} else {
|
|
1900
1967
|
// The effect bounds (record 0064): an offer past the per-run or total
|
|
1901
1968
|
// cap is refused by the cap's name — the throw aborts the transaction,
|
|
@@ -2016,9 +2083,20 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
2016
2083
|
* drain posts land as the resident-drain window's open (`above`) and lift
|
|
2017
2084
|
* (`below` — a `cleared` or the alarm's `expired`). */
|
|
2018
2085
|
planeLevel(
|
|
2019
|
-
post:
|
|
2086
|
+
post:
|
|
2087
|
+
| { resident: string; name: "seat" | "memory" | "drain"; side: "below" | "above"; generation: string }
|
|
2088
|
+
| { provider: string; name: "provider"; side: "up" | "down" },
|
|
2020
2089
|
now: number,
|
|
2021
2090
|
): { admitted: number } {
|
|
2091
|
+
// The model proxy's provider level (record 0064): `up` re-issues every
|
|
2092
|
+
// held turn parked on the provider — the steers land as inbox writes in the
|
|
2093
|
+
// decider's transaction — and walks anything queued on `provider_up`.
|
|
2094
|
+
if (post.name === "provider") {
|
|
2095
|
+
const r = this.planeApply({ kind: "provider_level", at: now, provider: post.provider, level: post.side });
|
|
2096
|
+
const admitted = r.effects.filter((e) => e.kind === "admit").length;
|
|
2097
|
+
console.log(`[plane/level] provider ${post.provider} ${post.side} — ${admitted} admission(s)`);
|
|
2098
|
+
return { admitted };
|
|
2099
|
+
}
|
|
2022
2100
|
const r =
|
|
2023
2101
|
post.name === "drain"
|
|
2024
2102
|
? this.planeApply({
|
|
@@ -2040,6 +2118,20 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
2040
2118
|
return { admitted };
|
|
2041
2119
|
}
|
|
2042
2120
|
|
|
2121
|
+
/** A run parked on its provider (`POST /plane/park`, record 0064): the
|
|
2122
|
+
* proxy could not complete the turn after its retry; the harness holds the
|
|
2123
|
+
* turn and the provider's next `up` steers the run to re-issue it. */
|
|
2124
|
+
planePark(runId: string, provider: string, now: number): { parked: boolean } {
|
|
2125
|
+
let parked = false;
|
|
2126
|
+
this.ctx.storage.transactionSync(() => {
|
|
2127
|
+
const decision = decide(this.planeState(), { kind: "park", at: now, runId, provider });
|
|
2128
|
+
this.applyPlaneWrites(decision.writes);
|
|
2129
|
+
parked = decision.writes.length > 0;
|
|
2130
|
+
});
|
|
2131
|
+
console.log(`[plane/park] run ${runId} on ${provider} — ${parked ? "parked" : "already parked"}`);
|
|
2132
|
+
return { parked };
|
|
2133
|
+
}
|
|
2134
|
+
|
|
2043
2135
|
/** A refusal-by-name the bot met at attach or exec (`POST /plane/observe`,
|
|
2044
2136
|
* record 0064): an admitted run re-enters the queue at its old position. */
|
|
2045
2137
|
planeObserve(post: { runId: string; resident: string; refusal: string }, now: number): { reentered: boolean } {
|
|
@@ -2084,6 +2176,54 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
2084
2176
|
if (set === null || set > due) await this.ctx.storage.setAlarm(due);
|
|
2085
2177
|
}
|
|
2086
2178
|
|
|
2179
|
+
/** The earliest instant the plane must wake at (record 0064): the
|
|
2180
|
+
* earliest hosting deadline, lease end or re-ask across its rows. A due
|
|
2181
|
+
* already past re-offers at the re-ask cadence, never in a hot loop, and
|
|
2182
|
+
* contributes nothing to end a run — the alarm only offers. */
|
|
2183
|
+
private planeEarliestDue(now: number): number | undefined {
|
|
2184
|
+
const dues: number[] = [];
|
|
2185
|
+
const reoffer = now + this.planeReaskMs();
|
|
2186
|
+
for (const r of this.sql
|
|
2187
|
+
.exec<{ lease_until: number; state_json: string }>(`SELECT lease_until, state_json FROM live_runs`)
|
|
2188
|
+
.toArray()) {
|
|
2189
|
+
dues.push(r.lease_until > now ? r.lease_until : reoffer);
|
|
2190
|
+
try {
|
|
2191
|
+
const hosting = (JSON.parse(r.state_json) as Record<string, unknown>).hosting as { until?: unknown };
|
|
2192
|
+
if (typeof hosting?.until === "number") dues.push(hosting.until > now ? hosting.until : reoffer);
|
|
2193
|
+
} catch {
|
|
2194
|
+
// A malformed state contributes no deadline.
|
|
2195
|
+
}
|
|
2196
|
+
}
|
|
2197
|
+
if (this.planeWaitsOnResident()) dues.push(reoffer);
|
|
2198
|
+
return dues.length === 0 ? undefined : Math.min(...dues);
|
|
2199
|
+
}
|
|
2200
|
+
|
|
2201
|
+
/** The plane's one alarm (record 0064): set to the earliest due across
|
|
2202
|
+
* its rows — the owner's heartbeat, moving the lease end, moves an alarm
|
|
2203
|
+
* the plane armed; an alarm someone else armed earlier is left to fire
|
|
2204
|
+
* first (the handler re-arms). The armed slot is judged directly, never
|
|
2205
|
+
* the meta row alone: the slot is shared with the re-ask and the sweep and
|
|
2206
|
+
* a fired alarm is consumed, so a meta row equal to a static due (a
|
|
2207
|
+
* hosting deadline) can claim a wake that no longer exists. The due is
|
|
2208
|
+
* capped to the sweep interval so the retention sweep never starves behind
|
|
2209
|
+
* a distant hosting deadline. */
|
|
2210
|
+
private async ensurePlaneAlarm(now: number): Promise<void> {
|
|
2211
|
+
const raw = this.planeEarliestDue(now);
|
|
2212
|
+
const prior = this.sql.exec<{ value: string }>(`SELECT value FROM meta WHERE key = 'plane_alarm_at'`).toArray()[0];
|
|
2213
|
+
const priorAt = prior ? Number(prior.value) : undefined;
|
|
2214
|
+
if (raw === undefined) {
|
|
2215
|
+
if (prior) this.sql.exec(`DELETE FROM meta WHERE key = 'plane_alarm_at'`);
|
|
2216
|
+
return; // nothing waits: the sweep's own arming stands
|
|
2217
|
+
}
|
|
2218
|
+
const due = Math.min(raw, now + RUN_SWEEP_INTERVAL_MS);
|
|
2219
|
+
const set = await this.ctx.storage.getAlarm();
|
|
2220
|
+
// Re-arm when the slot is empty, later than the due, or holds an alarm
|
|
2221
|
+
// this plane armed itself; an earlier foreign alarm fires first.
|
|
2222
|
+
if (set !== due && (set === null || set > due || set === priorAt)) await this.ctx.storage.setAlarm(due);
|
|
2223
|
+
if (due !== priorAt)
|
|
2224
|
+
this.sql.exec(`INSERT OR REPLACE INTO meta (key, value) VALUES ('plane_alarm_at', ?)`, String(due));
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2087
2227
|
/** A window's open or lift over the RPC seam (`/plane/deploy`; a later
|
|
2088
2228
|
* unit's `plane window lift`): kind `deploy` is the pending deploy. */
|
|
2089
2229
|
planeWindow(window: string, phase: "opened" | "lifted", now: number): { admitted: number } {
|
|
@@ -2108,6 +2248,78 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
2108
2248
|
return this.planeState().queue.find((r) => r.runId === runId) ?? null;
|
|
2109
2249
|
}
|
|
2110
2250
|
|
|
2251
|
+
/** The ending's cause (record 0064, "Endings and the watches"): one
|
|
2252
|
+
* `ended { kind, cause }` per closed row, recorded only when a live row
|
|
2253
|
+
* closes and first-writer-wins — a roll that resumes every row assigns
|
|
2254
|
+
* nothing. One keyed exception, taken only by the finish (`supersedes`): a
|
|
2255
|
+
* standing `resident_replaced` was a `restarting` close — the run carried
|
|
2256
|
+
* on under its own id (run-history item 42's restart) — so that run's own
|
|
2257
|
+
* later finish replaces it and the ending agrees with the record. The bot
|
|
2258
|
+
* renders the word; the object never renders. */
|
|
2259
|
+
private recordPlaneEnding(
|
|
2260
|
+
runId: string,
|
|
2261
|
+
kind: string,
|
|
2262
|
+
cause: PlaneEndingCause,
|
|
2263
|
+
at: number,
|
|
2264
|
+
supersedes = false,
|
|
2265
|
+
): PlaneEndingCause {
|
|
2266
|
+
this.sql.exec(
|
|
2267
|
+
`INSERT OR IGNORE INTO plane_endings (run_id, kind, cause, at) VALUES (?, ?, ?, ?)`,
|
|
2268
|
+
runId,
|
|
2269
|
+
kind,
|
|
2270
|
+
cause,
|
|
2271
|
+
at,
|
|
2272
|
+
);
|
|
2273
|
+
const standing = this.sql
|
|
2274
|
+
.exec<{ cause: string }>(`SELECT cause FROM plane_endings WHERE run_id = ?`, runId)
|
|
2275
|
+
.toArray()[0];
|
|
2276
|
+
if (
|
|
2277
|
+
supersedes &&
|
|
2278
|
+
standing !== undefined &&
|
|
2279
|
+
standing.cause === "resident_replaced" &&
|
|
2280
|
+
cause !== "resident_replaced"
|
|
2281
|
+
) {
|
|
2282
|
+
this.sql.exec(`UPDATE plane_endings SET kind = ?, cause = ?, at = ? WHERE run_id = ?`, kind, cause, at, runId);
|
|
2283
|
+
return cause;
|
|
2284
|
+
}
|
|
2285
|
+
return (standing?.cause as PlaneEndingCause | undefined) ?? cause;
|
|
2286
|
+
}
|
|
2287
|
+
|
|
2288
|
+
/** The reclaim's outcome per row (record 0064; run-history item 36): only a
|
|
2289
|
+
* `closed` row records an ending — `lease_lapsed`, which is true and blames
|
|
2290
|
+
* nobody — and the standing cause is answered back so the bot's interrupted
|
|
2291
|
+
* note renders the plane's word. `resume`, `restart` and `rehost` record
|
|
2292
|
+
* nothing: the run carries on. */
|
|
2293
|
+
planeReclaimed(
|
|
2294
|
+
outcomes: readonly { runId: string; outcome: PlaneReclaimWord }[],
|
|
2295
|
+
now: number,
|
|
2296
|
+
): { recorded: { runId: string; cause: PlaneEndingCause }[] } {
|
|
2297
|
+
const recorded: { runId: string; cause: PlaneEndingCause }[] = [];
|
|
2298
|
+
this.ctx.storage.transactionSync(() => {
|
|
2299
|
+
for (const o of outcomes) {
|
|
2300
|
+
const cause = causeOfReclaim(o.outcome);
|
|
2301
|
+
if (cause === undefined) continue;
|
|
2302
|
+
recorded.push({ runId: o.runId, cause: this.recordPlaneEnding(o.runId, "interrupted", cause, now) });
|
|
2303
|
+
}
|
|
2304
|
+
});
|
|
2305
|
+
if (outcomes.length > 0)
|
|
2306
|
+
console.log(
|
|
2307
|
+
`[plane/reclaimed] ${outcomes.map((o) => `${o.runId}=${o.outcome}`).join(", ")} — ${recorded.length} ending(s) recorded`,
|
|
2308
|
+
);
|
|
2309
|
+
return { recorded };
|
|
2310
|
+
}
|
|
2311
|
+
|
|
2312
|
+
/** One recorded ending (record 0064), or none: what a reader renders. */
|
|
2313
|
+
planeEndingOf(runId: string): { kind: string; cause: PlaneEndingCause; at: number } | null {
|
|
2314
|
+
const row = this.sql
|
|
2315
|
+
.exec<{ kind: string; cause: string; at: number }>(
|
|
2316
|
+
`SELECT kind, cause, at FROM plane_endings WHERE run_id = ?`,
|
|
2317
|
+
runId,
|
|
2318
|
+
)
|
|
2319
|
+
.toArray()[0];
|
|
2320
|
+
return row ? { kind: row.kind, cause: row.cause as PlaneEndingCause, at: row.at } : null;
|
|
2321
|
+
}
|
|
2322
|
+
|
|
2111
2323
|
/** The seal (record 0064, "The queue"): the run's own open effects are
|
|
2112
2324
|
* dropped — an admit for a run that just ended is stale — then the sealed
|
|
2113
2325
|
* event frees the thread and walks the queue, all in one transaction. */
|
|
@@ -2118,6 +2330,9 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
2118
2330
|
// Exact id matching (`admit:<runId>`), never LIKE: a bot-minted run id
|
|
2119
2331
|
// can carry `%` or `_`, which a pattern would read as wildcards.
|
|
2120
2332
|
this.sql.exec(`DELETE FROM plane_effects WHERE acked_at IS NULL AND id = 'admit:' || ?`, runId);
|
|
2333
|
+
// The run's steer dedupe rows and any park go with it (record 0064):
|
|
2334
|
+
// a sealed run holds no turn and reads no steer.
|
|
2335
|
+
this.sql.exec(`DELETE FROM plane_reservations WHERE kind IN ('steer', 'park') AND run_id = ?`, runId);
|
|
2121
2336
|
const decision = decide(this.planeState(), { kind: "sealed", at: now, threadKey });
|
|
2122
2337
|
this.applyPlaneWrites(decision.writes);
|
|
2123
2338
|
effects = decision.effects;
|
|
@@ -2527,11 +2742,35 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
2527
2742
|
JSON.stringify(req.state ?? {}),
|
|
2528
2743
|
);
|
|
2529
2744
|
});
|
|
2745
|
+
if (out.ok) {
|
|
2746
|
+
// A `restartOf` claim under a coordinator (record 0064): the plane
|
|
2747
|
+
// tells the waiting parent the child resumed — best effort, beside the
|
|
2748
|
+
// bot's own announcement; a duplicate is consumed and re-armed, harmless.
|
|
2749
|
+
if (req.meta.restartOf !== undefined && req.meta.parentInstanceId !== undefined) {
|
|
2750
|
+
const sent = await sendChildSignal(this.env.SHIP_COORDINATOR, {
|
|
2751
|
+
runId: req.runId,
|
|
2752
|
+
parentInstanceId: req.meta.parentInstanceId,
|
|
2753
|
+
kind: "resumed",
|
|
2754
|
+
reason: `restarted from run ${req.meta.restartOf}`,
|
|
2755
|
+
at: now,
|
|
2756
|
+
});
|
|
2757
|
+
if (sent.kind === "failed")
|
|
2758
|
+
console.warn(`[runs/claim] ${req.runId} → ${sent.type} not delivered to ${sent.instance}: ${sent.reason}`);
|
|
2759
|
+
}
|
|
2760
|
+
// The lease end joins the plane's alarm (record 0064): armed at the earliest due.
|
|
2761
|
+
await this.ensurePlaneAlarm(now);
|
|
2762
|
+
}
|
|
2530
2763
|
return out;
|
|
2531
2764
|
}
|
|
2532
2765
|
|
|
2533
2766
|
/** Extends the lease iff the caller owns the run; answers what another generation asked for. */
|
|
2534
|
-
async heartbeat(
|
|
2767
|
+
async heartbeat(
|
|
2768
|
+
runId: string,
|
|
2769
|
+
gen: string,
|
|
2770
|
+
leaseMs: number,
|
|
2771
|
+
now: number,
|
|
2772
|
+
facts?: HeartbeatFacts,
|
|
2773
|
+
): Promise<HeartbeatAnswer> {
|
|
2535
2774
|
let out: HeartbeatAnswer = { ok: false, reason: "unknown-run" };
|
|
2536
2775
|
this.ctx.storage.transactionSync(() => {
|
|
2537
2776
|
const row = this.liveRow(runId);
|
|
@@ -2541,11 +2780,28 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
2541
2780
|
return;
|
|
2542
2781
|
}
|
|
2543
2782
|
this.sql.exec(`UPDATE live_runs SET lease_until = ? WHERE run_id = ?`, now + leaseMs, runId);
|
|
2783
|
+
// The heartbeat body (record 0064): the facts are judged for the
|
|
2784
|
+
// checkpoint steer in this same transaction, so the inbox row and its
|
|
2785
|
+
// dedupe row land with the lease or not at all. A cap throw here must
|
|
2786
|
+
// not undo the lease of a healthy run, so the decision guards itself.
|
|
2787
|
+
if (facts !== undefined) {
|
|
2788
|
+
try {
|
|
2789
|
+
const decision = decide(this.planeState(), { kind: "heartbeat", at: now, runId, facts });
|
|
2790
|
+
this.applyPlaneWrites(decision.writes);
|
|
2791
|
+
if (decision.writes.some((w) => w.table === "run_inbox"))
|
|
2792
|
+
console.log(`[plane/steer] run ${runId} round ${facts.round} — checkpoint steer written`);
|
|
2793
|
+
} catch (err) {
|
|
2794
|
+
console.warn(`[plane/steer] run ${runId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
2795
|
+
}
|
|
2796
|
+
}
|
|
2544
2797
|
// The plane's open effects ride every owner's heartbeat answer (record
|
|
2545
2798
|
// 0064; orchestration-plane item 7) — empty until a unit writes them, but always present, so the
|
|
2546
2799
|
// client's ack loop needs no version probe.
|
|
2547
2800
|
out = { ok: true, stop: row.stop, phase: row.phase, effects: this.openPlaneEffects() };
|
|
2548
2801
|
});
|
|
2802
|
+
// The owner's heartbeat refreshes the lease, so the plane's alarm moves
|
|
2803
|
+
// with it (record 0064): re-armed only when the earliest due changed.
|
|
2804
|
+
if (out.ok) await this.ensurePlaneAlarm(now);
|
|
2549
2805
|
return out;
|
|
2550
2806
|
}
|
|
2551
2807
|
|
|
@@ -2688,6 +2944,16 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
2688
2944
|
}
|
|
2689
2945
|
const put = this.upsertInTransaction(record, proposal);
|
|
2690
2946
|
this.deleteLiveRows([runId]);
|
|
2947
|
+
// The ending's cause (record 0064), recorded exactly when the live
|
|
2948
|
+
// row closes: the record's own status word, `restarting` reading as the
|
|
2949
|
+
// resident replacement the reattach path observed.
|
|
2950
|
+
this.recordPlaneEnding(
|
|
2951
|
+
runId,
|
|
2952
|
+
record.status,
|
|
2953
|
+
causeOfClose(record.status, record.restarting === true),
|
|
2954
|
+
record.finishedAt,
|
|
2955
|
+
true,
|
|
2956
|
+
);
|
|
2691
2957
|
turnedFinal = put.turnedFinal;
|
|
2692
2958
|
out = { ok: true, stored: put.stored };
|
|
2693
2959
|
});
|
|
@@ -3199,8 +3465,25 @@ export class RunHistoryDO extends DurableObject<Env> {
|
|
|
3199
3465
|
const probes = this.planeApply({ kind: "reask", at: now, cadenceMs: this.planeReaskMs() }).effects;
|
|
3200
3466
|
if (probes.length > 0) console.log(`[plane/reask] ${probes.map((e) => e.id).join(", ")}`);
|
|
3201
3467
|
}
|
|
3468
|
+
// A lease end offers the row to a generation other than its owner and
|
|
3469
|
+
// never ends a run (record 0064): the row stays exactly as it is —
|
|
3470
|
+
// any reclaim can take it now — and the open effects are re-pushed so a
|
|
3471
|
+
// listening bot sweeps sooner. The owner's next heartbeat refreshes an
|
|
3472
|
+
// unreclaimed row and moves the alarm on.
|
|
3473
|
+
const lapsed = this.sql
|
|
3474
|
+
.exec<{ run_id: string; owner_gen: string }>(
|
|
3475
|
+
`SELECT run_id, owner_gen FROM live_runs WHERE lease_until <= ?`,
|
|
3476
|
+
now,
|
|
3477
|
+
)
|
|
3478
|
+
.toArray();
|
|
3479
|
+
if (lapsed.length > 0) {
|
|
3480
|
+
console.log(
|
|
3481
|
+
`[plane/alarm] ${lapsed.length} lease(s) lapsed (${lapsed.map((l) => l.run_id).join(", ")}) — offered to any generation but the owner; nothing closed`,
|
|
3482
|
+
);
|
|
3483
|
+
this.pushPlaneEffects(this.openPlaneEffects());
|
|
3484
|
+
}
|
|
3202
3485
|
await this.ctx.storage.setAlarm(now + RUN_SWEEP_INTERVAL_MS);
|
|
3203
|
-
await this.
|
|
3486
|
+
await this.ensurePlaneAlarm(now);
|
|
3204
3487
|
root.end("ok", { swept: deleted });
|
|
3205
3488
|
} catch (err) {
|
|
3206
3489
|
root.fail(err);
|
|
@@ -4583,8 +4866,12 @@ const PLANE_ROUTES = new Set([
|
|
|
4583
4866
|
"/plane/queued",
|
|
4584
4867
|
"/plane/level",
|
|
4585
4868
|
"/plane/observe",
|
|
4869
|
+
"/plane/park",
|
|
4870
|
+
"/plane/reclaimed",
|
|
4586
4871
|
]);
|
|
4587
4872
|
|
|
4873
|
+
const PLANE_RECLAIM_WORDS = new Set(["resume", "restart", "rehost", "closed"]);
|
|
4874
|
+
|
|
4588
4875
|
const PLANE_LEVEL_NAMES = new Set(["seat", "memory", "drain"]);
|
|
4589
4876
|
const PLANE_LEVEL_SIDES = new Set(["below", "above"]);
|
|
4590
4877
|
|
|
@@ -4675,6 +4962,14 @@ async function handlePlane(pathname: string, body: unknown, env: Env): Promise<R
|
|
|
4675
4962
|
}
|
|
4676
4963
|
}
|
|
4677
4964
|
if (pathname === "/plane/level") {
|
|
4965
|
+
// The model proxy's provider level (record 0064): `up` on a relayed
|
|
4966
|
+
// success, `down` on a failure past its one retry.
|
|
4967
|
+
if (b.name === "provider") {
|
|
4968
|
+
if (typeof b.provider !== "string" || b.provider.length === 0)
|
|
4969
|
+
return json({ error: "provider must be a non-empty string" }, 400);
|
|
4970
|
+
if (b.side !== "up" && b.side !== "down") return json({ error: "side must be up or down" }, 400);
|
|
4971
|
+
return json(await stub.planeLevel({ provider: b.provider, name: "provider", side: b.side }, now));
|
|
4972
|
+
}
|
|
4678
4973
|
// A resident's level report (record 0064): forwarded by the bot from the levels a
|
|
4679
4974
|
// resident answer carried, or from the registry's drain outbox.
|
|
4680
4975
|
if (typeof b.resident !== "string" || b.resident.length === 0)
|
|
@@ -4706,11 +5001,34 @@ async function handlePlane(pathname: string, body: unknown, env: Env): Promise<R
|
|
|
4706
5001
|
return json({ error: "refusal must be a non-empty string" }, 400);
|
|
4707
5002
|
return json(await stub.planeObserve({ runId: parsed.value, resident: b.resident, refusal: b.refusal }, now));
|
|
4708
5003
|
}
|
|
5004
|
+
if (pathname === "/plane/park") {
|
|
5005
|
+
// A run parked on its provider (record 0064).
|
|
5006
|
+
const parsed = parseRunId(b.runId);
|
|
5007
|
+
if (!parsed.ok) return json({ error: parsed.error }, 400);
|
|
5008
|
+
if (typeof b.provider !== "string" || b.provider.length === 0)
|
|
5009
|
+
return json({ error: "provider must be a non-empty string" }, 400);
|
|
5010
|
+
return json(await stub.planePark(parsed.value, b.provider, now));
|
|
5011
|
+
}
|
|
4709
5012
|
if (pathname === "/plane/withdraw") {
|
|
4710
5013
|
const parsed = parseRunId(b.runId);
|
|
4711
5014
|
if (!parsed.ok) return json({ error: parsed.error }, 400);
|
|
4712
5015
|
return json(await stub.planeWithdraw(parsed.value, now));
|
|
4713
5016
|
}
|
|
5017
|
+
if (pathname === "/plane/reclaimed") {
|
|
5018
|
+
// The reclaim's outcome per row (record 0064; run-history item 36): only
|
|
5019
|
+
// `closed` records an ending; the standing causes are answered back.
|
|
5020
|
+
if (!Array.isArray(b.outcomes)) return json({ error: "outcomes must be an array" }, 400);
|
|
5021
|
+
const outcomes: { runId: string; outcome: PlaneReclaimWord }[] = [];
|
|
5022
|
+
for (const o of b.outcomes as unknown[]) {
|
|
5023
|
+
const row = o as Record<string, unknown>;
|
|
5024
|
+
const parsed = parseRunId(row?.runId);
|
|
5025
|
+
if (!parsed.ok) return json({ error: parsed.error }, 400);
|
|
5026
|
+
if (typeof row.outcome !== "string" || !PLANE_RECLAIM_WORDS.has(row.outcome))
|
|
5027
|
+
return json({ error: "outcome must be resume, restart, rehost or closed" }, 400);
|
|
5028
|
+
outcomes.push({ runId: parsed.value, outcome: row.outcome as PlaneReclaimWord });
|
|
5029
|
+
}
|
|
5030
|
+
return json(await stub.planeReclaimed(outcomes, now));
|
|
5031
|
+
}
|
|
4714
5032
|
if (pathname === "/plane/queued") {
|
|
4715
5033
|
const parsed = parseRunId(b.runId);
|
|
4716
5034
|
if (!parsed.ok) return json({ error: parsed.error }, 400);
|
|
@@ -4801,6 +5119,11 @@ function parseClaim(b: Record<string, unknown>): Validated<ClaimRequest> {
|
|
|
4801
5119
|
if (typeof meta.idempotencyKey !== "string" || !IDEMPOTENCY_KEY_PATTERN.test(meta.idempotencyKey))
|
|
4802
5120
|
return invalid("run.meta.idempotencyKey must be <parentInstanceId>:<step>");
|
|
4803
5121
|
}
|
|
5122
|
+
// The restart tag (record 0064) names the predecessor run whose windows the
|
|
5123
|
+
// claim reuses and rides the `child-resumed` event's reason: a run id or
|
|
5124
|
+
// absent, never another shape.
|
|
5125
|
+
if (meta.restartOf !== undefined && (typeof meta.restartOf !== "string" || !RUN_ID_PATTERN.test(meta.restartOf)))
|
|
5126
|
+
return invalid("run.meta.restartOf must be a run id");
|
|
4804
5127
|
if (typeof r.system !== "string") return invalid("run.system must be a string");
|
|
4805
5128
|
if (!Array.isArray(r.tools)) return invalid("run.tools must be an array");
|
|
4806
5129
|
if (r.card !== undefined && r.card !== null) {
|
|
@@ -5220,7 +5543,17 @@ async function handleLedger(pathname: string, body: unknown, env: Env): Promise<
|
|
|
5220
5543
|
// The RPC type mapping reads the effects' open-ended `request` JSON as
|
|
5221
5544
|
// unserializable; the values are plain JSON, so the cast only restores the
|
|
5222
5545
|
// declared shape (as the reclaim route's does).
|
|
5223
|
-
|
|
5546
|
+
// The heartbeat body (record 0064): plain JSON facts, validated by
|
|
5547
|
+
// shape — a malformed body beats without facts rather than dropping the lease.
|
|
5548
|
+
const facts =
|
|
5549
|
+
typeof b.facts === "object" &&
|
|
5550
|
+
b.facts !== null &&
|
|
5551
|
+
typeof (b.facts as Record<string, unknown>).round === "number" &&
|
|
5552
|
+
typeof (b.facts as Record<string, unknown>).coding === "boolean" &&
|
|
5553
|
+
typeof (b.facts as Record<string, unknown>).startedAt === "number"
|
|
5554
|
+
? (b.facts as unknown as HeartbeatFacts)
|
|
5555
|
+
: undefined;
|
|
5556
|
+
const r = (await stub.heartbeat(runId.value, g.value, lease.value, now, facts)) as unknown as HeartbeatAnswer;
|
|
5224
5557
|
return r.ok ? json(r) : json(r, 409);
|
|
5225
5558
|
}
|
|
5226
5559
|
if (pathname === "/runs/append") {
|
|
@@ -30,6 +30,14 @@ export interface DrainRecord {
|
|
|
30
30
|
* container on the deploy's image, and the fleet reopens on the LAST
|
|
31
31
|
* report, a fact, never a wait. Absent or empty: no swap outstanding. */
|
|
32
32
|
holds?: string[];
|
|
33
|
+
/** The hold's own liveness bound (issue 2044): stamped when the first hold
|
|
34
|
+
* lands, the cycle's measured bound out (`DRAIN.cycleBoundMinutes`, clamped
|
|
35
|
+
* to `until`). The gate asked the containers to cycle, so it owns the
|
|
36
|
+
* outcome: past this time a cycle that never happened reopens the fleet
|
|
37
|
+
* anyway — `liveDrain` reads the record as no drain — with the stale
|
|
38
|
+
* containers named in the registry's warning, never a silence to `until`.
|
|
39
|
+
* A record without it (an older build's) keeps `until` as its only end. */
|
|
40
|
+
holdsUntil?: string;
|
|
33
41
|
/** Whether `POST /undrain` already asked for the lift while holds stood:
|
|
34
42
|
* the record then clears itself on the last hold's report instead of
|
|
35
43
|
* waiting for a second lift. */
|
|
@@ -42,6 +50,9 @@ export interface DrainRecord {
|
|
|
42
50
|
* and a half, not a day. */
|
|
43
51
|
export const DRAIN_MAX_MINUTES: number = DRAIN.maxMinutes;
|
|
44
52
|
export const DRAIN_DEFAULT_MINUTES: number = DRAIN.defaultMinutes;
|
|
53
|
+
/** The post-deploy container cycle's measured bound (issue 2044): a hold that
|
|
54
|
+
* outlives it reopens the fleet with the container named stale. */
|
|
55
|
+
export const HOLD_CYCLE_BOUND_MINUTES: number = DRAIN.cycleBoundMinutes;
|
|
45
56
|
const REASON_MAX = 200;
|
|
46
57
|
const BY_MAX = 80;
|
|
47
58
|
|
|
@@ -100,24 +111,55 @@ export function liveDrain(stored: unknown, now: number): DrainRecord | null {
|
|
|
100
111
|
const until = Date.parse(r.until);
|
|
101
112
|
if (!Number.isFinite(until) || until <= now) return null;
|
|
102
113
|
const holds = Array.isArray(r.holds) ? r.holds.filter((h): h is string => typeof h === "string") : [];
|
|
114
|
+
const holdsUntil = typeof r.holdsUntil === "string" ? r.holdsUntil : undefined;
|
|
115
|
+
// The hold's liveness (issue 2044): holds whose cycle bound has passed are a
|
|
116
|
+
// cycle that never happened — the fleet reopens by construction, whoever
|
|
117
|
+
// died between the reconcile and the lift; `staleHolds` names the containers.
|
|
118
|
+
if (holds.length > 0 && holdsUntil !== undefined) {
|
|
119
|
+
const bound = Date.parse(holdsUntil);
|
|
120
|
+
if (Number.isFinite(bound) && bound <= now) return null;
|
|
121
|
+
}
|
|
103
122
|
return {
|
|
104
123
|
since: r.since,
|
|
105
124
|
until: r.until,
|
|
106
125
|
by: r.by,
|
|
107
126
|
reason: r.reason,
|
|
108
127
|
...(holds.length > 0 ? { holds } : {}),
|
|
128
|
+
...(holds.length > 0 && holdsUntil !== undefined ? { holdsUntil } : {}),
|
|
109
129
|
...(r.liftAsked === true ? { liftAsked: true } : {}),
|
|
110
130
|
};
|
|
111
131
|
}
|
|
112
132
|
|
|
133
|
+
/** The stale containers of a stored record whose hold bound has passed while
|
|
134
|
+
* its `until` had not (issue 2044): what `liveDrain` just reopened past, for
|
|
135
|
+
* the registry's warning — the reopen is never silent about who never cycled.
|
|
136
|
+
* Null when the record is not that case (no holds, bound still ahead, or the
|
|
137
|
+
* record expired on `until` itself). */
|
|
138
|
+
export function staleHolds(stored: unknown, now: number): string[] | null {
|
|
139
|
+
if (typeof stored !== "object" || stored === null) return null;
|
|
140
|
+
const r = stored as Record<string, unknown>;
|
|
141
|
+
const until = Date.parse(typeof r.until === "string" ? r.until : "");
|
|
142
|
+
if (!Number.isFinite(until) || until <= now) return null;
|
|
143
|
+
const holds = Array.isArray(r.holds) ? r.holds.filter((h): h is string => typeof h === "string") : [];
|
|
144
|
+
const bound = Date.parse(typeof r.holdsUntil === "string" ? r.holdsUntil : "");
|
|
145
|
+
if (holds.length === 0 || !Number.isFinite(bound) || bound > now) return null;
|
|
146
|
+
return holds;
|
|
147
|
+
}
|
|
148
|
+
|
|
113
149
|
/** The record with the named residents held (issue 1931): the deploy's
|
|
114
150
|
* reconcile could not verify their running containers on the new image, so
|
|
115
151
|
* the fleet must not reopen onto them until each reports. Deduplicated;
|
|
116
|
-
* an empty set changes nothing.
|
|
117
|
-
|
|
152
|
+
* an empty set changes nothing. The first hold stamps the record's cycle
|
|
153
|
+
* bound (issue 2044): the reconcile just asked each container to cycle, so
|
|
154
|
+
* the cycle either lands within its measured bound or is not coming — past
|
|
155
|
+
* `holdsUntil` the fleet reopens with the holdouts named, `until` staying
|
|
156
|
+
* the last resort for a record from before the bound. */
|
|
157
|
+
export function holdDrain(record: DrainRecord, resources: readonly string[], now: number): DrainRecord {
|
|
118
158
|
const holds = [...new Set([...(record.holds ?? []), ...resources])];
|
|
119
159
|
if (holds.length === 0) return record;
|
|
120
|
-
|
|
160
|
+
const bound = Math.min(Date.parse(record.until), now + minutesToMs(HOLD_CYCLE_BOUND_MINUTES));
|
|
161
|
+
const holdsUntil = record.holdsUntil ?? new Date(bound).toISOString();
|
|
162
|
+
return { ...record, holds, holdsUntil };
|
|
121
163
|
}
|
|
122
164
|
|
|
123
165
|
/** `POST /undrain`'s decision over the stored record: with no holds the drain
|
|
@@ -140,7 +182,10 @@ export function reportImageCurrent(
|
|
|
140
182
|
): { lifted: boolean; record: DrainRecord | null } {
|
|
141
183
|
const holds = (record.holds ?? []).filter((h) => h !== resource);
|
|
142
184
|
const next: DrainRecord = { ...record, ...(holds.length > 0 ? { holds } : {}) };
|
|
143
|
-
if (holds.length === 0)
|
|
185
|
+
if (holds.length === 0) {
|
|
186
|
+
delete next.holds;
|
|
187
|
+
delete next.holdsUntil;
|
|
188
|
+
}
|
|
144
189
|
if (holds.length === 0 && record.liftAsked === true) return { lifted: true, record: null };
|
|
145
190
|
return { lifted: false, record: next };
|
|
146
191
|
}
|