@coreplane/switchboard 1.256.0 → 1.258.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/dist/assets/config/config.example.yaml +14 -0
  2. package/dist/assets/deploy/cloudflare-memory/worker.ts +340 -7
  3. package/dist/assets/deploy/cloudflare-resident/drain.ts +49 -4
  4. package/dist/assets/deploy/cloudflare-resident/worker.ts +52 -4
  5. package/dist/assets/package-lock.json +3 -3
  6. package/dist/assets/package.json +5 -3
  7. package/dist/assets/project.json +17 -9
  8. package/dist/assets/source.json +3 -3
  9. package/dist/assets/src/agents/registry.ts +65 -2
  10. package/dist/assets/src/core/authz/grants.ts +12 -0
  11. package/dist/assets/src/core/authz/policy.ts +15 -0
  12. package/dist/assets/src/core/budgets.ts +66 -18
  13. package/dist/assets/src/core/coordinator/contract.ts +45 -5
  14. package/dist/assets/src/core/coordinator/driver.ts +6 -3
  15. package/dist/assets/src/core/delivery.ts +8 -5
  16. package/dist/assets/src/core/pipelineStanding.ts +2 -0
  17. package/dist/assets/src/core/plane/decide.ts +477 -12
  18. package/dist/assets/src/core/plane/findings.ts +120 -0
  19. package/dist/assets/src/core/runEvents.ts +54 -14
  20. package/dist/assets/src/core/runFriction.ts +16 -0
  21. package/dist/assets/src/core/runLedger/types.ts +16 -3
  22. package/dist/assets/src/core/runRecord.ts +12 -0
  23. package/dist/assets/src/core/ship/contract.ts +14 -2
  24. package/dist/assets/src/core/ship/coordinator.ts +179 -51
  25. package/dist/assets/src/core/ship/renewal.ts +23 -20
  26. package/dist/assets/src/core/trace/attrs.ts +1 -1
  27. package/dist/assets/src/core/untrusted.ts +35 -0
  28. package/dist/assets/src/execution/residentCredentials.ts +7 -4
  29. package/dist/assets/web/dist/.vite/manifest.json +97 -72
  30. package/dist/assets/web/dist/assets/{DeliveryPage-BBAyLwPq.js → DeliveryPage-DadntjSp.js} +1 -1
  31. package/dist/assets/web/dist/assets/HomePage-BEEpTnVT.js +1 -0
  32. package/dist/assets/web/dist/assets/{PendingTurnRow-DGINv9XT.js → PendingTurnRow-DBPMLVsm.js} +1 -1
  33. package/dist/assets/web/dist/assets/PlanePage-DLusMGQ0.js +1 -0
  34. package/dist/assets/web/dist/assets/{ResidentDetailPage-jYhEeeyu.js → ResidentDetailPage-CPnHnrxy.js} +1 -1
  35. package/dist/assets/web/dist/assets/{ResidentsIndexPage-DvFmlV4U.js → ResidentsIndexPage-C9hASwHL.js} +1 -1
  36. package/dist/assets/web/dist/assets/RunFoldRow-OPe7Z0Cz.js +1 -0
  37. package/dist/assets/web/dist/assets/{RunRoutePage-CvZ-TOT3.js → RunRoutePage-o1xzkTGZ.js} +4 -4
  38. package/dist/assets/web/dist/assets/RunsIndexPage-0YAClYQh.js +1 -0
  39. package/dist/assets/web/dist/assets/{ScheduledPage-Viim-bus.js → ScheduledPage-BIdfC0V3.js} +1 -1
  40. package/dist/assets/web/dist/assets/{SettingsPage-CYUy8McC.js → SettingsPage-CnqWAjl3.js} +1 -1
  41. package/dist/assets/web/dist/assets/SilentTurn-DGNg0sIZ.js +2 -0
  42. package/dist/assets/web/dist/assets/SlackMark-CtiHrNjr.js +1 -0
  43. package/dist/assets/web/dist/assets/{StatusDot-Dv6UMaPy.js → StatusDot-D_D_iJS8.js} +1 -1
  44. package/dist/assets/web/dist/assets/{Tooltip-Bd-5Rypv.js → Tooltip-DVFmM7on.js} +1 -1
  45. package/dist/assets/web/dist/assets/UnitRoutePage-7fAXFoB_.js +1 -0
  46. package/dist/assets/web/dist/assets/budgets-BRLm6oID.js +1 -0
  47. package/dist/assets/web/dist/assets/{dist-Nl3uaxrP.js → dist-C1VX-_mm.js} +1 -1
  48. package/dist/assets/web/dist/assets/durationTone-CVpX_yIk.js +1 -0
  49. package/dist/assets/web/dist/assets/{indexRow-DborJPFp.js → indexRow-BO_GIdF6.js} +1 -1
  50. package/dist/assets/web/dist/assets/main-Cl7xHG34.js +28 -0
  51. package/dist/assets/web/dist/assets/main-DGC6WFqS.css +1 -0
  52. package/dist/assets/web/dist/assets/{sseReplay-DE6wv1Ua.js → sseReplay-BjlKmQsY.js} +1 -1
  53. package/dist/cli.js +3096 -1136
  54. package/package.json +1 -1
  55. package/dist/assets/web/dist/assets/HomePage-BEiStFwW.js +0 -3
  56. package/dist/assets/web/dist/assets/PlanePage-JEj-lqgz.js +0 -1
  57. package/dist/assets/web/dist/assets/RunFoldRow-DvWzR1JQ.js +0 -1
  58. package/dist/assets/web/dist/assets/RunsIndexPage-JqVSDOok.js +0 -1
  59. package/dist/assets/web/dist/assets/UnitRoutePage-B81wEnKN.js +0 -1
  60. package/dist/assets/web/dist/assets/budgets-c1eumrqD.js +0 -1
  61. package/dist/assets/web/dist/assets/durationTone-U_rOqo6r.js +0 -1
  62. package/dist/assets/web/dist/assets/main-C4QfwybO.css +0 -1
  63. package/dist/assets/web/dist/assets/main-DRxWlffc.js +0 -28
@@ -132,6 +132,20 @@ defaults:
132
132
  # maxIdentity: write
133
133
  # machines: [none, blank, repo-cold, repo-resident]
134
134
  # confirm: write
135
+ # Watch until merge (docs/reference/specs/routing-and-config.md item 32;
136
+ # agent-ship.md item 21): keep a merge-ready pipeline unit on its pull
137
+ # request until the pull request merges — each push to its base that leaves
138
+ # it conflicting buys one rebase of the sweep's resolver. OFF by default.
139
+ # `rebaseInFlight` caps the watch's rebases per repository (default 1);
140
+ # `spendLimitUsd` caps one pull request's model rounds — read in whole
141
+ # rounds today, so the default buys one rebase round per pull request and
142
+ # a finer value has no extra effect yet. This block is the
143
+ # org tier; `config set org --pulls.…` layers over it at run time, and
144
+ # `config set repo --repo <owner/name> --pulls.…` overrides per repository.
145
+ # pulls:
146
+ # watch: false
147
+ # rebaseInFlight: 1
148
+ # spendLimitUsd: 5
135
149
 
136
150
  # PR reviews and the reading diff (docs/reference/specs/reading-diff.md). Every
137
151
  # review records its full `git diff`; the ABRIDGED version (meat.dev, run on
@@ -79,11 +79,16 @@ import {
79
79
  } from "../../src/core/runLedger/decisions.ts";
80
80
  import { intakeReceiptRetentionMs, minutesToMs, PLANE } from "../../src/core/budgets.ts";
81
81
  import {
82
+ causeOfClose,
83
+ causeOfReclaim,
82
84
  decide,
83
85
  effectCapRefusal,
84
86
  planeAskAnswerOf,
85
87
  planeAskWordOf,
86
88
  RESIDENT_DRAIN_WINDOW,
89
+ type PlaneEndingCause,
90
+ type PlaneReclaimWord,
91
+ type HeartbeatFacts,
87
92
  type PlaneAskAnswer,
88
93
  type PlaneLevelRow,
89
94
  type PlaneAckOutcome,
@@ -96,6 +101,7 @@ import {
96
101
  type PlaneState,
97
102
  type PlaneWrite,
98
103
  } from "../../src/core/plane/decide.ts";
104
+ import { mergePlaneFindings, planeFindingKey, type PlaneFinding } from "../../src/core/plane/findings.ts";
99
105
  import {
100
106
  IDEMPOTENCY_KEY_PATTERN,
101
107
  capThreadEvent,
@@ -103,6 +109,7 @@ import {
103
109
  isCoordinatorInstance,
104
110
  isCoordinatorUnit,
105
111
  isThreadEvent,
112
+ sendChildSignal,
106
113
  sendRunFinished,
107
114
  UNIT_PATTERN,
108
115
  type CoordinatorInstance,
@@ -1777,6 +1784,12 @@ export class RunHistoryDO extends DurableObject<Env> {
1777
1784
  generation TEXT NOT NULL,
1778
1785
  PRIMARY KEY (resident, name)
1779
1786
  );
1787
+ CREATE TABLE IF NOT EXISTS plane_endings (
1788
+ run_id TEXT PRIMARY KEY,
1789
+ kind TEXT NOT NULL,
1790
+ cause TEXT NOT NULL,
1791
+ at INTEGER NOT NULL
1792
+ );
1780
1793
  `);
1781
1794
  }
1782
1795
 
@@ -1793,13 +1806,20 @@ export class RunHistoryDO extends DurableObject<Env> {
1793
1806
  // a dispatch that died between the admission answer and its claim must not
1794
1807
  // hold the thread forever. Promotion deletes the row (the live row holds
1795
1808
  // the thread from there), so age alone is the test.
1809
+ // `steer` and `park` rows (record 0064) have no expiry window: a steer's
1810
+ // dedupe row and a parked run's wait live until the run's seal deletes them.
1796
1811
  const reservations = this.sql
1797
1812
  .exec<{ kind: string; key: string; run_id: string; at: number }>(
1798
- `SELECT * FROM plane_reservations WHERE kind = 'thread' AND at > ?`,
1813
+ `SELECT * FROM plane_reservations WHERE kind != 'thread' OR at > ?`,
1799
1814
  now - minutesToMs(PLANE.reservationMinutes),
1800
1815
  )
1801
1816
  .toArray()
1802
- .map((r): PlaneReservation => ({ kind: "thread", key: r.key, runId: r.run_id, at: r.at }));
1817
+ .map((r): PlaneReservation => ({
1818
+ kind: r.kind as PlaneReservation["kind"],
1819
+ key: r.key,
1820
+ runId: r.run_id,
1821
+ at: r.at,
1822
+ }));
1803
1823
  const openWindows = this.sql
1804
1824
  .exec<{ kind: string }>(`SELECT kind FROM plane_windows WHERE phase = 'open'`)
1805
1825
  .toArray()
@@ -1877,7 +1897,22 @@ export class RunHistoryDO extends DurableObject<Env> {
1877
1897
  w.row.at,
1878
1898
  );
1879
1899
  } else if (w.table === "plane_reservations" && w.op === "del") {
1880
- this.sql.exec(`DELETE FROM plane_reservations WHERE kind = 'thread' AND key = ?`, w.key);
1900
+ this.sql.exec(`DELETE FROM plane_reservations WHERE kind = ? AND key = ?`, w.kind ?? "thread", w.key);
1901
+ } else if (w.table === "run_inbox" && w.op === "push") {
1902
+ // The plane's steer into a live run's durable inbox (record 0064;
1903
+ // run-history item 40), in the decider's own transaction: the run reads
1904
+ // it at its next boundary like any follow-up; a run with no live row
1905
+ // reads nothing and the row would be an orphan, so it is skipped.
1906
+ if (!this.liveRow(w.runId)) continue;
1907
+ const last = this.sql
1908
+ .exec<{ m: number | null }>(`SELECT MAX(seq) AS m FROM run_inbox WHERE run_id = ?`, w.runId)
1909
+ .one().m;
1910
+ this.sql.exec(
1911
+ `INSERT INTO run_inbox (run_id, seq, json) VALUES (?, ?, ?)`,
1912
+ w.runId,
1913
+ (last ?? 0) + 1,
1914
+ JSON.stringify(w.message),
1915
+ );
1881
1916
  } else if (w.table === "plane_windows" && w.op === "put") {
1882
1917
  this.sql.exec(
1883
1918
  `INSERT OR REPLACE INTO plane_windows (kind, key, phase, opened_at, reason_json) VALUES (?, ?, 'open', ?, '{}')`,
@@ -1896,6 +1931,38 @@ export class RunHistoryDO extends DurableObject<Env> {
1896
1931
  w.row.reportedAt,
1897
1932
  w.row.generation,
1898
1933
  );
1934
+ } else if (w.table === "plane_findings") {
1935
+ // A finding (record 0064, "Endings and the watches"): keyed by watch
1936
+ // and subject, so two findings on one subject are ONE row — an
1937
+ // existing row absorbs the timeline through the same merge the
1938
+ // decider's module defines, its first-seen stamp kept.
1939
+ const key = planeFindingKey(w.finding);
1940
+ const prior = this.sql
1941
+ .exec<{ timeline_json: string; filed_at: number }>(
1942
+ `SELECT timeline_json, filed_at FROM plane_findings WHERE id = ?`,
1943
+ key,
1944
+ )
1945
+ .toArray()[0];
1946
+ const existing = prior
1947
+ ? [
1948
+ {
1949
+ watch: w.finding.watch,
1950
+ subject: w.finding.subject,
1951
+ timeline: JSON.parse(prior.timeline_json) as PlaneFinding["timeline"],
1952
+ firstAt: Number(prior.filed_at),
1953
+ lastAt: w.finding.lastAt,
1954
+ },
1955
+ ]
1956
+ : [];
1957
+ const merged = mergePlaneFindings(existing, w.finding)[0]!;
1958
+ this.sql.exec(
1959
+ `INSERT OR REPLACE INTO plane_findings (id, watch, subject, timeline_json, filed_at) VALUES (?, ?, ?, ?, ?)`,
1960
+ key,
1961
+ merged.watch,
1962
+ merged.subject,
1963
+ JSON.stringify(merged.timeline),
1964
+ merged.firstAt,
1965
+ );
1899
1966
  } else {
1900
1967
  // The effect bounds (record 0064): an offer past the per-run or total
1901
1968
  // cap is refused by the cap's name — the throw aborts the transaction,
@@ -2016,9 +2083,20 @@ export class RunHistoryDO extends DurableObject<Env> {
2016
2083
  * drain posts land as the resident-drain window's open (`above`) and lift
2017
2084
  * (`below` — a `cleared` or the alarm's `expired`). */
2018
2085
  planeLevel(
2019
- post: { resident: string; name: "seat" | "memory" | "drain"; side: "below" | "above"; generation: string },
2086
+ post:
2087
+ | { resident: string; name: "seat" | "memory" | "drain"; side: "below" | "above"; generation: string }
2088
+ | { provider: string; name: "provider"; side: "up" | "down" },
2020
2089
  now: number,
2021
2090
  ): { admitted: number } {
2091
+ // The model proxy's provider level (record 0064): `up` re-issues every
2092
+ // held turn parked on the provider — the steers land as inbox writes in the
2093
+ // decider's transaction — and walks anything queued on `provider_up`.
2094
+ if (post.name === "provider") {
2095
+ const r = this.planeApply({ kind: "provider_level", at: now, provider: post.provider, level: post.side });
2096
+ const admitted = r.effects.filter((e) => e.kind === "admit").length;
2097
+ console.log(`[plane/level] provider ${post.provider} ${post.side} — ${admitted} admission(s)`);
2098
+ return { admitted };
2099
+ }
2022
2100
  const r =
2023
2101
  post.name === "drain"
2024
2102
  ? this.planeApply({
@@ -2040,6 +2118,20 @@ export class RunHistoryDO extends DurableObject<Env> {
2040
2118
  return { admitted };
2041
2119
  }
2042
2120
 
2121
+ /** A run parked on its provider (`POST /plane/park`, record 0064): the
2122
+ * proxy could not complete the turn after its retry; the harness holds the
2123
+ * turn and the provider's next `up` steers the run to re-issue it. */
2124
+ planePark(runId: string, provider: string, now: number): { parked: boolean } {
2125
+ let parked = false;
2126
+ this.ctx.storage.transactionSync(() => {
2127
+ const decision = decide(this.planeState(), { kind: "park", at: now, runId, provider });
2128
+ this.applyPlaneWrites(decision.writes);
2129
+ parked = decision.writes.length > 0;
2130
+ });
2131
+ console.log(`[plane/park] run ${runId} on ${provider} — ${parked ? "parked" : "already parked"}`);
2132
+ return { parked };
2133
+ }
2134
+
2043
2135
  /** A refusal-by-name the bot met at attach or exec (`POST /plane/observe`,
2044
2136
  * record 0064): an admitted run re-enters the queue at its old position. */
2045
2137
  planeObserve(post: { runId: string; resident: string; refusal: string }, now: number): { reentered: boolean } {
@@ -2084,6 +2176,54 @@ export class RunHistoryDO extends DurableObject<Env> {
2084
2176
  if (set === null || set > due) await this.ctx.storage.setAlarm(due);
2085
2177
  }
2086
2178
 
2179
+ /** The earliest instant the plane must wake at (record 0064): the
2180
+ * earliest hosting deadline, lease end or re-ask across its rows. A due
2181
+ * already past re-offers at the re-ask cadence, never in a hot loop, and
2182
+ * contributes nothing to end a run — the alarm only offers. */
2183
+ private planeEarliestDue(now: number): number | undefined {
2184
+ const dues: number[] = [];
2185
+ const reoffer = now + this.planeReaskMs();
2186
+ for (const r of this.sql
2187
+ .exec<{ lease_until: number; state_json: string }>(`SELECT lease_until, state_json FROM live_runs`)
2188
+ .toArray()) {
2189
+ dues.push(r.lease_until > now ? r.lease_until : reoffer);
2190
+ try {
2191
+ const hosting = (JSON.parse(r.state_json) as Record<string, unknown>).hosting as { until?: unknown };
2192
+ if (typeof hosting?.until === "number") dues.push(hosting.until > now ? hosting.until : reoffer);
2193
+ } catch {
2194
+ // A malformed state contributes no deadline.
2195
+ }
2196
+ }
2197
+ if (this.planeWaitsOnResident()) dues.push(reoffer);
2198
+ return dues.length === 0 ? undefined : Math.min(...dues);
2199
+ }
2200
+
2201
+ /** The plane's one alarm (record 0064): set to the earliest due across
2202
+ * its rows — the owner's heartbeat, moving the lease end, moves an alarm
2203
+ * the plane armed; an alarm someone else armed earlier is left to fire
2204
+ * first (the handler re-arms). The armed slot is judged directly, never
2205
+ * the meta row alone: the slot is shared with the re-ask and the sweep and
2206
+ * a fired alarm is consumed, so a meta row equal to a static due (a
2207
+ * hosting deadline) can claim a wake that no longer exists. The due is
2208
+ * capped to the sweep interval so the retention sweep never starves behind
2209
+ * a distant hosting deadline. */
2210
+ private async ensurePlaneAlarm(now: number): Promise<void> {
2211
+ const raw = this.planeEarliestDue(now);
2212
+ const prior = this.sql.exec<{ value: string }>(`SELECT value FROM meta WHERE key = 'plane_alarm_at'`).toArray()[0];
2213
+ const priorAt = prior ? Number(prior.value) : undefined;
2214
+ if (raw === undefined) {
2215
+ if (prior) this.sql.exec(`DELETE FROM meta WHERE key = 'plane_alarm_at'`);
2216
+ return; // nothing waits: the sweep's own arming stands
2217
+ }
2218
+ const due = Math.min(raw, now + RUN_SWEEP_INTERVAL_MS);
2219
+ const set = await this.ctx.storage.getAlarm();
2220
+ // Re-arm when the slot is empty, later than the due, or holds an alarm
2221
+ // this plane armed itself; an earlier foreign alarm fires first.
2222
+ if (set !== due && (set === null || set > due || set === priorAt)) await this.ctx.storage.setAlarm(due);
2223
+ if (due !== priorAt)
2224
+ this.sql.exec(`INSERT OR REPLACE INTO meta (key, value) VALUES ('plane_alarm_at', ?)`, String(due));
2225
+ }
2226
+
2087
2227
  /** A window's open or lift over the RPC seam (`/plane/deploy`; a later
2088
2228
  * unit's `plane window lift`): kind `deploy` is the pending deploy. */
2089
2229
  planeWindow(window: string, phase: "opened" | "lifted", now: number): { admitted: number } {
@@ -2108,6 +2248,78 @@ export class RunHistoryDO extends DurableObject<Env> {
2108
2248
  return this.planeState().queue.find((r) => r.runId === runId) ?? null;
2109
2249
  }
2110
2250
 
2251
+ /** The ending's cause (record 0064, "Endings and the watches"): one
2252
+ * `ended { kind, cause }` per closed row, recorded only when a live row
2253
+ * closes and first-writer-wins — a roll that resumes every row assigns
2254
+ * nothing. One keyed exception, taken only by the finish (`supersedes`): a
2255
+ * standing `resident_replaced` was a `restarting` close — the run carried
2256
+ * on under its own id (run-history item 42's restart) — so that run's own
2257
+ * later finish replaces it and the ending agrees with the record. The bot
2258
+ * renders the word; the object never renders. */
2259
+ private recordPlaneEnding(
2260
+ runId: string,
2261
+ kind: string,
2262
+ cause: PlaneEndingCause,
2263
+ at: number,
2264
+ supersedes = false,
2265
+ ): PlaneEndingCause {
2266
+ this.sql.exec(
2267
+ `INSERT OR IGNORE INTO plane_endings (run_id, kind, cause, at) VALUES (?, ?, ?, ?)`,
2268
+ runId,
2269
+ kind,
2270
+ cause,
2271
+ at,
2272
+ );
2273
+ const standing = this.sql
2274
+ .exec<{ cause: string }>(`SELECT cause FROM plane_endings WHERE run_id = ?`, runId)
2275
+ .toArray()[0];
2276
+ if (
2277
+ supersedes &&
2278
+ standing !== undefined &&
2279
+ standing.cause === "resident_replaced" &&
2280
+ cause !== "resident_replaced"
2281
+ ) {
2282
+ this.sql.exec(`UPDATE plane_endings SET kind = ?, cause = ?, at = ? WHERE run_id = ?`, kind, cause, at, runId);
2283
+ return cause;
2284
+ }
2285
+ return (standing?.cause as PlaneEndingCause | undefined) ?? cause;
2286
+ }
2287
+
2288
+ /** The reclaim's outcome per row (record 0064; run-history item 36): only a
2289
+ * `closed` row records an ending — `lease_lapsed`, which is true and blames
2290
+ * nobody — and the standing cause is answered back so the bot's interrupted
2291
+ * note renders the plane's word. `resume`, `restart` and `rehost` record
2292
+ * nothing: the run carries on. */
2293
+ planeReclaimed(
2294
+ outcomes: readonly { runId: string; outcome: PlaneReclaimWord }[],
2295
+ now: number,
2296
+ ): { recorded: { runId: string; cause: PlaneEndingCause }[] } {
2297
+ const recorded: { runId: string; cause: PlaneEndingCause }[] = [];
2298
+ this.ctx.storage.transactionSync(() => {
2299
+ for (const o of outcomes) {
2300
+ const cause = causeOfReclaim(o.outcome);
2301
+ if (cause === undefined) continue;
2302
+ recorded.push({ runId: o.runId, cause: this.recordPlaneEnding(o.runId, "interrupted", cause, now) });
2303
+ }
2304
+ });
2305
+ if (outcomes.length > 0)
2306
+ console.log(
2307
+ `[plane/reclaimed] ${outcomes.map((o) => `${o.runId}=${o.outcome}`).join(", ")} — ${recorded.length} ending(s) recorded`,
2308
+ );
2309
+ return { recorded };
2310
+ }
2311
+
2312
+ /** One recorded ending (record 0064), or none: what a reader renders. */
2313
+ planeEndingOf(runId: string): { kind: string; cause: PlaneEndingCause; at: number } | null {
2314
+ const row = this.sql
2315
+ .exec<{ kind: string; cause: string; at: number }>(
2316
+ `SELECT kind, cause, at FROM plane_endings WHERE run_id = ?`,
2317
+ runId,
2318
+ )
2319
+ .toArray()[0];
2320
+ return row ? { kind: row.kind, cause: row.cause as PlaneEndingCause, at: row.at } : null;
2321
+ }
2322
+
2111
2323
  /** The seal (record 0064, "The queue"): the run's own open effects are
2112
2324
  * dropped — an admit for a run that just ended is stale — then the sealed
2113
2325
  * event frees the thread and walks the queue, all in one transaction. */
@@ -2118,6 +2330,9 @@ export class RunHistoryDO extends DurableObject<Env> {
2118
2330
  // Exact id matching (`admit:<runId>`), never LIKE: a bot-minted run id
2119
2331
  // can carry `%` or `_`, which a pattern would read as wildcards.
2120
2332
  this.sql.exec(`DELETE FROM plane_effects WHERE acked_at IS NULL AND id = 'admit:' || ?`, runId);
2333
+ // The run's steer dedupe rows and any park go with it (record 0064):
2334
+ // a sealed run holds no turn and reads no steer.
2335
+ this.sql.exec(`DELETE FROM plane_reservations WHERE kind IN ('steer', 'park') AND run_id = ?`, runId);
2121
2336
  const decision = decide(this.planeState(), { kind: "sealed", at: now, threadKey });
2122
2337
  this.applyPlaneWrites(decision.writes);
2123
2338
  effects = decision.effects;
@@ -2527,11 +2742,35 @@ export class RunHistoryDO extends DurableObject<Env> {
2527
2742
  JSON.stringify(req.state ?? {}),
2528
2743
  );
2529
2744
  });
2745
+ if (out.ok) {
2746
+ // A `restartOf` claim under a coordinator (record 0064): the plane
2747
+ // tells the waiting parent the child resumed — best effort, beside the
2748
+ // bot's own announcement; a duplicate is consumed and re-armed, harmless.
2749
+ if (req.meta.restartOf !== undefined && req.meta.parentInstanceId !== undefined) {
2750
+ const sent = await sendChildSignal(this.env.SHIP_COORDINATOR, {
2751
+ runId: req.runId,
2752
+ parentInstanceId: req.meta.parentInstanceId,
2753
+ kind: "resumed",
2754
+ reason: `restarted from run ${req.meta.restartOf}`,
2755
+ at: now,
2756
+ });
2757
+ if (sent.kind === "failed")
2758
+ console.warn(`[runs/claim] ${req.runId} → ${sent.type} not delivered to ${sent.instance}: ${sent.reason}`);
2759
+ }
2760
+ // The lease end joins the plane's alarm (record 0064): armed at the earliest due.
2761
+ await this.ensurePlaneAlarm(now);
2762
+ }
2530
2763
  return out;
2531
2764
  }
2532
2765
 
2533
2766
  /** Extends the lease iff the caller owns the run; answers what another generation asked for. */
2534
- async heartbeat(runId: string, gen: string, leaseMs: number, now: number): Promise<HeartbeatAnswer> {
2767
+ async heartbeat(
2768
+ runId: string,
2769
+ gen: string,
2770
+ leaseMs: number,
2771
+ now: number,
2772
+ facts?: HeartbeatFacts,
2773
+ ): Promise<HeartbeatAnswer> {
2535
2774
  let out: HeartbeatAnswer = { ok: false, reason: "unknown-run" };
2536
2775
  this.ctx.storage.transactionSync(() => {
2537
2776
  const row = this.liveRow(runId);
@@ -2541,11 +2780,28 @@ export class RunHistoryDO extends DurableObject<Env> {
2541
2780
  return;
2542
2781
  }
2543
2782
  this.sql.exec(`UPDATE live_runs SET lease_until = ? WHERE run_id = ?`, now + leaseMs, runId);
2783
+ // The heartbeat body (record 0064): the facts are judged for the
2784
+ // checkpoint steer in this same transaction, so the inbox row and its
2785
+ // dedupe row land with the lease or not at all. A cap throw here must
2786
+ // not undo the lease of a healthy run, so the decision guards itself.
2787
+ if (facts !== undefined) {
2788
+ try {
2789
+ const decision = decide(this.planeState(), { kind: "heartbeat", at: now, runId, facts });
2790
+ this.applyPlaneWrites(decision.writes);
2791
+ if (decision.writes.some((w) => w.table === "run_inbox"))
2792
+ console.log(`[plane/steer] run ${runId} round ${facts.round} — checkpoint steer written`);
2793
+ } catch (err) {
2794
+ console.warn(`[plane/steer] run ${runId}: ${err instanceof Error ? err.message : String(err)}`);
2795
+ }
2796
+ }
2544
2797
  // The plane's open effects ride every owner's heartbeat answer (record
2545
2798
  // 0064; orchestration-plane item 7) — empty until a unit writes them, but always present, so the
2546
2799
  // client's ack loop needs no version probe.
2547
2800
  out = { ok: true, stop: row.stop, phase: row.phase, effects: this.openPlaneEffects() };
2548
2801
  });
2802
+ // The owner's heartbeat refreshes the lease, so the plane's alarm moves
2803
+ // with it (record 0064): re-armed only when the earliest due changed.
2804
+ if (out.ok) await this.ensurePlaneAlarm(now);
2549
2805
  return out;
2550
2806
  }
2551
2807
 
@@ -2688,6 +2944,16 @@ export class RunHistoryDO extends DurableObject<Env> {
2688
2944
  }
2689
2945
  const put = this.upsertInTransaction(record, proposal);
2690
2946
  this.deleteLiveRows([runId]);
2947
+ // The ending's cause (record 0064), recorded exactly when the live
2948
+ // row closes: the record's own status word, `restarting` reading as the
2949
+ // resident replacement the reattach path observed.
2950
+ this.recordPlaneEnding(
2951
+ runId,
2952
+ record.status,
2953
+ causeOfClose(record.status, record.restarting === true),
2954
+ record.finishedAt,
2955
+ true,
2956
+ );
2691
2957
  turnedFinal = put.turnedFinal;
2692
2958
  out = { ok: true, stored: put.stored };
2693
2959
  });
@@ -3199,8 +3465,25 @@ export class RunHistoryDO extends DurableObject<Env> {
3199
3465
  const probes = this.planeApply({ kind: "reask", at: now, cadenceMs: this.planeReaskMs() }).effects;
3200
3466
  if (probes.length > 0) console.log(`[plane/reask] ${probes.map((e) => e.id).join(", ")}`);
3201
3467
  }
3468
+ // A lease end offers the row to a generation other than its owner and
3469
+ // never ends a run (record 0064): the row stays exactly as it is —
3470
+ // any reclaim can take it now — and the open effects are re-pushed so a
3471
+ // listening bot sweeps sooner. The owner's next heartbeat refreshes an
3472
+ // unreclaimed row and moves the alarm on.
3473
+ const lapsed = this.sql
3474
+ .exec<{ run_id: string; owner_gen: string }>(
3475
+ `SELECT run_id, owner_gen FROM live_runs WHERE lease_until <= ?`,
3476
+ now,
3477
+ )
3478
+ .toArray();
3479
+ if (lapsed.length > 0) {
3480
+ console.log(
3481
+ `[plane/alarm] ${lapsed.length} lease(s) lapsed (${lapsed.map((l) => l.run_id).join(", ")}) — offered to any generation but the owner; nothing closed`,
3482
+ );
3483
+ this.pushPlaneEffects(this.openPlaneEffects());
3484
+ }
3202
3485
  await this.ctx.storage.setAlarm(now + RUN_SWEEP_INTERVAL_MS);
3203
- await this.ensurePlaneReaskAlarm(now);
3486
+ await this.ensurePlaneAlarm(now);
3204
3487
  root.end("ok", { swept: deleted });
3205
3488
  } catch (err) {
3206
3489
  root.fail(err);
@@ -4583,8 +4866,12 @@ const PLANE_ROUTES = new Set([
4583
4866
  "/plane/queued",
4584
4867
  "/plane/level",
4585
4868
  "/plane/observe",
4869
+ "/plane/park",
4870
+ "/plane/reclaimed",
4586
4871
  ]);
4587
4872
 
4873
+ const PLANE_RECLAIM_WORDS = new Set(["resume", "restart", "rehost", "closed"]);
4874
+
4588
4875
  const PLANE_LEVEL_NAMES = new Set(["seat", "memory", "drain"]);
4589
4876
  const PLANE_LEVEL_SIDES = new Set(["below", "above"]);
4590
4877
 
@@ -4675,6 +4962,14 @@ async function handlePlane(pathname: string, body: unknown, env: Env): Promise<R
4675
4962
  }
4676
4963
  }
4677
4964
  if (pathname === "/plane/level") {
4965
+ // The model proxy's provider level (record 0064): `up` on a relayed
4966
+ // success, `down` on a failure past its one retry.
4967
+ if (b.name === "provider") {
4968
+ if (typeof b.provider !== "string" || b.provider.length === 0)
4969
+ return json({ error: "provider must be a non-empty string" }, 400);
4970
+ if (b.side !== "up" && b.side !== "down") return json({ error: "side must be up or down" }, 400);
4971
+ return json(await stub.planeLevel({ provider: b.provider, name: "provider", side: b.side }, now));
4972
+ }
4678
4973
  // A resident's level report (record 0064): forwarded by the bot from the levels a
4679
4974
  // resident answer carried, or from the registry's drain outbox.
4680
4975
  if (typeof b.resident !== "string" || b.resident.length === 0)
@@ -4706,11 +5001,34 @@ async function handlePlane(pathname: string, body: unknown, env: Env): Promise<R
4706
5001
  return json({ error: "refusal must be a non-empty string" }, 400);
4707
5002
  return json(await stub.planeObserve({ runId: parsed.value, resident: b.resident, refusal: b.refusal }, now));
4708
5003
  }
5004
+ if (pathname === "/plane/park") {
5005
+ // A run parked on its provider (record 0064).
5006
+ const parsed = parseRunId(b.runId);
5007
+ if (!parsed.ok) return json({ error: parsed.error }, 400);
5008
+ if (typeof b.provider !== "string" || b.provider.length === 0)
5009
+ return json({ error: "provider must be a non-empty string" }, 400);
5010
+ return json(await stub.planePark(parsed.value, b.provider, now));
5011
+ }
4709
5012
  if (pathname === "/plane/withdraw") {
4710
5013
  const parsed = parseRunId(b.runId);
4711
5014
  if (!parsed.ok) return json({ error: parsed.error }, 400);
4712
5015
  return json(await stub.planeWithdraw(parsed.value, now));
4713
5016
  }
5017
+ if (pathname === "/plane/reclaimed") {
5018
+ // The reclaim's outcome per row (record 0064; run-history item 36): only
5019
+ // `closed` records an ending; the standing causes are answered back.
5020
+ if (!Array.isArray(b.outcomes)) return json({ error: "outcomes must be an array" }, 400);
5021
+ const outcomes: { runId: string; outcome: PlaneReclaimWord }[] = [];
5022
+ for (const o of b.outcomes as unknown[]) {
5023
+ const row = o as Record<string, unknown>;
5024
+ const parsed = parseRunId(row?.runId);
5025
+ if (!parsed.ok) return json({ error: parsed.error }, 400);
5026
+ if (typeof row.outcome !== "string" || !PLANE_RECLAIM_WORDS.has(row.outcome))
5027
+ return json({ error: "outcome must be resume, restart, rehost or closed" }, 400);
5028
+ outcomes.push({ runId: parsed.value, outcome: row.outcome as PlaneReclaimWord });
5029
+ }
5030
+ return json(await stub.planeReclaimed(outcomes, now));
5031
+ }
4714
5032
  if (pathname === "/plane/queued") {
4715
5033
  const parsed = parseRunId(b.runId);
4716
5034
  if (!parsed.ok) return json({ error: parsed.error }, 400);
@@ -4801,6 +5119,11 @@ function parseClaim(b: Record<string, unknown>): Validated<ClaimRequest> {
4801
5119
  if (typeof meta.idempotencyKey !== "string" || !IDEMPOTENCY_KEY_PATTERN.test(meta.idempotencyKey))
4802
5120
  return invalid("run.meta.idempotencyKey must be <parentInstanceId>:<step>");
4803
5121
  }
5122
+ // The restart tag (record 0064) names the predecessor run whose windows the
5123
+ // claim reuses and rides the `child-resumed` event's reason: a run id or
5124
+ // absent, never another shape.
5125
+ if (meta.restartOf !== undefined && (typeof meta.restartOf !== "string" || !RUN_ID_PATTERN.test(meta.restartOf)))
5126
+ return invalid("run.meta.restartOf must be a run id");
4804
5127
  if (typeof r.system !== "string") return invalid("run.system must be a string");
4805
5128
  if (!Array.isArray(r.tools)) return invalid("run.tools must be an array");
4806
5129
  if (r.card !== undefined && r.card !== null) {
@@ -5220,7 +5543,17 @@ async function handleLedger(pathname: string, body: unknown, env: Env): Promise<
5220
5543
  // The RPC type mapping reads the effects' open-ended `request` JSON as
5221
5544
  // unserializable; the values are plain JSON, so the cast only restores the
5222
5545
  // declared shape (as the reclaim route's does).
5223
- const r = (await stub.heartbeat(runId.value, g.value, lease.value, now)) as unknown as HeartbeatAnswer;
5546
+ // The heartbeat body (record 0064): plain JSON facts, validated by
5547
+ // shape — a malformed body beats without facts rather than dropping the lease.
5548
+ const facts =
5549
+ typeof b.facts === "object" &&
5550
+ b.facts !== null &&
5551
+ typeof (b.facts as Record<string, unknown>).round === "number" &&
5552
+ typeof (b.facts as Record<string, unknown>).coding === "boolean" &&
5553
+ typeof (b.facts as Record<string, unknown>).startedAt === "number"
5554
+ ? (b.facts as unknown as HeartbeatFacts)
5555
+ : undefined;
5556
+ const r = (await stub.heartbeat(runId.value, g.value, lease.value, now, facts)) as unknown as HeartbeatAnswer;
5224
5557
  return r.ok ? json(r) : json(r, 409);
5225
5558
  }
5226
5559
  if (pathname === "/runs/append") {
@@ -30,6 +30,14 @@ export interface DrainRecord {
30
30
  * container on the deploy's image, and the fleet reopens on the LAST
31
31
  * report, a fact, never a wait. Absent or empty: no swap outstanding. */
32
32
  holds?: string[];
33
+ /** The hold's own liveness bound (issue 2044): stamped when the first hold
34
+ * lands, the cycle's measured bound out (`DRAIN.cycleBoundMinutes`, clamped
35
+ * to `until`). The gate asked the containers to cycle, so it owns the
36
+ * outcome: past this time a cycle that never happened reopens the fleet
37
+ * anyway — `liveDrain` reads the record as no drain — with the stale
38
+ * containers named in the registry's warning, never a silence to `until`.
39
+ * A record without it (an older build's) keeps `until` as its only end. */
40
+ holdsUntil?: string;
33
41
  /** Whether `POST /undrain` already asked for the lift while holds stood:
34
42
  * the record then clears itself on the last hold's report instead of
35
43
  * waiting for a second lift. */
@@ -42,6 +50,9 @@ export interface DrainRecord {
42
50
  * and a half, not a day. */
43
51
  export const DRAIN_MAX_MINUTES: number = DRAIN.maxMinutes;
44
52
  export const DRAIN_DEFAULT_MINUTES: number = DRAIN.defaultMinutes;
53
+ /** The post-deploy container cycle's measured bound (issue 2044): a hold that
54
+ * outlives it reopens the fleet with the container named stale. */
55
+ export const HOLD_CYCLE_BOUND_MINUTES: number = DRAIN.cycleBoundMinutes;
45
56
  const REASON_MAX = 200;
46
57
  const BY_MAX = 80;
47
58
 
@@ -100,24 +111,55 @@ export function liveDrain(stored: unknown, now: number): DrainRecord | null {
100
111
  const until = Date.parse(r.until);
101
112
  if (!Number.isFinite(until) || until <= now) return null;
102
113
  const holds = Array.isArray(r.holds) ? r.holds.filter((h): h is string => typeof h === "string") : [];
114
+ const holdsUntil = typeof r.holdsUntil === "string" ? r.holdsUntil : undefined;
115
+ // The hold's liveness (issue 2044): holds whose cycle bound has passed are a
116
+ // cycle that never happened — the fleet reopens by construction, whoever
117
+ // died between the reconcile and the lift; `staleHolds` names the containers.
118
+ if (holds.length > 0 && holdsUntil !== undefined) {
119
+ const bound = Date.parse(holdsUntil);
120
+ if (Number.isFinite(bound) && bound <= now) return null;
121
+ }
103
122
  return {
104
123
  since: r.since,
105
124
  until: r.until,
106
125
  by: r.by,
107
126
  reason: r.reason,
108
127
  ...(holds.length > 0 ? { holds } : {}),
128
+ ...(holds.length > 0 && holdsUntil !== undefined ? { holdsUntil } : {}),
109
129
  ...(r.liftAsked === true ? { liftAsked: true } : {}),
110
130
  };
111
131
  }
112
132
 
133
+ /** The stale containers of a stored record whose hold bound has passed while
134
+ * its `until` had not (issue 2044): what `liveDrain` just reopened past, for
135
+ * the registry's warning — the reopen is never silent about who never cycled.
136
+ * Null when the record is not that case (no holds, bound still ahead, or the
137
+ * record expired on `until` itself). */
138
+ export function staleHolds(stored: unknown, now: number): string[] | null {
139
+ if (typeof stored !== "object" || stored === null) return null;
140
+ const r = stored as Record<string, unknown>;
141
+ const until = Date.parse(typeof r.until === "string" ? r.until : "");
142
+ if (!Number.isFinite(until) || until <= now) return null;
143
+ const holds = Array.isArray(r.holds) ? r.holds.filter((h): h is string => typeof h === "string") : [];
144
+ const bound = Date.parse(typeof r.holdsUntil === "string" ? r.holdsUntil : "");
145
+ if (holds.length === 0 || !Number.isFinite(bound) || bound > now) return null;
146
+ return holds;
147
+ }
148
+
113
149
  /** The record with the named residents held (issue 1931): the deploy's
114
150
  * reconcile could not verify their running containers on the new image, so
115
151
  * the fleet must not reopen onto them until each reports. Deduplicated;
116
- * an empty set changes nothing. */
117
- export function holdDrain(record: DrainRecord, resources: readonly string[]): DrainRecord {
152
+ * an empty set changes nothing. The first hold stamps the record's cycle
153
+ * bound (issue 2044): the reconcile just asked each container to cycle, so
154
+ * the cycle either lands within its measured bound or is not coming — past
155
+ * `holdsUntil` the fleet reopens with the holdouts named, `until` staying
156
+ * the last resort for a record from before the bound. */
157
+ export function holdDrain(record: DrainRecord, resources: readonly string[], now: number): DrainRecord {
118
158
  const holds = [...new Set([...(record.holds ?? []), ...resources])];
119
159
  if (holds.length === 0) return record;
120
- return { ...record, holds };
160
+ const bound = Math.min(Date.parse(record.until), now + minutesToMs(HOLD_CYCLE_BOUND_MINUTES));
161
+ const holdsUntil = record.holdsUntil ?? new Date(bound).toISOString();
162
+ return { ...record, holds, holdsUntil };
121
163
  }
122
164
 
123
165
  /** `POST /undrain`'s decision over the stored record: with no holds the drain
@@ -140,7 +182,10 @@ export function reportImageCurrent(
140
182
  ): { lifted: boolean; record: DrainRecord | null } {
141
183
  const holds = (record.holds ?? []).filter((h) => h !== resource);
142
184
  const next: DrainRecord = { ...record, ...(holds.length > 0 ? { holds } : {}) };
143
- if (holds.length === 0) delete next.holds;
185
+ if (holds.length === 0) {
186
+ delete next.holds;
187
+ delete next.holdsUntil;
188
+ }
144
189
  if (holds.length === 0 && record.liftAsked === true) return { lifted: true, record: null };
145
190
  return { lifted: false, record: next };
146
191
  }