@coreplane/switchboard 1.267.2 → 1.268.0

package/dist/assets/deploy/cloudflare-memory/testDiagnostics.ts

@@ -1,11 +1,16 @@
 import { afterEach, beforeEach, type TestContext } from "vitest";
+import { systemClock } from "../../src/core/trace/clock.ts";
 import {
   assertNoPendingBackgroundTasks,
   backgroundTaskDiagnostics,
   beginBackgroundTaskDiagnostics,
   endBackgroundTaskDiagnostics,
 } from "./backgroundTasks.ts";
-import { MEMORY_DIAGNOSTICS_ANNOTATION, type MemoryTestAnnotation } from "./testDiagnosticsProtocol.ts";
+import {
+  MEMORY_DIAGNOSTICS_ANNOTATION,
+  type MemoryTestAnnotation,
+  type MemoryTestOperation,
+} from "./testDiagnosticsProtocol.ts";
 
 type TimerKind = "timeout" | "interval";
 type TimerHandler = (...args: unknown[]) => unknown;
@@ -24,6 +29,7 @@ interface RunningTest {
   name: string;
   trackTimers: boolean;
   timers: Map<ReturnType<typeof setTimeout>, PendingTimer>;
+  operations: MemoryTestOperation[];
 }
 interface TimerOriginals {
   setTimeout: typeof globalThis.setTimeout;
@@ -167,6 +173,40 @@ function poolWorker(): string {
   return `pool ${poolId ?? "unknown"} / worker ${workerId ?? "unknown"}`;
 }
 
+export async function recordMemoryTestOperation<T>(label: string, run: () => Promise<T>): Promise<T> {
+  const running = diagnosticGlobal()[RUNNING];
+  const operation: MemoryTestOperation = { label, state: "pending" };
+  running?.operations.push(operation);
+  const started = systemClock();
+  try {
+    const result = await run();
+    operation.state = "passed";
+    return result;
+  } catch (error) {
+    operation.state = "failed";
+    throw error;
+  } finally {
+    operation.durationMs = Math.max(0, systemClock() - started);
+  }
+}
+
+export function recordMemoryTestRequest(label: string, fetch: () => Promise<Response>): Promise<Response>;
+export function recordMemoryTestRequest<T>(
+  label: string,
+  fetch: () => Promise<Response>,
+  consume: (response: Response) => T | Promise<T>,
+): Promise<T>;
+export function recordMemoryTestRequest<T>(
+  label: string,
+  fetch: () => Promise<Response>,
+  consume?: (response: Response) => T | Promise<T>,
+): Promise<Response | T> {
+  return recordMemoryTestOperation(label, async () => {
+    const response = await fetch();
+    return consume ? consume(response) : response;
+  });
+}
+
 function pendingTimerLabels(running: RunningTest): string[] {
   running.trackTimers = false;
   return [...running.timers.values()].map((timer) => timer.label);
@@ -191,19 +231,20 @@ export function installMemoryTestDiagnostics(): void {
   installTimerTrap();
 
   beforeEach((context) => {
-    diagnosticGlobal()[RUNNING] = { name: testName(context), trackTimers: true, timers: new Map() };
+    diagnosticGlobal()[RUNNING] = { name: testName(context), trackTimers: true, timers: new Map(), operations: [] };
     beginBackgroundTaskDiagnostics();
   });
 
   afterEach((context) => {
     const root = diagnosticGlobal();
-    const running = root[RUNNING] ?? { name: testName(context), trackTimers: false, timers: new Map() };
+    const running = root[RUNNING] ?? { name: testName(context), trackTimers: false, timers: new Map(), operations: [] };
     const background = backgroundTaskDiagnostics();
     const timers = pendingTimerLabels(running);
     const annotation: MemoryTestAnnotation = {
       poolWorker: poolWorker(),
       ...background,
       pendingTimers: timers,
+      operations: running.operations.map((operation) => ({ ...operation })),
     };
 
     const problems: Error[] = [];

package/dist/assets/deploy/cloudflare-memory/testDiagnosticsProtocol.ts

@@ -1,10 +1,17 @@
 export const MEMORY_DIAGNOSTICS_ANNOTATION = "memory-worker-diagnostics";
 
+export interface MemoryTestOperation {
+  label: string;
+  state: "pending" | "passed" | "failed";
+  durationMs?: number;
+}
+
 export interface MemoryTestAnnotation {
   poolWorker: string;
   registeredBackgroundTasks: string[];
   pendingPromises: string[];
   pendingTimers: string[];
+  operations: MemoryTestOperation[];
 }
 
 export interface MemoryTestDiagnostic extends MemoryTestAnnotation {

package/dist/assets/deploy/cloudflare-memory/testDiagnosticsReporter.ts

@@ -30,6 +30,7 @@ function annotationFor(testCase: TestCase): MemoryTestAnnotation {
     registeredBackgroundTasks: [],
     pendingPromises: [],
     pendingTimers: [],
+    operations: [],
   };
 }
 
@@ -84,5 +85,26 @@ export class MemoryDiagnosticsReporter implements Reporter {
         ? "no completed tests"
         : slowest.map((test) => `${Math.round(test.durationMs)}ms ${test.test}`).join("; ");
     this.#writeLog(`[memory diagnostics] slowest ${slowest.length}: ${summary}`);
+    for (const test of this.#tests) {
+      if (test.operations.length === 0 || (test.state !== "failed" && !slowest.includes(test))) continue;
+      const selected =
+        test.state === "failed"
+          ? [
+              ...new Set([
+                ...test.operations.filter((operation) => operation.state === "pending"),
+                ...test.operations.slice(-8),
+              ]),
+            ]
+          : [...test.operations].sort((a, b) => (b.durationMs ?? Infinity) - (a.durationMs ?? Infinity)).slice(0, 3);
+      const operations = selected
+        .map(
+          (operation) =>
+            `${operation.label} ${operation.state} ${operation.durationMs === undefined ? "pending" : `${operation.durationMs}ms`}`,
+        )
+        .join("; ");
+      this.#writeLog(
+        `[memory diagnostics] ${test.test}: ${selected.length}/${test.operations.length} operation(s): ${operations}`,
+      );
+    }
   }
 }

package/dist/assets/deploy/cloudflare-memory/testFetch.ts

@@ -0,0 +1,21 @@
+import { SELF } from "cloudflare:test";
+import { recordMemoryTestRequest } from "./testDiagnostics.ts";
+
+/** A request stays pending until the test has consumed its response body. */
+export function fetchMemoryTest(url: string, init?: RequestInit): Promise<Response>;
+export function fetchMemoryTest<T>(
+  url: string,
+  init: RequestInit | undefined,
+  consume: (response: Response) => T | Promise<T>,
+): Promise<T>;
+export function fetchMemoryTest<T>(
+  url: string,
+  init?: RequestInit,
+  consume?: (response: Response) => T | Promise<T>,
+): Promise<Response | T> {
+  const method = init?.method ?? "GET";
+  const pathname = new URL(url).pathname;
+  return consume
+    ? recordMemoryTestRequest(`${method} ${pathname}`, () => SELF.fetch(url, init), consume)
+    : recordMemoryTestRequest(`${method} ${pathname}`, () => SELF.fetch(url, init));
+}

package/dist/assets/deploy/cloudflare-memory/worker.ts

@@ -1674,6 +1674,11 @@ export class RunHistoryDO extends DurableObject<Env> {
         tools_json TEXT NOT NULL,
         state_json TEXT NOT NULL
       );
+      CREATE TABLE IF NOT EXISTS resident_claim_clock (
+        id INTEGER PRIMARY KEY CHECK (id = 1),
+        value INTEGER NOT NULL
+      );
+      INSERT OR IGNORE INTO resident_claim_clock (id, value) VALUES (1, 0);
       CREATE TABLE IF NOT EXISTS run_steps (
         run_id TEXT NOT NULL,
         step INTEGER NOT NULL,
@@ -2920,6 +2925,33 @@ export class RunHistoryDO extends DurableObject<Env> {
     return r ? rowToLive(r) : undefined;
   }
 
+  /** A later ledger owner gets a larger attachment fence, even for the same run ID. */
+  async residentClaim(
+    runId: string,
+    gen: string,
+    threadKey: string,
+  ): Promise<{ ok: true; fence: number } | { ok: false; reason: "fenced" | "unknown-run" }> {
+    let out: { ok: true; fence: number } | { ok: false; reason: "fenced" | "unknown-run" } = {
+      ok: false,
+      reason: "unknown-run",
+    };
+    this.ctx.storage.transactionSync(() => {
+      const row = this.liveRow(runId);
+      if (!row) return;
+      if (row.ownerGen !== gen || row.threadKey !== threadKey) {
+        out = { ok: false, reason: "fenced" };
+        return;
+      }
+      this.sql.exec(`UPDATE resident_claim_clock SET value = value + 1 WHERE id = 1`);
+      const fence = this.sql
+        .exec<{ value: number }>(`SELECT value FROM resident_claim_clock WHERE id = 1`)
+        .toArray()[0]?.value;
+      if (fence === undefined || !Number.isSafeInteger(fence)) throw new Error("resident claim fence unavailable");
+      out = { ok: true, fence };
+    });
+    return out;
+  }
+
   /** The `runs` table's column migrations, run at every construction and
    *  idempotent: a table created before a column existed gains it, with the
    *  value a row written back then should read. The run-visibility stamp: a
@@ -4099,6 +4131,9 @@ export class RunHistoryDO extends DurableObject<Env> {
         userId: row.user_id,
         ...(who.userName ? { userName: who.userName } : {}),
         ...(who.parentRunId ? { parentRunId: who.parentRunId } : {}),
+        ...(who.parentInstanceId ? { parentInstanceId: who.parentInstanceId } : {}),
+        ...(who.idempotencyKey ? { idempotencyKey: who.idempotencyKey } : {}),
+        ...(who.costCapUsd !== undefined ? { costCapUsd: who.costCapUsd } : {}),
         threadKey: row.thread_key,
         channelId: row.channel_id,
         ...(row.agent ? { agent: row.agent } : {}),
@@ -4347,12 +4382,23 @@ function parseUsageJson(raw: string | null): RunUsage | undefined {
 }
 
 /** The two identity fields the by-user aggregate needs off a summary, read leniently. */
-function identityOfSummary(raw: string): { userName?: string; parentRunId?: string } {
+function identityOfSummary(raw: string): {
+  userName?: string;
+  parentRunId?: string;
+  parentInstanceId?: string;
+  idempotencyKey?: string;
+  costCapUsd?: number;
+} {
   try {
     const s = JSON.parse(raw) as Record<string, unknown>;
     return {
       ...(typeof s.userName === "string" && s.userName ? { userName: s.userName } : {}),
       ...(typeof s.parentRunId === "string" && s.parentRunId ? { parentRunId: s.parentRunId } : {}),
+      ...(typeof s.parentInstanceId === "string" && s.parentInstanceId ? { parentInstanceId: s.parentInstanceId } : {}),
+      ...(typeof s.idempotencyKey === "string" && s.idempotencyKey ? { idempotencyKey: s.idempotencyKey } : {}),
+      ...(typeof s.costCapUsd === "number" && Number.isFinite(s.costCapUsd) && s.costCapUsd > 0
+        ? { costCapUsd: s.costCapUsd }
+        : {}),
     };
   } catch {
     return {};
@@ -5318,6 +5364,7 @@ const LEDGER_ROUTES = new Set([
   "/runs/coordinator/wake",
   "/runs/decision-record/reserve",
   "/runs/claim",
+  "/runs/resident-claim",
   "/runs/heartbeat",
   "/runs/append",
   "/runs/live-state",
@@ -5656,6 +5703,14 @@ function parseClaim(b: Record<string, unknown>): Validated<ClaimRequest> {
     if (typeof meta.idempotencyKey !== "string" || !IDEMPOTENCY_KEY_PATTERN.test(meta.idempotencyKey))
       return invalid("run.meta.idempotencyKey must be <parentInstanceId>:<step>");
   }
+  if (
+    meta.costCapUsd !== undefined &&
+    (meta.parentInstanceId === undefined ||
+      typeof meta.costCapUsd !== "number" ||
+      !Number.isFinite(meta.costCapUsd) ||
+      meta.costCapUsd <= 0)
+  )
+    return invalid("run.meta.costCapUsd must be a positive coordinator cap");
   // The restart tag (record 0064) names the predecessor run whose windows the
   // claim reuses and rides the `child-resumed` event's reason: a run id or
   // absent, never another shape.
@@ -5924,6 +5979,16 @@ async function handleLedger(pathname: string, body: unknown, env: Env): Promise<
     );
     return r.ok ? json(r) : json(r, 409);
   }
+  if (pathname === "/runs/resident-claim") {
+    const runId = parseRunId(b.runId);
+    if (!runId.ok) return json({ error: runId.error }, 400);
+    const g = gen(b.gen);
+    if (!g.ok) return json({ error: g.error }, 400);
+    if (typeof b.threadKey !== "string" || b.threadKey.length === 0 || b.threadKey.length > 256)
+      return json({ error: "threadKey must be a non-empty string" }, 400);
+    const result = await stub.residentClaim(runId.value, g.value, b.threadKey);
+    return result.ok ? json(result) : json(result, 409);
+  }
   if (pathname === "/runs/live") return json({ runs: await stub.listLive() });
   if (pathname === "/runs/reclaim") {
     const g = gen(b.gen);

package/dist/assets/deploy/cloudflare-resident/imageReconcileState.ts

@@ -0,0 +1,28 @@
+/** One deploy's durable progress for a resident container. A repeated
+ *  reconcile may retry a deferred stop or collect an inactive container's
+ *  report, but must not stop a replacement that is still hydrating. */
+export interface DeployImageReconcileState {
+  build: string;
+  cycleIssued: boolean;
+  cycleStoppedAt?: string;
+  containerBeforeCycle?: string;
+}
+
+export type DeployImageReconcileAction = "start" | "retry-cycle" | "await-report" | "verified";
+
+export function nextDeployImageReconcile(
+  build: string,
+  stored: unknown,
+  reportPending: boolean,
+): DeployImageReconcileAction {
+  if (!stored || typeof stored !== "object") return "start";
+  const state = stored as Record<string, unknown>;
+  if (state.build !== build || typeof state.cycleIssued !== "boolean") return "start";
+  if (!reportPending) return "verified";
+  return state.cycleIssued ? "await-report" : "retry-cycle";
+}
+
+/** A fresh boot after the requested stop can finish hydration and report. */
+export function replacementContainerStarted(before: string | undefined, current: string | undefined): boolean {
+  return before !== undefined && current !== undefined && before !== current;
+}

package/dist/assets/deploy/cloudflare-resident/runRegistration.ts

@@ -0,0 +1,68 @@
+/** A command is counted separately. Between commands, a registration protects
+ * the run only through its budget or the sweep's inactivity boundary. */
+export function registeredRunNeedsProtection(
+  lastAttachAt: string | undefined,
+  opInFlight: number,
+  cutoff: number,
+  deadlineAt?: number,
+  now?: number,
+): boolean {
+  if (opInFlight > 0) return false; // the operation counter already covers it
+  if (deadlineAt !== undefined && now !== undefined && Number.isSafeInteger(deadlineAt)) return now <= deadlineAt;
+  const last = Date.parse(lastAttachAt ?? "");
+  return !Number.isFinite(last) || last >= cutoff;
+}
+
+/** Only a provably live registration owned by this run can bypass the gates
+ * that keep new work off a draining or stale resident. */
+export function registeredRunAllowsClaim(
+  registration: { runId?: string; ownerGen?: string; ownerFence?: number } | undefined,
+  runId: string | undefined,
+  ownerGen: string | undefined,
+  ownerFence?: number,
+): boolean {
+  if (!registration) return true;
+  if (ownerFence !== undefined && registration.ownerFence !== undefined)
+    return (
+      ownerFence > registration.ownerFence ||
+      (ownerFence === registration.ownerFence && registration.runId === runId && registration.ownerGen === ownerGen)
+    );
+  if (registration.ownerFence !== undefined) return false;
+  if (ownerFence !== undefined) return true; // first ledger-backed claim over a legacy row
+  return registration.runId === runId && registration.ownerGen === ownerGen;
+}
+
+export function registeredRunAllowsReattach(
+  registration: { runId?: string; ownerGen?: string; ownerFence?: number; deadlineAt?: number } | undefined,
+  runId: string | undefined,
+  now: number,
+  graceMs: number,
+  ownerGen?: string,
+  ownerFence?: number,
+): boolean {
+  const deadlineAt = registration?.deadlineAt;
+  return (
+    runId !== undefined &&
+    registration?.runId === runId &&
+    registeredRunAllowsClaim(registration, runId, ownerGen, ownerFence) &&
+    deadlineAt !== undefined &&
+    Number.isSafeInteger(deadlineAt) &&
+    now <= deadlineAt + graceMs
+  );
+}
+
+/** A release may affect only the registration opened by its run. Legacy
+ * callers may release legacy registrations, but cannot release a named run. */
+export function registeredRunOwnsRelease(
+  registration: { runId?: string; ownerGen?: string; ownerFence?: number } | undefined,
+  runId: string | undefined,
+  ownerGen?: string,
+  ownerFence?: number,
+): boolean {
+  return (
+    registration !== undefined &&
+    registration.runId === runId &&
+    registration.ownerGen === ownerGen &&
+    registration.ownerFence === ownerFence
+  );
+}

package/dist/assets/deploy/cloudflare-resident/shared.ts

@@ -30,9 +30,8 @@ export const REFRESH_INTERVAL_S = 600;
  *  ten-minute buckets since the last instance. */
 export const IDLE_REFRESH_INTERVAL_S = 6 * 60 * 60;
 
-/** The thread user pool the image carries (`worker2`..`worker17`): one OS
- *  user per attached thread, and the bound on how many bindings one sweep
- *  step can have to check. */
+/** The image's `worker2`..`worker17` pool: 16 one-use identities per VM
+ *  generation for thread owners and disposable operations. */
 export const THREAD_POOL_SIZE = 16;
 
 /** Exec budgets. Each is a refresh step's own budget too (refresh.ts), so a