deepline 0.1.243 → 0.1.244

package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts

@@ -1,1386 +0,0 @@
-/**
- * PlayDedup Durable Object — cross-attempt idempotency cache for one play run.
- *
- * One DO instance per runId. Workers and queue-consumers fetch into it to:
- *   - claim a hash to compute (atomic; first claimer wins)
- *   - publish a result (unblocks awaiters)
- *   - long-poll await for a result claimed by another caller
- *
- * Storage shape (DO storage):
- *   key:  `${toolId}:${inputHash}`
- *   value: DedupEntry
- *
- * The whole DO is evicted via `/finish` at run end. Storage is single-region
- * but strongly-consistent within the DO instance — exactly what dedup needs.
- *
- * Latency: ~5ms per request (DO storage is sub-ms; HTTP add ~3-5ms).
- */
-
-import { sanitizeLiveLogLines } from './runtime/live-progress';
-import {
-  DB_SESSION_MAX_TTL_SECONDS,
-  type PreloadedRuntimeDbSession,
-} from '../../../shared_libs/play-runtime/db-session';
-import { createSecretRedactionContext } from '../../../shared_libs/play-runtime/secret-redaction';
-
-const durableStorageRedactor = createSecretRedactionContext();
-
-type DurableObjectStorage = {
-  get<T>(key: string): Promise<T | undefined>;
-  get<T>(keys: string[]): Promise<Map<string, T>>;
-  put<T>(key: string, value: T): Promise<void>;
-  put<T>(entries: Record<string, T>): Promise<void>;
-  delete(key: string | string[]): Promise<unknown>;
-  deleteAll(): Promise<void>;
-  list<T>(options?: { prefix?: string }): Promise<Map<string, T>>;
-  setAlarm(scheduledTime: number): Promise<void>;
-};
-
-type DurableObjectState = {
-  storage: DurableObjectStorage;
-  blockConcurrencyWhile<T>(callback: () => Promise<T>): Promise<T>;
-};
-
-type DurableObject = {
-  fetch(request: Request): Promise<Response>;
-  alarm?(): Promise<void>;
-};
-
-type DurableObjectId = {
-  toString(): string;
-};
-
-type DurableObjectStub = {
-  fetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response>;
-};
-
-type DurableObjectNamespace = {
-  idFromName(name: string): DurableObjectId;
-  get(id: DurableObjectId): DurableObjectStub;
-};
-
-type DedupEntry =
-  | { status: 'in_flight'; claimedBy: string; claimedAt: number }
-  | { status: 'completed'; result: unknown; completedAt: number }
-  | { status: 'failed'; error: string; failedAt: number };
-
-type LookupOrClaimRequest = {
-  runId: string;
-  toolId: string;
-  inputHashes: string[];
-  /** Identifier for the caller — used as `claimedBy` for in-flight entries. */
-  callerId: string;
-};
-
-type LookupOrClaimResponse = {
-  hits: Record<string, unknown>;
-  claimed: string[];
-  inFlight: string[];
-};
-
-type PublishRequest = {
-  runId: string;
-  toolId: string;
-  results: Array<{ inputHash: string; result: unknown }>;
-};
-
-type AwaitRequest = {
-  runId: string;
-  toolId: string;
-  inputHashes: string[];
-  timeoutMs: number;
-};
-
-type WorkflowRunRetryState = {
-  runId: string;
-  params: unknown;
-  paramsRef?: unknown;
-  paramsBytes?: number;
-  retryAttempts: number;
-  updatedAt: number;
-  expiresAt: number;
-};
-
-type WorkflowInstanceState = {
-  runId: string;
-  instanceId: string;
-  updatedAt: number;
-  expiresAt: number;
-};
-
-type WorkflowSubmitClaimState = {
-  runId: string;
-  status: 'creating' | 'created';
-  instanceId?: string;
-  claimedAt: number;
-  updatedAt: number;
-  expiresAt: number;
-};
-
-type WorkflowDbSessionsState = {
-  runId: string;
-  sessions: PreloadedRuntimeDbSession[];
-  storedAt: number;
-  expiresAt: number;
-};
-
-type CoordinatorTraceEntry = {
-  ts: number;
-  source: 'coordinator' | 'dynamic_worker';
-  runId: string;
-  phase: string;
-  ms: number;
-  graphHash?: string | null;
-  [key: string]: unknown;
-};
-
-type CoordinatorTerminalState = {
-  runId: string;
-  status: 'completed' | 'failed' | 'cancelled';
-  result?: unknown;
-  error?: string | null;
-  totalRows?: unknown;
-  durationMs?: unknown;
-  playName?: string | null;
-  liveLogs?: string[] | null;
-  liveNodeProgress?: unknown;
-  completedAt: number;
-};
-
-type CoordinatorRunEvent =
-  | {
-      seq: number;
-      runId: string;
-      type: 'status';
-      status: string;
-      ts: number;
-      logs?: string[];
-    }
-  | {
-      seq: number;
-      runId: string;
-      type: 'log';
-      line: string;
-      ts: number;
-    }
-  | {
-      seq: number;
-      runId: string;
-      type: 'progress';
-      status: string;
-      ts: number;
-      logs?: string[];
-      activeNodeId?: string | null;
-      activeArtifactTableNamespace?: string | null;
-      updatedAt?: number | null;
-      liveNodeProgress?: unknown;
-    }
-  | {
-      seq: number;
-      runId: string;
-      type: 'terminal';
-      status: 'completed' | 'failed' | 'cancelled';
-      ts: number;
-      result?: unknown;
-      error?: string | null;
-      totalRows?: unknown;
-      durationMs?: unknown;
-      playName?: string | null;
-      liveLogs?: string[] | null;
-      liveNodeProgress?: unknown;
-    };
-
-type OmitRunEventSequence<T> = T extends unknown ? Omit<T, 'seq'> : never;
-type CoordinatorRunEventInput = OmitRunEventSequence<CoordinatorRunEvent>;
-
-/**
- * Per-(org,provider) rate-state for the distributed Rate State Backend.
- *
- * One PlayDedup DO instance is addressed per bucket via
- * `idFromName('rate:<orgId>:<provider>')`, so a single instance owns the rate
- * window for that bucket and is single-threaded — which is exactly why the
- * in-process algorithm from `InMemoryRateStateBackend` is correct here. The
- * coordinator RPCs `/rate-acquire` (lease N permits) and `/rate-penalize`
- * (Retry-After cooldown) into it; the esm_workers runtime never blocks per call
- * on a full round-trip because it leases small permit BLOCKS at a time.
- *
- * Rate windows are intentionally kept in DO memory (not durable storage): a
- * window is sub-second-to-seconds ephemeral state, the DO instance outlives any
- * single window, and persisting it would only add storage latency to the hot
- * path. This mirrors the in-memory backend and the Redis sliding-window limiter
- * (`src/lib/redis/customer-rate-limiter.ts`), which also keeps window state in a
- * volatile store with TTL rather than durable rows.
- */
-type RateRule = {
-  ruleId: string;
-  requestsPerWindow: number;
-  windowMs: number;
-  maxConcurrency: number | null;
-};
-
-type RateRuleWindowState = {
-  windowStartedAt: number;
-  startedInWindow: number;
-};
-
-type RateAcquireRequest = {
-  bucketId: string;
-  rules: RateRule[];
-  /** How many permits the caller wants to lease in this round-trip. */
-  requested: number;
-};
-
-type RateAcquireResponse = {
-  /** Permits actually granted (0..requested). */
-  granted: number;
-  /** Suggested wait before the next acquire when granted === 0. */
-  waitMs: number;
-};
-
-type RatePenalizeRequest = {
-  bucketId: string;
-  cooldownMs: number;
-};
-
-const RATE_MIN_WAIT_MS = 10;
-const RATE_STATE_KEY = (bucketId: string, ruleId: string) =>
-  `${bucketId}::${ruleId}`;
-
-const DEDUP_KEY_PREFIX = 'd:';
-const WORKFLOW_RUN_RETRY_KEY_PREFIX = 'r:';
-const WORKFLOW_DB_SESSIONS_KEY = 'db-sessions';
-const COORDINATOR_TRACE_KEY_PREFIX = 't:';
-const COORDINATOR_RUN_EVENT_KEY_PREFIX = 'e:';
-const COORDINATOR_TERMINAL_KEY = 'terminal';
-const COORDINATOR_RUN_EVENT_SEQUENCE_KEY = 'event-seq';
-const COORDINATOR_TRACE_MAX_ENTRIES = 200;
-const COORDINATOR_RUN_EVENT_MAX_ENTRIES = 500;
-const FINISH_ALARM_DELAY_MS = 60_000; // self-evict 1 min after finish() called
-const WORKFLOW_RUN_STATE_TTL_MS = 60 * 60 * 1000;
-const WORKFLOW_RUN_RETRY_STATE_MAX_BYTES = 110_000;
-// Invariant: DB sessions must not expire before the workflow run/retry state.
-// hydrateWorkflowDbSessions re-reads sessions on every workflow re-entry, so a
-// shorter session TTL used to kill any run that ran longer than the session
-// window (10 min) with a misleading transport 410.
-const WORKFLOW_DB_SESSIONS_TTL_MS = WORKFLOW_RUN_STATE_TTL_MS;
-const WORKFLOW_DB_SESSIONS_MAX_TTL_MS = DB_SESSION_MAX_TTL_SECONDS * 1000;
-
-function jsonByteLength(value: unknown): number {
-  return new TextEncoder().encode(JSON.stringify(value)).byteLength;
-}
-
-function assertEncryptedDbSessionsForStorage(
-  sessions: PreloadedRuntimeDbSession[],
-): void {
-  for (const session of sessions) {
-    if (session.session.postgresUrl) {
-      throw new Error('preloaded db session storage rejects raw Postgres URLs');
-    }
-    if (!session.session.encryptedPostgresUrl) {
-      throw new Error(
-        'preloaded db session storage requires encrypted Postgres URLs',
-      );
-    }
-  }
-}
-
-interface DedupEnv {
-  PLAY_DEDUP: DurableObjectNamespace;
-}
-
-export class PlayDedup implements DurableObject {
-  // Promises waiting on /await endpoints. Keyed by `${toolId}:${inputHash}`.
-  // Resolved by /publish; rejected on timeout.
-  private waiters: Map<string, Set<(value: unknown) => void>> = new Map();
-  private runEventWaiters: Set<(value: CoordinatorRunEvent) => void> =
-    new Set();
-  // Per-(bucketId, ruleId) request-window state for the rate-state backend.
-  // In-memory by design — see the RateRule doc block. Single-threaded DO access
-  // makes this the same sliding-window algorithm as InMemoryRateStateBackend.
-  private rateWindows: Map<string, RateRuleWindowState> = new Map();
-  // Per-bucket Retry-After cooldown floors (penalize()).
-  private rateCooldownUntil: Map<string, number> = new Map();
-
-  constructor(
-    private readonly state: DurableObjectState,
-    private readonly env: DedupEnv,
-  ) {}
-
-  private storageKey(toolId: string, inputHash: string): string {
-    return `${DEDUP_KEY_PREFIX}${toolId}:${inputHash}`;
-  }
-
-  private traceKey(entry: CoordinatorTraceEntry): string {
-    return `${COORDINATOR_TRACE_KEY_PREFIX}${String(entry.ts).padStart(13, '0')}:${crypto.randomUUID().slice(0, 8)}`;
-  }
-
-  private runEventKey(event: CoordinatorRunEvent): string {
-    return `${COORDINATOR_RUN_EVENT_KEY_PREFIX}${String(event.seq).padStart(13, '0')}`;
-  }
-
-  private runEventKeyForSeq(seq: number): string {
-    return `${COORDINATOR_RUN_EVENT_KEY_PREFIX}${String(seq).padStart(13, '0')}`;
-  }
-
-  private waiterKey(toolId: string, inputHash: string): string {
-    return `${toolId}:${inputHash}`;
-  }
-
-  async fetch(req: Request): Promise<Response> {
-    const url = new URL(req.url);
-    try {
-      switch (url.pathname) {
-        case '/lookup-or-claim':
-          return await this.handleLookupOrClaim(req);
-        case '/publish':
-          return await this.handlePublish(req);
-        case '/await':
-          return await this.handleAwait(req);
-        case '/finish':
-          return await this.handleFinish(req);
-        case '/debug':
-          return await this.handleDebug(req);
-        case '/run-retry-state-put':
-          return await this.handleRunRetryStatePut(req);
-        case '/run-launch-state-put':
-          return await this.handleRunLaunchStatePut(req);
-        case '/workflow-instance-put':
-          return await this.handleWorkflowInstancePut(req);
-        case '/workflow-instance-get':
-          return await this.handleWorkflowInstanceGet(req);
-        case '/workflow-submit-claim':
-          return await this.handleWorkflowSubmitClaim(req);
-        case '/workflow-submit-release':
-          return await this.handleWorkflowSubmitRelease(req);
-        case '/run-retry-claim':
-          return await this.handleRunRetryClaim(req);
-        case '/db-sessions-put':
-          return await this.handleDbSessionsPut(req);
-        case '/db-sessions-get':
-          return await this.handleDbSessionsGet(req);
-        case '/trace-add':
-          return await this.handleTraceAdd(req);
-        case '/trace-list':
-          return await this.handleTraceList();
-        case '/event-add':
-          return await this.handleRunEventAdd(req);
-        case '/event-list':
-          return await this.handleRunEventList(req);
-        case '/terminal-set':
-          return await this.handleTerminalSet(req);
-        case '/terminal-get':
-          return await this.handleTerminalGet();
-        case '/rate-acquire':
-          return await this.handleRateAcquire(req);
-        case '/rate-penalize':
-          return await this.handleRatePenalize(req);
-        default:
-          return new Response('not found', { status: 404 });
-      }
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      return new Response(JSON.stringify({ error: message }), {
-        status: 500,
-        headers: { 'content-type': 'application/json' },
-      });
-    }
-  }
-
-  async alarm(): Promise<void> {
-    // Fired after /finish was called. Evict storage and let the DO instance
-    // be garbage-collected by Cloudflare.
-    await this.state.storage.deleteAll();
-    this.waiters.clear();
-    this.runEventWaiters.clear();
-  }
-
-  private async handleLookupOrClaim(req: Request): Promise<Response> {
-    const body = (await req.json()) as LookupOrClaimRequest;
-    const hits: Record<string, unknown> = {};
-    const claimed: string[] = [];
-    const inFlight: string[] = [];
-
-    // Atomic transaction: read all keys, decide outcome per hash, write claims
-    // back. DO storage transactions are serialized within the DO instance.
-    await this.state.blockConcurrencyWhile(async () => {
-      const keys = body.inputHashes.map((h) => this.storageKey(body.toolId, h));
-      const existing = (await this.state.storage.get<DedupEntry>(keys)) as Map<
-        string,
-        DedupEntry
-      >;
-      const now = Date.now();
-      const toWrite = new Map<string, DedupEntry>();
-
-      for (let i = 0; i < body.inputHashes.length; i += 1) {
-        const hash = body.inputHashes[i]!;
-        const key = keys[i]!;
-        const entry = existing.get(key);
-        if (!entry) {
-          // Claim it for the caller.
-          claimed.push(hash);
-          toWrite.set(key, {
-            status: 'in_flight',
-            claimedBy: body.callerId,
-            claimedAt: now,
-          });
-        } else if (entry.status === 'completed') {
-          hits[hash] = entry.result;
-        } else if (entry.status === 'failed') {
-          // Re-claim failed entries — the caller can retry.
-          claimed.push(hash);
-          toWrite.set(key, {
-            status: 'in_flight',
-            claimedBy: body.callerId,
-            claimedAt: now,
-          });
-        } else {
-          // Already in-flight by someone else; tell caller to await.
-          inFlight.push(hash);
-        }
-      }
-
-      if (toWrite.size > 0) {
-        await this.state.storage.put(Object.fromEntries(toWrite));
-      }
-    });
-
-    const response: LookupOrClaimResponse = { hits, claimed, inFlight };
-    return new Response(JSON.stringify(response), {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handlePublish(req: Request): Promise<Response> {
-    const body = (await req.json()) as PublishRequest;
-    const now = Date.now();
-    const writes: Record<string, DedupEntry> = {};
-    for (const { inputHash, result } of body.results) {
-      writes[this.storageKey(body.toolId, inputHash)] = {
-        status: 'completed',
-        result,
-        completedAt: now,
-      };
-    }
-    await this.state.storage.put(writes);
-
-    // Wake any awaiters.
-    for (const { inputHash, result } of body.results) {
-      const key = this.waiterKey(body.toolId, inputHash);
-      const set = this.waiters.get(key);
-      if (set) {
-        for (const resolve of set) resolve(result);
-        this.waiters.delete(key);
-      }
-    }
-
-    return new Response('{}', {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handleAwait(req: Request): Promise<Response> {
-    const body = (await req.json()) as AwaitRequest;
-    const results: Record<string, unknown> = {};
-    const stillWaiting: string[] = [];
-
-    // Synchronous read first.
-    const keys = body.inputHashes.map((h) => this.storageKey(body.toolId, h));
-    const initial = (await this.state.storage.get<DedupEntry>(keys)) as Map<
-      string,
-      DedupEntry
-    >;
-    for (let i = 0; i < body.inputHashes.length; i += 1) {
-      const hash = body.inputHashes[i]!;
-      const entry = initial.get(keys[i]!);
-      if (entry?.status === 'completed') {
-        results[hash] = entry.result;
-      } else {
-        stillWaiting.push(hash);
-      }
-    }
-
-    if (stillWaiting.length === 0) {
-      return new Response(JSON.stringify({ results }), {
-        headers: { 'content-type': 'application/json' },
-      });
-    }
-
-    // Long-poll. Register waiter promises; resolve via /publish or timeout.
-    const promises = stillWaiting.map(
-      (hash) =>
-        new Promise<{ hash: string; result: unknown }>((resolve, reject) => {
-          const wKey = this.waiterKey(body.toolId, hash);
-          if (!this.waiters.has(wKey)) this.waiters.set(wKey, new Set());
-          const onResolve = (result: unknown) => resolve({ hash, result });
-          this.waiters.get(wKey)!.add(onResolve);
-          setTimeout(() => {
-            this.waiters.get(wKey)?.delete(onResolve);
-            reject(new Error(`Dedup await timeout for ${hash}`));
-          }, body.timeoutMs);
-        }),
-    );
-
-    try {
-      const settled = await Promise.allSettled(promises);
-      for (const s of settled) {
-        if (s.status === 'fulfilled') {
-          results[s.value.hash] = s.value.result;
-        }
-      }
-    } catch {
-      // Individual timeouts are not fatal; return whatever resolved.
-    }
-
-    return new Response(JSON.stringify({ results }), {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handleFinish(_req: Request): Promise<Response> {
-    // Schedule alarm to evict after a grace period (any straggler awaits
-    // get a chance to resolve). Grace = 60s.
-    await this.state.storage.setAlarm(Date.now() + FINISH_ALARM_DELAY_MS);
-    return new Response('{}', {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handleDebug(_req: Request): Promise<Response> {
-    const all = await this.state.storage.list({ prefix: DEDUP_KEY_PREFIX });
-    const dump: Record<string, DedupEntry> = {};
-    for (const [k, v] of all) dump[k] = v as DedupEntry;
-    return new Response(
-      JSON.stringify({
-        size: all.size,
-        entries: dump,
-        waiters: this.waiters.size,
-      }),
-      { headers: { 'content-type': 'application/json' } },
-    );
-  }
-
-  private async handleRunRetryStatePut(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as {
-      runId?: unknown;
-      params?: unknown;
-      ttlMs?: unknown;
-    } | null;
-    const runId = typeof body?.runId === 'string' ? body.runId : '';
-    if (!runId || !body || (!('params' in body) && !('paramsRef' in body))) {
-      return new Response('runId and params or paramsRef are required', {
-        status: 400,
-      });
-    }
-    const now = Date.now();
-    const ttlMs =
-      typeof body.ttlMs === 'number' && Number.isFinite(body.ttlMs)
-        ? Math.max(60_000, Math.min(body.ttlMs, WORKFLOW_RUN_STATE_TTL_MS))
-        : WORKFLOW_RUN_STATE_TTL_MS;
-    const key = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
-    await this.state.blockConcurrencyWhile(async () => {
-      const existing = await this.state.storage.get<WorkflowRunRetryState>(key);
-      const retryState = {
-        runId,
-        params: 'params' in body ? body.params : null,
-        paramsRef: 'paramsRef' in body ? body.paramsRef : null,
-        paramsBytes:
-          typeof (body as { paramsBytes?: unknown }).paramsBytes === 'number' &&
-          Number.isFinite((body as { paramsBytes?: number }).paramsBytes)
-            ? (body as { paramsBytes: number }).paramsBytes
-            : undefined,
-        retryAttempts:
-          existing?.runId === runId &&
-          typeof existing.retryAttempts === 'number'
-            ? existing.retryAttempts
-            : 0,
-        updatedAt: now,
-        expiresAt: now + ttlMs,
-      } satisfies WorkflowRunRetryState;
-      const bytes = jsonByteLength(retryState);
-      if (bytes > WORKFLOW_RUN_RETRY_STATE_MAX_BYTES) {
-        throw new Error(
-          `workflow retry state too large: ${bytes} bytes exceeds ${WORKFLOW_RUN_RETRY_STATE_MAX_BYTES}`,
-        );
-      }
-      await this.state.storage.put(key, retryState);
-    });
-    return new Response(JSON.stringify({ ok: true }), {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handleRunLaunchStatePut(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as {
-      runId?: unknown;
-      params?: unknown;
-      paramsRef?: unknown;
-      paramsBytes?: unknown;
-      sessions?: unknown;
-      retryTtlMs?: unknown;
-      dbSessionsTtlMs?: unknown;
-    } | null;
-    const runId = typeof body?.runId === 'string' ? body.runId : '';
-    const sessions = Array.isArray(body?.sessions)
-      ? (body.sessions as PreloadedRuntimeDbSession[])
-      : null;
-    if (
-      !runId ||
-      !body ||
-      (!('params' in body) && !('paramsRef' in body)) ||
-      !sessions
-    ) {
-      return new Response(
-        'runId, params or paramsRef, and sessions are required',
-        {
-          status: 400,
-        },
-      );
-    }
-    assertEncryptedDbSessionsForStorage(sessions);
-    const now = Date.now();
-    const retryTtlMs =
-      typeof body.retryTtlMs === 'number' && Number.isFinite(body.retryTtlMs)
-        ? Math.max(60_000, Math.min(body.retryTtlMs, WORKFLOW_RUN_STATE_TTL_MS))
-        : WORKFLOW_RUN_STATE_TTL_MS;
-    const dbSessionsTtlMs =
-      typeof body.dbSessionsTtlMs === 'number' &&
-      Number.isFinite(body.dbSessionsTtlMs) &&
-      body.dbSessionsTtlMs > 0
-        ? Math.min(
-            Math.max(Math.floor(body.dbSessionsTtlMs), 60_000),
-            WORKFLOW_DB_SESSIONS_MAX_TTL_MS,
-          )
-        : WORKFLOW_DB_SESSIONS_TTL_MS;
-    const retryKey = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
-    const dbSessionsState: WorkflowDbSessionsState = {
-      runId,
-      sessions,
-      storedAt: now,
-      expiresAt: now + dbSessionsTtlMs,
-    };
-    let retryStateExpiresAt = now + retryTtlMs;
-    await this.state.blockConcurrencyWhile(async () => {
-      const existing =
-        await this.state.storage.get<WorkflowRunRetryState>(retryKey);
-      const retryState = {
-        runId,
-        params: 'params' in body ? body.params : null,
-        paramsRef: 'paramsRef' in body ? body.paramsRef : null,
-        paramsBytes:
-          typeof body.paramsBytes === 'number' &&
-          Number.isFinite(body.paramsBytes)
-            ? body.paramsBytes
-            : undefined,
-        retryAttempts:
-          existing?.runId === runId &&
-          typeof existing.retryAttempts === 'number'
-            ? existing.retryAttempts
-            : 0,
-        updatedAt: now,
-        expiresAt: retryStateExpiresAt,
-      } satisfies WorkflowRunRetryState;
-      const bytes = jsonByteLength(retryState);
-      if (bytes > WORKFLOW_RUN_RETRY_STATE_MAX_BYTES) {
-        throw new Error(
-          `workflow retry state too large: ${bytes} bytes exceeds ${WORKFLOW_RUN_RETRY_STATE_MAX_BYTES}`,
-        );
-      }
-      retryStateExpiresAt = retryState.expiresAt;
-      await this.state.storage.put(retryKey, retryState);
-      await this.state.storage.put(WORKFLOW_DB_SESSIONS_KEY, dbSessionsState);
-    });
-    return new Response(
-      JSON.stringify({
-        ok: true,
-        sessionCount: sessions.length,
-        retryExpiresAt: retryStateExpiresAt,
-        dbSessionsExpiresAt: dbSessionsState.expiresAt,
-      }),
-      { headers: { 'content-type': 'application/json' } },
-    );
-  }
-
-  private async handleRunRetryClaim(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as {
-      runId?: unknown;
-      maxAttempts?: unknown;
-    } | null;
-    const runId = typeof body?.runId === 'string' ? body.runId : '';
-    if (!runId) {
-      return new Response('runId is required', { status: 400 });
-    }
-    const maxAttempts =
-      typeof body?.maxAttempts === 'number' && Number.isFinite(body.maxAttempts)
-        ? Math.max(0, Math.floor(body.maxAttempts))
-        : 1;
-    const now = Date.now();
-    const key = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
-    let response: Record<string, unknown> = {
-      claimed: false,
-      attempts: 0,
-      params: null,
-    };
-    await this.state.blockConcurrencyWhile(async () => {
-      const existing = await this.state.storage.get<WorkflowRunRetryState>(key);
-      if (!existing || existing.expiresAt <= now) {
-        response = { claimed: false, attempts: 0, params: null };
-        return;
-      }
-      if (existing.retryAttempts >= maxAttempts) {
-        response = {
-          claimed: false,
-          attempts: existing.retryAttempts,
-          params: existing.params,
-          paramsRef: existing.paramsRef ?? null,
-          paramsBytes: existing.paramsBytes ?? null,
-        };
-        return;
-      }
-      const nextAttempts = existing.retryAttempts + 1;
-      await this.state.storage.put(key, {
-        ...existing,
-        retryAttempts: nextAttempts,
-        updatedAt: now,
-        expiresAt: now + WORKFLOW_RUN_STATE_TTL_MS,
-      } satisfies WorkflowRunRetryState);
-      response = {
-        claimed: true,
-        attempts: nextAttempts,
-        params: existing.params,
-        paramsRef: existing.paramsRef ?? null,
-        paramsBytes: existing.paramsBytes ?? null,
-      };
-    });
-    return new Response(JSON.stringify(response), {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handleWorkflowInstancePut(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as {
-      runId?: unknown;
-      instanceId?: unknown;
-      ttlMs?: unknown;
-    } | null;
-    const runId = typeof body?.runId === 'string' ? body.runId : '';
-    const instanceId =
-      typeof body?.instanceId === 'string' ? body.instanceId : '';
-    if (!runId || !instanceId) {
-      return new Response('runId and instanceId are required', { status: 400 });
-    }
-    const now = Date.now();
-    const ttlMs =
-      typeof body?.ttlMs === 'number' &&
-      Number.isFinite(body.ttlMs) &&
-      body.ttlMs > 0
-        ? Math.max(60_000, Math.min(body.ttlMs, WORKFLOW_RUN_STATE_TTL_MS))
-        : WORKFLOW_RUN_STATE_TTL_MS;
-    await this.state.storage.put({
-      'workflow-instance': {
-        runId,
-        instanceId,
-        updatedAt: now,
-        expiresAt: now + ttlMs,
-      } satisfies WorkflowInstanceState,
-      'workflow-submit-claim': {
-        runId,
-        status: 'created',
-        instanceId,
-        claimedAt: now,
-        updatedAt: now,
-        expiresAt: now + ttlMs,
-      } satisfies WorkflowSubmitClaimState,
-    });
-    return new Response(JSON.stringify({ ok: true }), {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handleWorkflowInstanceGet(req: Request): Promise<Response> {
-    const runId = new URL(req.url).searchParams.get('runId') ?? '';
-    if (!runId) {
-      return new Response('runId is required', { status: 400 });
-    }
-    const state =
-      await this.state.storage.get<WorkflowInstanceState>('workflow-instance');
-    if (!state || state.runId !== runId) {
-      return new Response(JSON.stringify({ instanceId: null }), {
-        headers: { 'content-type': 'application/json' },
-      });
-    }
-    if (state.expiresAt <= Date.now()) {
-      await this.state.storage.delete('workflow-instance');
-      return new Response(JSON.stringify({ instanceId: null }), {
-        headers: { 'content-type': 'application/json' },
-      });
-    }
-    return new Response(JSON.stringify({ instanceId: state.instanceId }), {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handleWorkflowSubmitClaim(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as {
-      runId?: unknown;
-      ttlMs?: unknown;
-    } | null;
-    const runId = typeof body?.runId === 'string' ? body.runId : '';
-    if (!runId) {
-      return new Response('runId is required', { status: 400 });
-    }
-    const now = Date.now();
-    const ttlMs =
-      typeof body?.ttlMs === 'number' &&
-      Number.isFinite(body.ttlMs) &&
-      body.ttlMs > 0
-        ? Math.max(60_000, Math.min(body.ttlMs, WORKFLOW_RUN_STATE_TTL_MS))
-        : WORKFLOW_RUN_STATE_TTL_MS;
-    let response: {
-      claimed: boolean;
-      status: WorkflowSubmitClaimState['status'];
-      instanceId: string | null;
-    } = { claimed: true, status: 'creating', instanceId: null };
-    await this.state.blockConcurrencyWhile(async () => {
-      const key = 'workflow-submit-claim';
-      const existing =
-        await this.state.storage.get<WorkflowSubmitClaimState>(key);
-      if (existing?.runId === runId && existing.expiresAt > now) {
-        response = {
-          claimed: false,
-          status: existing.status,
-          instanceId: existing.instanceId ?? null,
-        };
-        return;
-      }
-      const state = {
-        runId,
-        status: 'creating',
-        claimedAt: now,
-        updatedAt: now,
-        expiresAt: now + ttlMs,
-      } satisfies WorkflowSubmitClaimState;
-      await this.state.storage.put(key, state);
-    });
-    return new Response(JSON.stringify(response), {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handleWorkflowSubmitRelease(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as {
-      runId?: unknown;
-    } | null;
-    const runId = typeof body?.runId === 'string' ? body.runId : '';
-    if (!runId) {
-      return new Response('runId is required', { status: 400 });
-    }
-    let released = false;
-    await this.state.blockConcurrencyWhile(async () => {
-      const key = 'workflow-submit-claim';
-      const existing =
-        await this.state.storage.get<WorkflowSubmitClaimState>(key);
-      if (existing?.runId === runId && existing.status === 'creating') {
-        await this.state.storage.delete(key);
-        released = true;
-      }
-    });
-    return new Response(JSON.stringify({ ok: true, released }), {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handleDbSessionsPut(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as {
-      runId?: unknown;
-      sessions?: unknown;
-      ttlMs?: unknown;
-    } | null;
-    const runId = typeof body?.runId === 'string' ? body.runId : '';
-    const sessions = Array.isArray(body?.sessions)
-      ? (body.sessions as PreloadedRuntimeDbSession[])
-      : null;
-    if (!runId || !sessions) {
-      return new Response('runId and sessions are required', { status: 400 });
-    }
-    assertEncryptedDbSessionsForStorage(sessions);
-    const now = Date.now();
-    const ttlMs =
-      typeof body?.ttlMs === 'number' &&
-      Number.isFinite(body.ttlMs) &&
-      body.ttlMs > 0
-        ? Math.min(
-            Math.max(Math.floor(body.ttlMs), 60_000),
-            WORKFLOW_DB_SESSIONS_MAX_TTL_MS,
-          )
-        : WORKFLOW_DB_SESSIONS_TTL_MS;
-    const state: WorkflowDbSessionsState = {
-      runId,
-      sessions,
-      storedAt: now,
-      expiresAt: now + ttlMs,
-    };
-    await this.state.storage.put(WORKFLOW_DB_SESSIONS_KEY, state);
-    return new Response(
-      JSON.stringify({
-        ok: true,
-        sessionCount: sessions.length,
-        expiresAt: state.expiresAt,
-      }),
-      { headers: { 'content-type': 'application/json' } },
-    );
-  }
-
-  private async handleDbSessionsGet(req: Request): Promise<Response> {
-    const url = new URL(req.url);
-    const runId = url.searchParams.get('runId') ?? '';
-    if (!runId) {
-      return new Response('runId is required', { status: 400 });
-    }
-    const state = await this.state.storage.get<WorkflowDbSessionsState>(
-      WORKFLOW_DB_SESSIONS_KEY,
-    );
-    if (!state || state.runId !== runId) {
-      return new Response(JSON.stringify({ error: 'db sessions not found' }), {
-        status: 404,
-        headers: { 'content-type': 'application/json' },
-      });
-    }
-    if (state.expiresAt <= Date.now()) {
-      await this.state.storage.delete(WORKFLOW_DB_SESSIONS_KEY);
-      return new Response(
-        JSON.stringify({
-          error:
-            'workflow db sessions expired: the run exceeded the session retention window before completing. This usually follows an earlier runtime persistence failure or a stalled run; re-run to resume.',
-        }),
-        {
-          status: 410,
-          headers: { 'content-type': 'application/json' },
-        },
-      );
-    }
-    assertEncryptedDbSessionsForStorage(state.sessions);
-    return new Response(
-      JSON.stringify({
-        sessions: state.sessions,
-        sessionCount: state.sessions.length,
-        expiresAt: state.expiresAt,
-      }),
-      { headers: { 'content-type': 'application/json' } },
-    );
-  }
-
-  private async handleTraceAdd(req: Request): Promise<Response> {
-    const body = (await req
-      .json()
-      .catch(() => null)) as Partial<CoordinatorTraceEntry> | null;
-    if (
-      !body ||
-      (body.source !== 'coordinator' && body.source !== 'dynamic_worker') ||
-      typeof body.runId !== 'string' ||
-      typeof body.phase !== 'string' ||
-      typeof body.ts !== 'number' ||
-      typeof body.ms !== 'number'
-    ) {
-      return new Response('invalid trace entry', { status: 400 });
-    }
-    const entry = durableStorageRedactor.redact({
-      ...body,
-      source: body.source,
-      runId: body.runId,
-      phase: body.phase,
-      ts: body.ts,
-      ms: body.ms,
-      graphHash:
-        typeof body.graphHash === 'string' || body.graphHash === null
-          ? body.graphHash
-          : undefined,
-    }) as CoordinatorTraceEntry;
-    await this.state.blockConcurrencyWhile(async () => {
-      await this.state.storage.put(this.traceKey(entry), entry);
-      const entries = await this.state.storage.list<CoordinatorTraceEntry>({
-        prefix: COORDINATOR_TRACE_KEY_PREFIX,
-      });
-      const overflow = entries.size - COORDINATOR_TRACE_MAX_ENTRIES;
-      if (overflow > 0) {
-        await this.state.storage.delete([...entries.keys()].slice(0, overflow));
-      }
-    });
-    return new Response('{}', {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handleTraceList(): Promise<Response> {
-    const entries = await this.state.storage.list<CoordinatorTraceEntry>({
-      prefix: COORDINATOR_TRACE_KEY_PREFIX,
-    });
-    return new Response(
-      JSON.stringify({
-        entries: [...entries.values()].sort(
-          (left, right) => left.ts - right.ts,
-        ),
-      }),
-      { headers: { 'content-type': 'application/json' } },
-    );
-  }
-
-  private async handleTerminalSet(req: Request): Promise<Response> {
-    const body = (await req
-      .json()
-      .catch(() => null)) as Partial<CoordinatorTerminalState> | null;
-    if (
-      !body ||
-      typeof body.runId !== 'string' ||
-      (body.status !== 'completed' &&
-        body.status !== 'failed' &&
-        body.status !== 'cancelled')
-    ) {
-      return new Response('invalid terminal state', { status: 400 });
-    }
-    const state: CoordinatorTerminalState = {
-      runId: body.runId,
-      status: body.status,
-      result: body.result,
-      error:
-        typeof body.error === 'string' && body.error.trim()
-          ? body.error.trim()
-          : body.status === 'failed'
-            ? 'Cloudflare workflow failed before the runner reported an error.'
-            : null,
-      totalRows: body.totalRows,
-      durationMs: body.durationMs,
-      playName: typeof body.playName === 'string' ? body.playName : null,
-      liveLogs: sanitizeLiveLogLines(body.liveLogs),
-      liveNodeProgress: body.liveNodeProgress,
-      completedAt:
-        typeof body.completedAt === 'number' ? body.completedAt : Date.now(),
-    };
-    await this.state.storage.put(COORDINATOR_TERMINAL_KEY, state);
-    const event = await this.storeRunEvent({
-      runId: state.runId,
-      type: 'terminal',
-      status: state.status,
-      result: state.result,
-      error: state.error,
-      totalRows: state.totalRows,
-      durationMs: state.durationMs,
-      playName: state.playName,
-      liveLogs: state.liveLogs,
-      liveNodeProgress: state.liveNodeProgress,
-      ts: state.completedAt,
-    });
-    this.wakeRunEventWaiters(event);
-    return new Response('{}', {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handleTerminalGet(): Promise<Response> {
-    const state = await this.state.storage.get<CoordinatorTerminalState>(
-      COORDINATOR_TERMINAL_KEY,
-    );
-    return new Response(JSON.stringify({ state: state ?? null }), {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  /**
-   * Lease up to `requested` request-window permits for `bucketId` under all
-   * `rules`. Single-threaded DO access means this is the same sliding-window
-   * math as InMemoryRateStateBackend.acquire, generalized to grant a BLOCK of
-   * permits at once so the runtime amortizes the round-trip across many calls.
-   *
-   * Concurrency rules (`maxConcurrency`) are NOT enforced here: simultaneous
-   * in-flight tracking across isolates needs a reliable release signal, which a
-   * dying isolate cannot guarantee. The Governor's global tool-concurrency
-   * semaphore is the cross-call concurrency backstop; this DO owns the
-   * cross-isolate REQUEST RATE, which is the throughput governor.
-   */
-  private async handleRateAcquire(req: Request): Promise<Response> {
-    const body = (await req
-      .json()
-      .catch(() => null)) as RateAcquireRequest | null;
-    if (
-      !body ||
-      typeof body.bucketId !== 'string' ||
-      !body.bucketId.trim() ||
-      !Array.isArray(body.rules)
-    ) {
-      return new Response('bucketId and rules are required', { status: 400 });
-    }
-    const requested =
-      typeof body.requested === 'number' && Number.isFinite(body.requested)
-        ? Math.max(1, Math.floor(body.requested))
-        : 1;
-    const result = this.computeRateAcquire(
-      body.bucketId,
-      body.rules,
-      requested,
-    );
-    return new Response(JSON.stringify(result), {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private computeRateAcquire(
-    bucketId: string,
-    rules: RateRule[],
-    requested: number,
-  ): RateAcquireResponse {
-    const now = Date.now();
-    const usableRules = rules.filter(
-      (rule) =>
-        rule &&
-        typeof rule.ruleId === 'string' &&
-        typeof rule.requestsPerWindow === 'number' &&
-        rule.requestsPerWindow > 0 &&
-        typeof rule.windowMs === 'number',
-    );
-    if (usableRules.length === 0) {
-      return { granted: requested, waitMs: 0 };
-    }
-
-    let waitMs = 0;
-    const cooldownUntil = this.rateCooldownUntil.get(bucketId) ?? 0;
-    if (cooldownUntil > now) waitMs = Math.max(waitMs, cooldownUntil - now);
-
-    // The grant is the min remaining capacity across every rule's window. A
-    // single round-trip can never debit more than the tightest rule allows.
-    let grantable = requested;
-    for (const rule of usableRules) {
-      const state = this.getRateWindowState(bucketId, rule, now);
-      this.resetExpiredRateWindow(state, rule.windowMs, now);
-      const remaining = rule.requestsPerWindow - state.startedInWindow;
-      grantable = Math.min(grantable, Math.max(0, remaining));
-      if (remaining <= 0) {
-        waitMs = Math.max(
-          waitMs,
-          rule.windowMs > 0
-            ? state.windowStartedAt + rule.windowMs - now
-            : RATE_MIN_WAIT_MS,
-        );
-      }
-    }
-
-    if (waitMs > 0 || grantable <= 0) {
-      return { granted: 0, waitMs: Math.max(RATE_MIN_WAIT_MS, waitMs) };
-    }
-
-    for (const rule of usableRules) {
-      const state = this.getRateWindowState(bucketId, rule, now);
-      state.startedInWindow += grantable;
-    }
-    return { granted: grantable, waitMs: 0 };
-  }
-
-  private async handleRatePenalize(req: Request): Promise<Response> {
-    const body = (await req
-      .json()
-      .catch(() => null)) as RatePenalizeRequest | null;
-    if (!body || typeof body.bucketId !== 'string' || !body.bucketId.trim()) {
-      return new Response('bucketId is required', { status: 400 });
-    }
-    const cooldownMs =
-      typeof body.cooldownMs === 'number' && Number.isFinite(body.cooldownMs)
-        ? Math.floor(body.cooldownMs)
-        : 0;
-    if (cooldownMs > 0) {
-      const until = Date.now() + cooldownMs;
-      const existing = this.rateCooldownUntil.get(body.bucketId) ?? 0;
-      this.rateCooldownUntil.set(body.bucketId, Math.max(existing, until));
-    }
-    return new Response('{}', {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private getRateWindowState(
-    bucketId: string,
-    rule: RateRule,
-    now: number,
-  ): RateRuleWindowState {
-    const key = RATE_STATE_KEY(bucketId, rule.ruleId);
-    const existing = this.rateWindows.get(key);
-    if (existing) return existing;
-    const created: RateRuleWindowState = {
-      windowStartedAt: now,
-      startedInWindow: 0,
-    };
-    this.rateWindows.set(key, created);
-    return created;
-  }
-
-  private resetExpiredRateWindow(
-    state: RateRuleWindowState,
-    windowMs: number,
-    now: number,
-  ): void {
-    if (windowMs <= 0) {
-      state.windowStartedAt = now;
-      state.startedInWindow = 0;
-      return;
-    }
-    if (now - state.windowStartedAt < windowMs) return;
-    const elapsed = Math.floor((now - state.windowStartedAt) / windowMs);
-    state.windowStartedAt += elapsed * windowMs;
-    state.startedInWindow = 0;
-    if (now - state.windowStartedAt >= windowMs) state.windowStartedAt = now;
-  }
-
-  private async storeRunEvent(
-    input: CoordinatorRunEventInput,
-  ): Promise<CoordinatorRunEvent> {
-    let event: CoordinatorRunEvent | null = null;
-    await this.state.blockConcurrencyWhile(async () => {
-      const current =
-        (await this.state.storage.get<number>(
-          COORDINATOR_RUN_EVENT_SEQUENCE_KEY,
-        )) ?? 0;
-      const seq = current + 1;
-      event = { ...input, seq } as CoordinatorRunEvent;
-      await this.state.storage.put({
-        [COORDINATOR_RUN_EVENT_SEQUENCE_KEY]: seq,
-        [this.runEventKey(event)]: event,
-      });
-      const pruneSeq = seq - COORDINATOR_RUN_EVENT_MAX_ENTRIES;
-      if (pruneSeq > 0) {
-        await this.state.storage.delete(this.runEventKeyForSeq(pruneSeq));
-      }
-    });
-    if (!event) {
-      throw new Error('failed to store run event');
-    }
-    return event;
-  }
-
-  private wakeRunEventWaiters(event: CoordinatorRunEvent): void {
-    for (const resolve of this.runEventWaiters) {
-      resolve(event);
-    }
-    this.runEventWaiters.clear();
-  }
-
-  private async handleRunEventAdd(req: Request): Promise<Response> {
-    const body = (await req
-      .json()
-      .catch(() => null)) as Partial<CoordinatorRunEvent> | null;
-    if (!body || typeof body.runId !== 'string') {
-      return new Response('invalid run event', { status: 400 });
-    }
-    let eventInput: CoordinatorRunEventInput | null = null;
-    const ts = typeof body.ts === 'number' ? body.ts : Date.now();
-    if (body.type === 'log' && typeof body.line === 'string') {
-      eventInput = {
-        runId: body.runId,
-        type: 'log',
-        line: body.line,
-        ts,
-      };
-    } else if (body.type === 'status' && typeof body.status === 'string') {
-      eventInput = {
-        runId: body.runId,
-        type: 'status',
-        status: body.status,
-        ts,
-        logs: Array.isArray(body.logs)
-          ? body.logs.filter((line): line is string => typeof line === 'string')
-          : undefined,
-      };
-    } else if (body.type === 'progress' && typeof body.status === 'string') {
-      eventInput = {
-        runId: body.runId,
-        type: 'progress',
-        status: body.status,
-        ts,
-        logs: Array.isArray(body.logs)
-          ? body.logs.filter((line): line is string => typeof line === 'string')
-          : undefined,
-        activeNodeId:
-          typeof body.activeNodeId === 'string' ? body.activeNodeId : null,
-        activeArtifactTableNamespace:
-          typeof body.activeArtifactTableNamespace === 'string'
-            ? body.activeArtifactTableNamespace
-            : null,
-        updatedAt: typeof body.updatedAt === 'number' ? body.updatedAt : null,
-        liveNodeProgress: body.liveNodeProgress,
-      };
-    } else if (
-      body.type === 'terminal' &&
-      (body.status === 'completed' ||
-        body.status === 'failed' ||
-        body.status === 'cancelled')
-    ) {
-      eventInput = {
-        runId: body.runId,
-        type: 'terminal',
-        status: body.status,
-        ts,
-        result: body.result,
-        error: typeof body.error === 'string' ? body.error : null,
-        totalRows: body.totalRows,
-        durationMs: body.durationMs,
-        playName: typeof body.playName === 'string' ? body.playName : null,
-      };
-    }
-    if (!eventInput) {
-      return new Response('invalid run event', { status: 400 });
-    }
-    const event = await this.storeRunEvent(eventInput);
-    this.wakeRunEventWaiters(event);
-    return new Response(JSON.stringify({ event }), {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-
-  private async handleRunEventList(req: Request): Promise<Response> {
-    const url = new URL(req.url);
-    const afterSeq = Math.max(
-      0,
-      Math.floor(Number(url.searchParams.get('afterSeq') ?? '0')),
-    );
-    const timeoutMs = Math.min(
-      Math.max(Number(url.searchParams.get('timeoutMs') ?? '0'), 0),
-      30_000,
-    );
-    const readLatestSeq = async () =>
-      (await this.state.storage.get<number>(
-        COORDINATOR_RUN_EVENT_SEQUENCE_KEY,
-      )) ?? afterSeq;
-    const readEvents = async (latestSeq: number) => {
-      const firstAvailableSeq = Math.max(
-        1,
-        latestSeq - COORDINATOR_RUN_EVENT_MAX_ENTRIES + 1,
-      );
-      const startSeq = Math.max(afterSeq + 1, firstAvailableSeq);
-      if (latestSeq < startSeq) return [];
-      const keys: string[] = [];
-      for (let seq = startSeq; seq <= latestSeq; seq += 1) {
-        keys.push(this.runEventKeyForSeq(seq));
-      }
-      const entries = await this.state.storage.get<CoordinatorRunEvent>(keys);
-      return [...entries.values()].sort((left, right) => left.seq - right.seq);
-    };
-    let latestSeq = await readLatestSeq();
-    let events = await readEvents(latestSeq);
-    if (events.length === 0 && timeoutMs > 0) {
-      await new Promise<void>((resolve) => {
-        let settled = false;
-        const finish = () => {
-          if (settled) return;
-          settled = true;
-          this.runEventWaiters.delete(onEvent);
-          clearTimeout(timeout);
-          resolve();
-        };
-        const onEvent = () => finish();
-        const timeout = setTimeout(finish, timeoutMs);
-        this.runEventWaiters.add(onEvent);
-      });
-      latestSeq = await readLatestSeq();
-      events = await readEvents(latestSeq);
-    }
-    return new Response(JSON.stringify({ events, latestSeq }), {
-      headers: { 'content-type': 'application/json' },
-    });
-  }
-}