deepline 0.1.19 → 0.1.21

package/dist/repo/apps/play-runner-workers/src/dedup-do.ts

@@ -48,17 +48,26 @@ type AwaitRequest = {
   timeoutMs: number;
 };
 
+type WorkflowPoolEntryState = 'warming' | 'ready';
+
 type WorkflowPoolEntry = {
   id: string;
   version: string;
+  state: WorkflowPoolEntryState;
   createdAt: number;
   readyAt: number | null;
   expiresAt: number;
 };
 
+type WorkflowRunMappingState = 'claimed' | 'started' | 'blocked';
+
 type WorkflowRunMapping = {
   runId: string;
   instanceId: string;
+  state: WorkflowRunMappingState;
+  blockedInstanceId?: string | null;
+  claimedAt?: number | null;
+  startedAt?: number | null;
   version: string;
   createdAt: number;
   expiresAt: number;
@@ -74,19 +83,83 @@ type CoordinatorTraceEntry = {
   [key: string]: unknown;
 };
 
+type CoordinatorTerminalState = {
+  runId: string;
+  status: 'completed' | 'failed' | 'cancelled';
+  result?: unknown;
+  error?: string | null;
+  totalRows?: unknown;
+  durationMs?: unknown;
+  playName?: string | null;
+  completedAt: number;
+};
+
+type CoordinatorRunEvent =
+  | {
+      seq: number;
+      runId: string;
+      type: 'status';
+      status: string;
+      ts: number;
+      logs?: string[];
+    }
+  | {
+      seq: number;
+      runId: string;
+      type: 'log';
+      line: string;
+      ts: number;
+    }
+  | {
+      seq: number;
+      runId: string;
+      type: 'progress';
+      status: string;
+      ts: number;
+      logs?: string[];
+      activeNodeId?: string | null;
+      activeArtifactTableNamespace?: string | null;
+      updatedAt?: number | null;
+    }
+  | {
+      seq: number;
+      runId: string;
+      type: 'terminal';
+      status: 'completed' | 'failed' | 'cancelled';
+      ts: number;
+      result?: unknown;
+      error?: string | null;
+      totalRows?: unknown;
+      durationMs?: unknown;
+      playName?: string | null;
+    };
+
+type OmitRunEventSequence<T> = T extends unknown ? Omit<T, 'seq'> : never;
+type CoordinatorRunEventInput = OmitRunEventSequence<CoordinatorRunEvent>;
+
 type ReadyWorkflowPoolEntryRecord = {
   key: string;
-  entry: WorkflowPoolEntry & { readyAt: number };
+  entry: WorkflowPoolEntry & { state: 'ready'; readyAt: number };
+};
+
+type WorkflowPoolCounts = {
+  available: number;
+  warming: number;
 };
 
 const DEDUP_KEY_PREFIX = 'd:';
 const WORKFLOW_POOL_KEY_PREFIX = 'p:';
 const WORKFLOW_POOL_RUN_KEY_PREFIX = 'm:';
 const COORDINATOR_TRACE_KEY_PREFIX = 't:';
+const COORDINATOR_RUN_EVENT_KEY_PREFIX = 'e:';
+const COORDINATOR_TERMINAL_KEY = 'terminal';
+const COORDINATOR_RUN_EVENT_SEQUENCE_KEY = 'event-seq';
 const COORDINATOR_TRACE_MAX_ENTRIES = 200;
+const COORDINATOR_RUN_EVENT_MAX_ENTRIES = 500;
 const FINISH_ALARM_DELAY_MS = 60_000; // self-evict 1 min after finish() called
 const WORKFLOW_POOL_DEFAULT_TTL_MS = 8 * 60 * 1000;
 const WORKFLOW_POOL_RUN_MAPPING_TTL_MS = 60 * 60 * 1000;
+const WORKFLOW_POOL_READY_MAX_AGE_MS = 7 * 60_000;
 
 interface DedupEnv {
   PLAY_DEDUP: DurableObjectNamespace;
@@ -96,6 +169,8 @@ export class PlayDedup implements DurableObject {
   // Promises waiting on /await endpoints. Keyed by `${toolId}:${inputHash}`.
   // Resolved by /publish; rejected on timeout.
   private waiters: Map<string, Set<(value: unknown) => void>> = new Map();
+  private runEventWaiters: Set<(value: CoordinatorRunEvent) => void> =
+    new Set();
 
   constructor(
     private readonly state: DurableObjectState,
@@ -110,6 +185,10 @@ export class PlayDedup implements DurableObject {
     return `${COORDINATOR_TRACE_KEY_PREFIX}${String(entry.ts).padStart(13, '0')}:${crypto.randomUUID().slice(0, 8)}`;
   }
 
+  private runEventKey(event: CoordinatorRunEvent): string {
+    return `${COORDINATOR_RUN_EVENT_KEY_PREFIX}${String(event.seq).padStart(13, '0')}`;
+  }
+
   private waiterKey(toolId: string, inputHash: string): string {
     return `${toolId}:${inputHash}`;
   }
@@ -130,18 +209,22 @@ export class PlayDedup implements DurableObject {
           return await this.handleDebug(req);
         case '/pool-add':
           return await this.handlePoolAdd(req);
-        case '/pool-lease':
-          return await this.handlePoolLease(req);
+        case '/pool-claim':
+          return await this.handlePoolClaim(req);
         case '/pool-count':
           return await this.handlePoolCount(req);
         case '/pool-list':
           return await this.handlePoolList(req);
         case '/pool-promote':
           return await this.handlePoolPromote(req);
+        case '/pool-ready':
+          return await this.handlePoolReady(req);
         case '/pool-delete':
           return await this.handlePoolDelete(req);
         case '/pool-map-run':
           return await this.handlePoolMapRun(req);
+        case '/pool-block-run':
+          return await this.handlePoolBlockRun(req);
         case '/pool-resolve-run':
           return await this.handlePoolResolveRun(req);
         case '/pool-clear':
@@ -150,6 +233,14 @@ export class PlayDedup implements DurableObject {
           return await this.handleTraceAdd(req);
         case '/trace-list':
           return await this.handleTraceList();
+        case '/event-add':
+          return await this.handleRunEventAdd(req);
+        case '/event-list':
+          return await this.handleRunEventList(req);
+        case '/terminal-set':
+          return await this.handleTerminalSet(req);
+        case '/terminal-get':
+          return await this.handleTerminalGet();
         default:
           return new Response('not found', { status: 404 });
       }
@@ -167,6 +258,7 @@ export class PlayDedup implements DurableObject {
     // be garbage-collected by CF's hot pool.
     await this.state.storage.deleteAll();
     this.waiters.clear();
+    this.runEventWaiters.clear();
   }
 
   private async handleLookupOrClaim(req: Request): Promise<Response> {
@@ -248,7 +340,9 @@ export class PlayDedup implements DurableObject {
       }
     }
 
-    return new Response('{}', { headers: { 'content-type': 'application/json' } });
+    return new Response('{}', {
+      headers: { 'content-type': 'application/json' },
+    });
   }
 
   private async handleAwait(req: Request): Promise<Response> {
@@ -313,7 +407,9 @@ export class PlayDedup implements DurableObject {
     // Schedule alarm to evict after a grace period (any straggler awaits
     // get a chance to resolve). Grace = 60s.
     await this.state.storage.setAlarm(Date.now() + FINISH_ALARM_DELAY_MS);
-    return new Response('{}', { headers: { 'content-type': 'application/json' } });
+    return new Response('{}', {
+      headers: { 'content-type': 'application/json' },
+    });
   }
 
   private async handleDebug(_req: Request): Promise<Response> {
@@ -321,7 +417,11 @@ export class PlayDedup implements DurableObject {
     const dump: Record<string, DedupEntry> = {};
     for (const [k, v] of all) dump[k] = v as DedupEntry;
     return new Response(
-      JSON.stringify({ size: all.size, entries: dump, waiters: this.waiters.size }),
+      JSON.stringify({
+        size: all.size,
+        entries: dump,
+        waiters: this.waiters.size,
+      }),
       { headers: { 'content-type': 'application/json' } },
     );
   }
@@ -330,16 +430,28 @@ export class PlayDedup implements DurableObject {
     return new URL(req.url).searchParams.get('version')?.trim() ?? '';
   }
 
+  private workflowPoolMinReadyAgeMs(req: Request): number {
+    const raw = Number(new URL(req.url).searchParams.get('minReadyAgeMs') ?? 0);
+    if (!Number.isFinite(raw) || raw <= 0) {
+      return 0;
+    }
+    return Math.min(Math.floor(raw), WORKFLOW_POOL_DEFAULT_TTL_MS);
+  }
+
   private isReadyWorkflowPoolEntry(
     value: { key: string; entry: WorkflowPoolEntry | undefined },
     version: string,
     now: number,
+    minReadyAgeMs = 0,
   ): value is ReadyWorkflowPoolEntryRecord {
     return (
       value.entry !== undefined &&
       value.entry.version === version &&
       value.entry.expiresAt > now &&
-      value.entry.readyAt !== null
+      value.entry.state === 'ready' &&
+      value.entry.readyAt !== null &&
+      now - value.entry.readyAt <= WORKFLOW_POOL_READY_MAX_AGE_MS &&
+      now - value.entry.readyAt >= minReadyAgeMs
     );
   }
 
@@ -360,6 +472,9 @@ export class PlayDedup implements DurableObject {
       if (
         !entry ||
         entry.expiresAt <= now ||
+        (entry.state === 'ready' &&
+          entry.readyAt !== null &&
+          now - entry.readyAt > WORKFLOW_POOL_READY_MAX_AGE_MS) ||
         (version && entry.version !== version)
       ) {
         expiredKeys.push(key);
@@ -380,16 +495,15 @@ export class PlayDedup implements DurableObject {
   }
 
   private async handlePoolAdd(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as
-      | {
-          ids?: unknown;
-          ttlMs?: unknown;
-          version?: unknown;
-          readyAt?: unknown;
-          ready?: unknown;
-        }
-      | null;
-    const version = typeof body?.version === 'string' ? body.version.trim() : '';
+    const body = (await req.json().catch(() => null)) as {
+      ids?: unknown;
+      ttlMs?: unknown;
+      version?: unknown;
+      readyAt?: unknown;
+      ready?: unknown;
+    } | null;
+    const version =
+      typeof body?.version === 'string' ? body.version.trim() : '';
     if (!version) {
       return new Response('version is required', { status: 400 });
     }
@@ -409,18 +523,34 @@ export class PlayDedup implements DurableObject {
       body.ttlMs > 0
         ? Math.min(body.ttlMs, WORKFLOW_POOL_DEFAULT_TTL_MS)
         : WORKFLOW_POOL_DEFAULT_TTL_MS;
-    const writes: Record<string, WorkflowPoolEntry> = {};
-    for (const id of ids) {
-      writes[`${WORKFLOW_POOL_KEY_PREFIX}${id}`] = {
-        id,
-        version,
-        createdAt: now,
-        readyAt: ready ? readyAt : null,
-        expiresAt: now + ttlMs,
-      };
-    }
     await this.state.blockConcurrencyWhile(async () => {
       await this.gcWorkflowPool(now, version);
+      const writes: Record<string, WorkflowPoolEntry> = {};
+      const keys = ids.map((id) => `${WORKFLOW_POOL_KEY_PREFIX}${id}`);
+      const existing = (await this.state.storage.get<WorkflowPoolEntry>(
+        keys,
+      )) as Map<string, WorkflowPoolEntry>;
+      for (const id of ids) {
+        const key = `${WORKFLOW_POOL_KEY_PREFIX}${id}`;
+        const existingEntry = existing.get(key);
+        const existingReadyAt =
+          existingEntry?.version === version &&
+          existingEntry.expiresAt > now &&
+          existingEntry.state === 'ready' &&
+          existingEntry.readyAt !== null
+            ? existingEntry.readyAt
+            : null;
+        const nextReadyAt = ready ? readyAt : existingReadyAt;
+        writes[key] = {
+          id,
+          version,
+          state: nextReadyAt !== null ? 'ready' : 'warming',
+          createdAt:
+            existingEntry?.version === version ? existingEntry.createdAt : now,
+          readyAt: nextReadyAt,
+          expiresAt: now + ttlMs,
+        };
+      }
       if (Object.keys(writes).length > 0) {
         await this.state.storage.put(writes);
       }
@@ -430,28 +560,72 @@ export class PlayDedup implements DurableObject {
     });
   }
 
-  private async handlePoolLease(req: Request): Promise<Response> {
+  private async handlePoolClaim(req: Request): Promise<Response> {
     const version = this.workflowPoolVersion(req);
     if (!version) {
       return new Response('version is required', { status: 400 });
     }
-    let leasedId: string | null = null;
+    const body = (await req.json().catch(() => null)) as {
+      runId?: unknown;
+    } | null;
+    const runId = typeof body?.runId === 'string' ? body.runId : '';
+    if (!runId) {
+      return new Response('runId is required', { status: 400 });
+    }
+    const minReadyAgeMs = this.workflowPoolMinReadyAgeMs(req);
+    let claimedId: string | null = null;
+    let counts: WorkflowPoolCounts = { available: 0, warming: 0 };
     await this.state.blockConcurrencyWhile(async () => {
       const now = Date.now();
       await this.gcWorkflowPool(now, version);
+      const mappingKey = `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`;
+      const existingMapping =
+        await this.state.storage.get<WorkflowRunMapping>(mappingKey);
+      if (existingMapping?.version === version) {
+        if (
+          existingMapping.state === 'claimed' ||
+          existingMapping.state === 'started'
+        ) {
+          claimedId = existingMapping.instanceId || null;
+        }
+        return;
+      }
       const entries = await this.state.storage.list<WorkflowPoolEntry>({
         prefix: WORKFLOW_POOL_KEY_PREFIX,
       });
+      counts = this.countWorkflowPoolEntries(
+        entries,
+        version,
+        now,
+        minReadyAgeMs,
+      );
       const sorted = [...entries.entries()]
         .map(([key, entry]) => ({ key, entry }))
-        .filter((entry) => this.isReadyWorkflowPoolEntry(entry, version, now))
-        .sort((a, b) => a.entry.readyAt - b.entry.readyAt);
+        .filter((entry) =>
+          this.isReadyWorkflowPoolEntry(entry, version, now, minReadyAgeMs),
+        )
+        .sort((a, b) => b.entry.readyAt - a.entry.readyAt);
       const selected = sorted[0];
       if (!selected) return;
-      leasedId = selected.entry.id;
+      claimedId = selected.entry.id;
       await this.state.storage.delete(selected.key);
+      await this.state.storage.put(mappingKey, {
+        runId,
+        instanceId: claimedId,
+        state: 'claimed',
+        blockedInstanceId: null,
+        claimedAt: now,
+        startedAt: null,
+        version,
+        createdAt: now,
+        expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
+      } satisfies WorkflowRunMapping);
+      counts = {
+        available: Math.max(0, counts.available - 1),
+        warming: counts.warming,
+      };
     });
-    return new Response(JSON.stringify({ id: leasedId }), {
+    return new Response(JSON.stringify({ id: claimedId, ...counts }), {
       headers: { 'content-type': 'application/json' },
     });
   }
@@ -461,20 +635,45 @@ export class PlayDedup implements DurableObject {
     if (!version) {
       return new Response('version is required', { status: 400 });
     }
-    await this.gcWorkflowPool(Date.now(), version);
+    const minReadyAgeMs = this.workflowPoolMinReadyAgeMs(req);
+    const now = Date.now();
+    await this.gcWorkflowPool(now, version);
     const entries = await this.state.storage.list<WorkflowPoolEntry>({
       prefix: WORKFLOW_POOL_KEY_PREFIX,
     });
+    const counts = this.countWorkflowPoolEntries(
+      entries,
+      version,
+      now,
+      minReadyAgeMs,
+    );
+    return new Response(JSON.stringify(counts), {
+      headers: { 'content-type': 'application/json' },
+    });
+  }
+
+  private countWorkflowPoolEntries(
+    entries: Map<string, WorkflowPoolEntry>,
+    version: string,
+    now: number,
+    minReadyAgeMs: number,
+  ): WorkflowPoolCounts {
     let available = 0;
     let warming = 0;
     for (const entry of entries.values()) {
       if (entry.version !== version) continue;
-      if (entry.readyAt === null) warming += 1;
-      else available += 1;
+      if (
+        entry.state !== 'ready' ||
+        entry.readyAt === null ||
+        now - entry.readyAt > WORKFLOW_POOL_READY_MAX_AGE_MS ||
+        now - entry.readyAt < minReadyAgeMs
+      ) {
+        warming += 1;
+      } else {
+        available += 1;
+      }
     }
-    return new Response(JSON.stringify({ available, warming }), {
-      headers: { 'content-type': 'application/json' },
-    });
+    return { available, warming };
   }
 
   private async handlePoolList(req: Request): Promise<Response> {
@@ -492,6 +691,7 @@ export class PlayDedup implements DurableObject {
           .filter((entry) => entry.version === version)
           .map((entry) => ({
             id: entry.id,
+            state: entry.state,
             createdAt: entry.createdAt,
             readyAt: entry.readyAt,
             expiresAt: entry.expiresAt,
@@ -502,13 +702,12 @@ export class PlayDedup implements DurableObject {
   }
 
   private async handlePoolPromote(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as
-      | {
-          ids?: unknown;
-          version?: unknown;
-        }
-      | null;
-    const version = typeof body?.version === 'string' ? body.version.trim() : '';
+    const body = (await req.json().catch(() => null)) as {
+      ids?: unknown;
+      version?: unknown;
+    } | null;
+    const version =
+      typeof body?.version === 'string' ? body.version.trim() : '';
     if (!version) {
       return new Response('version is required', { status: 400 });
     }
@@ -529,6 +728,7 @@ export class PlayDedup implements DurableObject {
         }
         writes[key] = {
           ...entry,
+          state: 'ready',
           readyAt: now,
         };
       }
@@ -536,19 +736,53 @@ export class PlayDedup implements DurableObject {
         await this.state.storage.put(writes);
       }
     });
-    return new Response(JSON.stringify({ promoted: Object.keys(writes).length }), {
+    return new Response(
+      JSON.stringify({ promoted: Object.keys(writes).length }),
+      {
+        headers: { 'content-type': 'application/json' },
+      },
+    );
+  }
+
+  private async handlePoolReady(req: Request): Promise<Response> {
+    const body = (await req.json().catch(() => null)) as {
+      poolId?: unknown;
+      version?: unknown;
+    } | null;
+    const poolId = typeof body?.poolId === 'string' ? body.poolId : '';
+    const version =
+      typeof body?.version === 'string' ? body.version.trim() : '';
+    if (!poolId || !version) {
+      return new Response('poolId and version are required', { status: 400 });
+    }
+    const now = Date.now();
+    let ready = false;
+    await this.state.blockConcurrencyWhile(async () => {
+      await this.gcWorkflowPool(now, version);
+      const key = `${WORKFLOW_POOL_KEY_PREFIX}${poolId}`;
+      const entry = await this.state.storage.get<WorkflowPoolEntry>(key);
+      if (!entry || entry.version !== version || entry.expiresAt <= now) {
+        return;
+      }
+      await this.state.storage.put(key, {
+        ...entry,
+        state: 'ready',
+        readyAt: entry.readyAt ?? now,
+      });
+      ready = true;
+    });
+    return new Response(JSON.stringify({ ready }), {
       headers: { 'content-type': 'application/json' },
     });
   }
 
   private async handlePoolDelete(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as
-      | {
-          ids?: unknown;
-          version?: unknown;
-        }
-      | null;
-    const version = typeof body?.version === 'string' ? body.version.trim() : '';
+    const body = (await req.json().catch(() => null)) as {
+      ids?: unknown;
+      version?: unknown;
+    } | null;
+    const version =
+      typeof body?.version === 'string' ? body.version.trim() : '';
     if (!version) {
       return new Response('version is required', { status: 400 });
     }
@@ -575,34 +809,118 @@ export class PlayDedup implements DurableObject {
   }
 
   private async handlePoolMapRun(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as
-      | {
-          runId?: unknown;
-          instanceId?: unknown;
-          version?: unknown;
-        }
-      | null;
+    const body = (await req.json().catch(() => null)) as {
+      runId?: unknown;
+      instanceId?: unknown;
+      version?: unknown;
+      started?: unknown;
+    } | null;
     const runId = typeof body?.runId === 'string' ? body.runId : '';
     const instanceId =
       typeof body?.instanceId === 'string' ? body.instanceId : '';
-    const version = typeof body?.version === 'string' ? body.version.trim() : '';
+    const version =
+      typeof body?.version === 'string' ? body.version.trim() : '';
+    const started = body?.started === true;
     if (!runId || !instanceId || !version) {
       return new Response('runId, instanceId, and version are required', {
         status: 400,
       });
     }
     const now = Date.now();
-    await this.state.storage.put(
-      `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`,
-      {
+    let mapped = true;
+    const key = `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`;
+    await this.state.blockConcurrencyWhile(async () => {
+      const existing = await this.state.storage.get<WorkflowRunMapping>(key);
+      if (
+        existing?.version === version &&
+        existing.state === 'blocked' &&
+        existing.blockedInstanceId === instanceId
+      ) {
+        mapped = false;
+        return;
+      }
+      const alreadyStarted =
+        existing?.version === version &&
+        existing.instanceId === instanceId &&
+        existing.state === 'started' &&
+        typeof existing.startedAt === 'number';
+      const startedAt = started || alreadyStarted ? now : null;
+      await this.state.storage.put(key, {
         runId,
         instanceId,
+        state: startedAt !== null ? 'started' : 'claimed',
+        blockedInstanceId: null,
+        claimedAt:
+          existing?.version === version && typeof existing.claimedAt === 'number'
+            ? existing.claimedAt
+            : now,
+        startedAt,
+        version,
+        createdAt:
+          existing?.version === version && existing.createdAt
+            ? existing.createdAt
+            : now,
+        expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
+      } satisfies WorkflowRunMapping);
+    });
+    return new Response(JSON.stringify({ mapped }), {
+      headers: { 'content-type': 'application/json' },
+    });
+  }
+
+  private async handlePoolBlockRun(req: Request): Promise<Response> {
+    const body = (await req.json().catch(() => null)) as {
+      runId?: unknown;
+      instanceId?: unknown;
+      version?: unknown;
+    } | null;
+    const runId = typeof body?.runId === 'string' ? body.runId : '';
+    const instanceId =
+      typeof body?.instanceId === 'string' ? body.instanceId : '';
+    const version =
+      typeof body?.version === 'string' ? body.version.trim() : '';
+    if (!runId || !instanceId || !version) {
+      return new Response('runId, instanceId, and version are required', {
+        status: 400,
+      });
+    }
+    const now = Date.now();
+    const key = `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`;
+    let started = false;
+    await this.state.blockConcurrencyWhile(async () => {
+      const existing = await this.state.storage.get<WorkflowRunMapping>(key);
+      if (
+        existing?.version === version &&
+        existing.instanceId === instanceId &&
+        existing.state === 'started' &&
+        typeof existing.startedAt === 'number'
+      ) {
+        started = true;
+        return;
+      }
+      await this.state.storage.put(key, {
+        runId,
+        instanceId: '',
+        state: 'blocked',
+        blockedInstanceId: instanceId,
+        claimedAt:
+          existing?.version === version && typeof existing.claimedAt === 'number'
+            ? existing.claimedAt
+            : now,
+        startedAt: null,
         version,
         createdAt: now,
         expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
-      } satisfies WorkflowRunMapping,
-    );
-    return new Response('{}', { headers: { 'content-type': 'application/json' } });
+      } satisfies WorkflowRunMapping);
+    });
+    if (started) {
+      return new Response(JSON.stringify({ blocked: false, started: true }), {
+        headers: { 'content-type': 'application/json' },
+      });
+    }
+    return new Response(JSON.stringify({ blocked: true, started: false }), {
+      headers: { 'content-type': 'application/json' },
+    });
   }
 
   private async handlePoolResolveRun(req: Request): Promise<Response> {
@@ -620,13 +938,18 @@ export class PlayDedup implements DurableObject {
       `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`,
     );
     if (mapping && mapping.version !== version) {
-      await this.state.storage.delete(`${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`);
+      await this.state.storage.delete(
+        `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`,
+      );
       return new Response(JSON.stringify({ instanceId: null }), {
         headers: { 'content-type': 'application/json' },
       });
     }
     return new Response(
-      JSON.stringify({ instanceId: mapping?.instanceId ?? null }),
+      JSON.stringify({
+        instanceId: mapping?.instanceId || null,
+        startedAt: mapping?.startedAt ?? null,
+      }),
       { headers: { 'content-type': 'application/json' } },
     );
   }
@@ -658,9 +981,9 @@ export class PlayDedup implements DurableObject {
   }
 
   private async handleTraceAdd(req: Request): Promise<Response> {
-    const body = (await req.json().catch(() => null)) as
-      | Partial<CoordinatorTraceEntry>
-      | null;
+    const body = (await req
+      .json()
+      .catch(() => null)) as Partial<CoordinatorTraceEntry> | null;
     if (
       !body ||
       (body.source !== 'coordinator' && body.source !== 'dynamic_worker') ||
@@ -693,7 +1016,9 @@ export class PlayDedup implements DurableObject {
         await this.state.storage.delete([...entries.keys()].slice(0, overflow));
       }
     });
-    return new Response('{}', { headers: { 'content-type': 'application/json' } });
+    return new Response('{}', {
+      headers: { 'content-type': 'application/json' },
+    });
   }
 
   private async handleTraceList(): Promise<Response> {
@@ -702,9 +1027,213 @@ export class PlayDedup implements DurableObject {
     });
     return new Response(
       JSON.stringify({
-        entries: [...entries.values()].sort((left, right) => left.ts - right.ts),
+        entries: [...entries.values()].sort(
+          (left, right) => left.ts - right.ts,
+        ),
       }),
       { headers: { 'content-type': 'application/json' } },
     );
   }
+
+  private async handleTerminalSet(req: Request): Promise<Response> {
+    const body = (await req
+      .json()
+      .catch(() => null)) as Partial<CoordinatorTerminalState> | null;
+    if (
+      !body ||
+      typeof body.runId !== 'string' ||
+      (body.status !== 'completed' &&
+        body.status !== 'failed' &&
+        body.status !== 'cancelled')
+    ) {
+      return new Response('invalid terminal state', { status: 400 });
+    }
+    const state: CoordinatorTerminalState = {
+      runId: body.runId,
+      status: body.status,
+      result: body.result,
+      error: typeof body.error === 'string' ? body.error : null,
+      totalRows: body.totalRows,
+      durationMs: body.durationMs,
+      playName: typeof body.playName === 'string' ? body.playName : null,
+      completedAt:
+        typeof body.completedAt === 'number' ? body.completedAt : Date.now(),
+    };
+    await this.state.storage.put(COORDINATOR_TERMINAL_KEY, state);
+    const event = await this.storeRunEvent({
+      runId: state.runId,
+      type: 'terminal',
+      status: state.status,
+      result: state.result,
+      error: state.error,
+      totalRows: state.totalRows,
+      durationMs: state.durationMs,
+      playName: state.playName,
+      ts: state.completedAt,
+    });
+    this.wakeRunEventWaiters(event);
+    return new Response('{}', {
+      headers: { 'content-type': 'application/json' },
+    });
+  }
+
+  private async handleTerminalGet(): Promise<Response> {
+    const state = await this.state.storage.get<CoordinatorTerminalState>(
+      COORDINATOR_TERMINAL_KEY,
+    );
+    return new Response(JSON.stringify({ state: state ?? null }), {
+      headers: { 'content-type': 'application/json' },
+    });
+  }
+
+  private async storeRunEvent(
+    input: CoordinatorRunEventInput,
+  ): Promise<CoordinatorRunEvent> {
+    let event: CoordinatorRunEvent | null = null;
+    await this.state.blockConcurrencyWhile(async () => {
+      const current =
+        (await this.state.storage.get<number>(
+          COORDINATOR_RUN_EVENT_SEQUENCE_KEY,
+        )) ?? 0;
+      const seq = current + 1;
+      event = { ...input, seq } as CoordinatorRunEvent;
+      await this.state.storage.put({
+        [COORDINATOR_RUN_EVENT_SEQUENCE_KEY]: seq,
+        [this.runEventKey(event)]: event,
+      });
+      const entries = await this.state.storage.list<CoordinatorRunEvent>({
+        prefix: COORDINATOR_RUN_EVENT_KEY_PREFIX,
+      });
+      const overflow = entries.size - COORDINATOR_RUN_EVENT_MAX_ENTRIES;
+      if (overflow > 0) {
+        await this.state.storage.delete([...entries.keys()].slice(0, overflow));
+      }
+    });
+    if (!event) {
+      throw new Error('failed to store run event');
+    }
+    return event;
+  }
+
+  private wakeRunEventWaiters(event: CoordinatorRunEvent): void {
+    for (const resolve of this.runEventWaiters) {
+      resolve(event);
+    }
+    this.runEventWaiters.clear();
+  }
+
+  private async handleRunEventAdd(req: Request): Promise<Response> {
+    const body = (await req.json().catch(() => null)) as
+      | Partial<CoordinatorRunEvent>
+      | null;
+    if (!body || typeof body.runId !== 'string') {
+      return new Response('invalid run event', { status: 400 });
+    }
+    let eventInput: CoordinatorRunEventInput | null = null;
+    const ts = typeof body.ts === 'number' ? body.ts : Date.now();
+    if (body.type === 'log' && typeof body.line === 'string') {
+      eventInput = {
+        runId: body.runId,
+        type: 'log',
+        line: body.line,
+        ts,
+      };
+    } else if (body.type === 'status' && typeof body.status === 'string') {
+      eventInput = {
+        runId: body.runId,
+        type: 'status',
+        status: body.status,
+        ts,
+        logs: Array.isArray(body.logs)
+          ? body.logs.filter((line): line is string => typeof line === 'string')
+          : undefined,
+      };
+    } else if (body.type === 'progress' && typeof body.status === 'string') {
+      eventInput = {
+        runId: body.runId,
+        type: 'progress',
+        status: body.status,
+        ts,
+        logs: Array.isArray(body.logs)
+          ? body.logs.filter((line): line is string => typeof line === 'string')
+          : undefined,
+        activeNodeId:
+          typeof body.activeNodeId === 'string' ? body.activeNodeId : null,
+        activeArtifactTableNamespace:
+          typeof body.activeArtifactTableNamespace === 'string'
+            ? body.activeArtifactTableNamespace
+            : null,
+        updatedAt: typeof body.updatedAt === 'number' ? body.updatedAt : null,
+      };
+    } else if (
+      body.type === 'terminal' &&
+      (body.status === 'completed' ||
+        body.status === 'failed' ||
+        body.status === 'cancelled')
+    ) {
+      eventInput = {
+        runId: body.runId,
+        type: 'terminal',
+        status: body.status,
+        ts,
+        result: body.result,
+        error: typeof body.error === 'string' ? body.error : null,
+        totalRows: body.totalRows,
+        durationMs: body.durationMs,
+        playName: typeof body.playName === 'string' ? body.playName : null,
+      };
+    }
+    if (!eventInput) {
+      return new Response('invalid run event', { status: 400 });
+    }
+    const event = await this.storeRunEvent(eventInput);
+    this.wakeRunEventWaiters(event);
+    return new Response(JSON.stringify({ event }), {
+      headers: { 'content-type': 'application/json' },
+    });
+  }
+
+  private async handleRunEventList(req: Request): Promise<Response> {
+    const url = new URL(req.url);
+    const afterSeq = Math.max(
+      0,
+      Math.floor(Number(url.searchParams.get('afterSeq') ?? '0')),
+    );
+    const timeoutMs = Math.min(
+      Math.max(Number(url.searchParams.get('timeoutMs') ?? '0'), 0),
+      30_000,
+    );
+    const readEvents = async () => {
+      const entries = await this.state.storage.list<CoordinatorRunEvent>({
+        prefix: COORDINATOR_RUN_EVENT_KEY_PREFIX,
+      });
+      return [...entries.values()]
+        .filter((event) => event.seq > afterSeq)
+        .sort((left, right) => left.seq - right.seq);
+    };
+    let events = await readEvents();
+    if (events.length === 0 && timeoutMs > 0) {
+      await new Promise<void>((resolve) => {
+        let settled = false;
+        const finish = () => {
+          if (settled) return;
+          settled = true;
+          this.runEventWaiters.delete(onEvent);
+          clearTimeout(timeout);
+          resolve();
+        };
+        const onEvent = () => finish();
+        const timeout = setTimeout(finish, timeoutMs);
+        this.runEventWaiters.add(onEvent);
+      });
+      events = await readEvents();
+    }
+    const latestSeq =
+      (await this.state.storage.get<number>(
+        COORDINATOR_RUN_EVENT_SEQUENCE_KEY,
+      )) ?? afterSeq;
+    return new Response(JSON.stringify({ events, latestSeq }), {
+      headers: { 'content-type': 'application/json' },
+    });
+  }
 }