deepline 0.1.78 → 0.1.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/dist/cli/index.js +69 -37
  2. package/dist/cli/index.mjs +69 -37
  3. package/dist/index.d.mts +32 -1
  4. package/dist/index.d.ts +32 -1
  5. package/dist/index.js +7 -4
  6. package/dist/index.mjs +7 -4
  7. package/dist/repo/apps/play-runner-workers/src/child-play-await.ts +192 -0
  8. package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +1320 -1644
  9. package/dist/repo/apps/play-runner-workers/src/dedup-do.ts +515 -648
  10. package/dist/repo/apps/play-runner-workers/src/entry.ts +896 -354
  11. package/dist/repo/apps/play-runner-workers/src/workflow-retry-state.ts +209 -0
  12. package/dist/repo/sdk/src/client.ts +9 -2
  13. package/dist/repo/sdk/src/release.ts +2 -2
  14. package/dist/repo/sdk/src/types.ts +5 -0
  15. package/dist/repo/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts +231 -0
  16. package/dist/repo/shared_libs/play-runtime/governor/governor.ts +376 -0
  17. package/dist/repo/shared_libs/play-runtime/governor/policy.ts +179 -0
  18. package/dist/repo/shared_libs/play-runtime/governor/rate-state-backend.ts +87 -0
  19. package/dist/repo/shared_libs/play-runtime/run-failure.ts +12 -0
  20. package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts +24 -0
  21. package/dist/repo/shared_libs/play-runtime/submit-limits.ts +35 -0
  22. package/dist/repo/shared_libs/plays/bundling/index.ts +4 -12
  23. package/dist/repo/shared_libs/plays/bundling/limits.ts +29 -0
  24. package/dist/repo/shared_libs/plays/static-pipeline.ts +314 -1
  25. package/dist/repo/shared_libs/temporal/constants.ts +38 -0
  26. package/package.json +1 -1
@@ -85,35 +85,19 @@ type AwaitRequest = {
85
85
  timeoutMs: number;
86
86
  };
87
87
 
88
- type WorkflowPoolEntryState = 'warming' | 'ready';
89
-
90
- type WorkflowPoolEntry = {
91
- id: string;
92
- version: string;
93
- state: WorkflowPoolEntryState;
94
- createdAt: number;
95
- readyAt: number | null;
96
- expiresAt: number;
97
- };
98
-
99
- type WorkflowRunMappingState = 'claimed' | 'started' | 'blocked';
100
-
101
- type WorkflowRunMapping = {
88
+ type WorkflowRunRetryState = {
102
89
  runId: string;
103
- instanceId: string;
104
- state: WorkflowRunMappingState;
105
- blockedInstanceId?: string | null;
106
- claimedAt?: number | null;
107
- startedAt?: number | null;
108
- version: string;
109
- createdAt: number;
90
+ params: unknown;
91
+ paramsRef?: unknown;
92
+ paramsBytes?: number;
93
+ retryAttempts: number;
94
+ updatedAt: number;
110
95
  expiresAt: number;
111
96
  };
112
97
 
113
- type WorkflowRunRetryState = {
98
+ type WorkflowInstanceState = {
114
99
  runId: string;
115
- params: unknown;
116
- retryAttempts: number;
100
+ instanceId: string;
117
101
  updatedAt: number;
118
102
  expiresAt: number;
119
103
  };
@@ -148,6 +132,12 @@ type CoordinatorTerminalState = {
148
132
  completedAt: number;
149
133
  };
150
134
 
135
+ type CoordinatorChildTerminalState = {
136
+ eventKey: string;
137
+ data: unknown;
138
+ storedAt: number;
139
+ };
140
+
151
141
  type CoordinatorRunEvent =
152
142
  | {
153
143
  seq: number;
@@ -194,31 +184,71 @@ type CoordinatorRunEvent =
194
184
  type OmitRunEventSequence<T> = T extends unknown ? Omit<T, 'seq'> : never;
195
185
  type CoordinatorRunEventInput = OmitRunEventSequence<CoordinatorRunEvent>;
196
186
 
197
- type ReadyWorkflowPoolEntryRecord = {
198
- key: string;
199
- entry: WorkflowPoolEntry & { state: 'ready'; readyAt: number };
187
+ /**
188
+ * Per-(org,provider) rate-state for the distributed Rate State Backend.
189
+ *
190
+ * One PlayDedup DO instance is addressed per bucket via
191
+ * `idFromName('rate:<orgId>:<provider>')`, so a single instance owns the rate
192
+ * window for that bucket and is single-threaded — which is exactly why the
193
+ * in-process algorithm from `InMemoryRateStateBackend` is correct here. The
194
+ * coordinator RPCs `/rate-acquire` (lease N permits) and `/rate-penalize`
195
+ * (Retry-After cooldown) into it; the esm_workers runtime never blocks per call
196
+ * on a full round-trip because it leases small permit BLOCKS at a time.
197
+ *
198
+ * Rate windows are intentionally kept in DO memory (not durable storage): a
199
+ * window is sub-second-to-seconds ephemeral state, the DO instance outlives any
200
+ * single window, and persisting it would only add storage latency to the hot
201
+ * path. This mirrors the in-memory backend and the Redis sliding-window limiter
202
+ * (`src/lib/redis/customer-rate-limiter.ts`), which also keeps window state in a
203
+ * volatile store with TTL rather than durable rows.
204
+ */
205
+ type RateRule = {
206
+ ruleId: string;
207
+ requestsPerWindow: number;
208
+ windowMs: number;
209
+ maxConcurrency: number | null;
210
+ };
211
+
212
+ type RateRuleWindowState = {
213
+ windowStartedAt: number;
214
+ startedInWindow: number;
215
+ };
216
+
217
+ type RateAcquireRequest = {
218
+ bucketId: string;
219
+ rules: RateRule[];
220
+ /** How many permits the caller wants to lease in this round-trip. */
221
+ requested: number;
222
+ };
223
+
224
+ type RateAcquireResponse = {
225
+ /** Permits actually granted (0..requested). */
226
+ granted: number;
227
+ /** Suggested wait before the next acquire when granted === 0. */
228
+ waitMs: number;
200
229
  };
201
230
 
202
- type WorkflowPoolCounts = {
203
- available: number;
204
- warming: number;
231
+ type RatePenalizeRequest = {
232
+ bucketId: string;
233
+ cooldownMs: number;
205
234
  };
206
235
 
236
+ const RATE_MIN_WAIT_MS = 10;
237
+ const RATE_STATE_KEY = (bucketId: string, ruleId: string) =>
238
+ `${bucketId}::${ruleId}`;
239
+
207
240
  const DEDUP_KEY_PREFIX = 'd:';
208
- const WORKFLOW_POOL_KEY_PREFIX = 'p:';
209
- const WORKFLOW_POOL_RUN_KEY_PREFIX = 'm:';
210
241
  const WORKFLOW_RUN_RETRY_KEY_PREFIX = 'r:';
211
242
  const WORKFLOW_DB_SESSIONS_KEY = 'db-sessions';
212
243
  const COORDINATOR_TRACE_KEY_PREFIX = 't:';
213
244
  const COORDINATOR_RUN_EVENT_KEY_PREFIX = 'e:';
214
245
  const COORDINATOR_TERMINAL_KEY = 'terminal';
246
+ const COORDINATOR_CHILD_TERMINAL_KEY_PREFIX = 'child-terminal:';
215
247
  const COORDINATOR_RUN_EVENT_SEQUENCE_KEY = 'event-seq';
216
248
  const COORDINATOR_TRACE_MAX_ENTRIES = 200;
217
249
  const COORDINATOR_RUN_EVENT_MAX_ENTRIES = 500;
218
250
  const FINISH_ALARM_DELAY_MS = 60_000; // self-evict 1 min after finish() called
219
- const WORKFLOW_POOL_DEFAULT_TTL_MS = 8 * 60 * 1000;
220
- const WORKFLOW_POOL_RUN_MAPPING_TTL_MS = 60 * 60 * 1000;
221
- const WORKFLOW_POOL_READY_MAX_AGE_MS = 7 * 60_000;
251
+ const WORKFLOW_RUN_STATE_TTL_MS = 60 * 60 * 1000;
222
252
  const WORKFLOW_RUN_RETRY_STATE_MAX_BYTES = 110_000;
223
253
  const WORKFLOW_DB_SESSIONS_TTL_MS = 10 * 60_000;
224
254
 
@@ -251,6 +281,13 @@ export class PlayDedup implements DurableObject {
251
281
  private waiters: Map<string, Set<(value: unknown) => void>> = new Map();
252
282
  private runEventWaiters: Set<(value: CoordinatorRunEvent) => void> =
253
283
  new Set();
284
+ private childTerminalWaiters: Map<string, Set<() => void>> = new Map();
285
+ // Per-(bucketId, ruleId) request-window state for the rate-state backend.
286
+ // In-memory by design — see the RateRule doc block. Single-threaded DO access
287
+ // makes this the same sliding-window algorithm as InMemoryRateStateBackend.
288
+ private rateWindows: Map<string, RateRuleWindowState> = new Map();
289
+ // Per-bucket Retry-After cooldown floors (penalize()).
290
+ private rateCooldownUntil: Map<string, number> = new Map();
254
291
 
255
292
  constructor(
256
293
  private readonly state: DurableObjectState,
@@ -269,10 +306,35 @@ export class PlayDedup implements DurableObject {
269
306
  return `${COORDINATOR_RUN_EVENT_KEY_PREFIX}${String(event.seq).padStart(13, '0')}`;
270
307
  }
271
308
 
309
+ private runEventKeyForSeq(seq: number): string {
310
+ return `${COORDINATOR_RUN_EVENT_KEY_PREFIX}${String(seq).padStart(13, '0')}`;
311
+ }
312
+
272
313
  private waiterKey(toolId: string, inputHash: string): string {
273
314
  return `${toolId}:${inputHash}`;
274
315
  }
275
316
 
317
+ private childTerminalKey(eventKey: string): string {
318
+ return `${COORDINATOR_CHILD_TERMINAL_KEY_PREFIX}${eventKey}`;
319
+ }
320
+
321
+ private wakeChildTerminalWaiters(eventKey: string): void {
322
+ const waiters = this.childTerminalWaiters.get(eventKey);
323
+ if (!waiters) return;
324
+ for (const resolve of waiters) {
325
+ resolve();
326
+ }
327
+ this.childTerminalWaiters.delete(eventKey);
328
+ }
329
+
330
+ private async readChildTerminalState(
331
+ eventKey: string,
332
+ ): Promise<CoordinatorChildTerminalState | undefined> {
333
+ return await this.state.storage.get<CoordinatorChildTerminalState>(
334
+ this.childTerminalKey(eventKey),
335
+ );
336
+ }
337
+
276
338
  async fetch(req: Request): Promise<Response> {
277
339
  const url = new URL(req.url);
278
340
  try {
@@ -287,36 +349,20 @@ export class PlayDedup implements DurableObject {
287
349
  return await this.handleFinish(req);
288
350
  case '/debug':
289
351
  return await this.handleDebug(req);
290
- case '/pool-add':
291
- return await this.handlePoolAdd(req);
292
- case '/pool-claim':
293
- return await this.handlePoolClaim(req);
294
- case '/pool-count':
295
- return await this.handlePoolCount(req);
296
- case '/pool-list':
297
- return await this.handlePoolList(req);
298
- case '/pool-promote':
299
- return await this.handlePoolPromote(req);
300
- case '/pool-ready':
301
- return await this.handlePoolReady(req);
302
- case '/pool-delete':
303
- return await this.handlePoolDelete(req);
304
- case '/pool-map-run':
305
- return await this.handlePoolMapRun(req);
306
- case '/pool-block-run':
307
- return await this.handlePoolBlockRun(req);
308
- case '/pool-resolve-run':
309
- return await this.handlePoolResolveRun(req);
310
352
  case '/run-retry-state-put':
311
353
  return await this.handleRunRetryStatePut(req);
354
+ case '/run-launch-state-put':
355
+ return await this.handleRunLaunchStatePut(req);
356
+ case '/workflow-instance-put':
357
+ return await this.handleWorkflowInstancePut(req);
358
+ case '/workflow-instance-get':
359
+ return await this.handleWorkflowInstanceGet(req);
312
360
  case '/run-retry-claim':
313
361
  return await this.handleRunRetryClaim(req);
314
362
  case '/db-sessions-put':
315
363
  return await this.handleDbSessionsPut(req);
316
364
  case '/db-sessions-get':
317
365
  return await this.handleDbSessionsGet(req);
318
- case '/pool-clear':
319
- return await this.handlePoolClear(req);
320
366
  case '/trace-add':
321
367
  return await this.handleTraceAdd(req);
322
368
  case '/trace-list':
@@ -329,6 +375,16 @@ export class PlayDedup implements DurableObject {
329
375
  return await this.handleTerminalSet(req);
330
376
  case '/terminal-get':
331
377
  return await this.handleTerminalGet();
378
+ case '/child-terminal-set':
379
+ return await this.handleChildTerminalSet(req);
380
+ case '/child-terminal-get':
381
+ return await this.handleChildTerminalGet(req);
382
+ case '/child-terminal-await':
383
+ return await this.handleChildTerminalAwait(req);
384
+ case '/rate-acquire':
385
+ return await this.handleRateAcquire(req);
386
+ case '/rate-penalize':
387
+ return await this.handleRatePenalize(req);
332
388
  default:
333
389
  return new Response('not found', { status: 404 });
334
390
  }
@@ -343,10 +399,11 @@ export class PlayDedup implements DurableObject {
343
399
 
344
400
  async alarm(): Promise<void> {
345
401
  // Fired after /finish was called. Evict storage and let the DO instance
346
- // be garbage-collected by CF's hot pool.
402
+ // be garbage-collected by Cloudflare.
347
403
  await this.state.storage.deleteAll();
348
404
  this.waiters.clear();
349
405
  this.runEventWaiters.clear();
406
+ this.childTerminalWaiters.clear();
350
407
  }
351
408
 
352
409
  private async handleLookupOrClaim(req: Request): Promise<Response> {
@@ -514,575 +571,125 @@ export class PlayDedup implements DurableObject {
514
571
  );
515
572
  }
516
573
 
517
- private workflowPoolVersion(req: Request): string {
518
- return new URL(req.url).searchParams.get('version')?.trim() ?? '';
519
- }
520
-
521
- private workflowPoolMinReadyAgeMs(req: Request): number {
522
- const raw = Number(new URL(req.url).searchParams.get('minReadyAgeMs') ?? 0);
523
- if (!Number.isFinite(raw) || raw <= 0) {
524
- return 0;
525
- }
526
- return Math.min(Math.floor(raw), WORKFLOW_POOL_DEFAULT_TTL_MS);
527
- }
528
-
529
- private isReadyWorkflowPoolEntry(
530
- value: { key: string; entry: WorkflowPoolEntry | undefined },
531
- version: string,
532
- now: number,
533
- minReadyAgeMs = 0,
534
- ): value is ReadyWorkflowPoolEntryRecord {
535
- return (
536
- value.entry !== undefined &&
537
- value.entry.version === version &&
538
- value.entry.expiresAt > now &&
539
- value.entry.state === 'ready' &&
540
- value.entry.readyAt !== null &&
541
- now - value.entry.readyAt <= WORKFLOW_POOL_READY_MAX_AGE_MS &&
542
- now - value.entry.readyAt >= minReadyAgeMs
543
- );
544
- }
545
-
546
- private async gcWorkflowPool(
547
- now = Date.now(),
548
- version?: string,
549
- ): Promise<void> {
550
- const [pool, mappings, retries] = await Promise.all([
551
- this.state.storage.list<WorkflowPoolEntry>({
552
- prefix: WORKFLOW_POOL_KEY_PREFIX,
553
- }),
554
- this.state.storage.list<WorkflowRunMapping>({
555
- prefix: WORKFLOW_POOL_RUN_KEY_PREFIX,
556
- }),
557
- this.state.storage.list<WorkflowRunRetryState>({
558
- prefix: WORKFLOW_RUN_RETRY_KEY_PREFIX,
559
- }),
560
- ]);
561
- const expiredKeys: string[] = [];
562
- for (const [key, entry] of pool) {
563
- if (
564
- !entry ||
565
- entry.expiresAt <= now ||
566
- (entry.state === 'ready' &&
567
- entry.readyAt !== null &&
568
- now - entry.readyAt > WORKFLOW_POOL_READY_MAX_AGE_MS) ||
569
- (version && entry.version !== version)
570
- ) {
571
- expiredKeys.push(key);
572
- }
573
- }
574
- for (const [key, mapping] of mappings) {
575
- if (
576
- !mapping ||
577
- mapping.expiresAt <= now ||
578
- (version && mapping.version !== version)
579
- ) {
580
- expiredKeys.push(key);
581
- }
582
- }
583
- for (const [key, retryState] of retries) {
584
- if (!retryState || retryState.expiresAt <= now) {
585
- expiredKeys.push(key);
586
- }
587
- }
588
- if (expiredKeys.length > 0) {
589
- await this.state.storage.delete(expiredKeys);
590
- }
591
- }
592
-
593
- private async handlePoolAdd(req: Request): Promise<Response> {
594
- const body = (await req.json().catch(() => null)) as {
595
- ids?: unknown;
596
- ttlMs?: unknown;
597
- version?: unknown;
598
- readyAt?: unknown;
599
- ready?: unknown;
600
- } | null;
601
- const version =
602
- typeof body?.version === 'string' ? body.version.trim() : '';
603
- if (!version) {
604
- return new Response('version is required', { status: 400 });
605
- }
606
- const ids = Array.isArray(body?.ids)
607
- ? body.ids.filter(
608
- (id): id is string => typeof id === 'string' && id.length > 0,
609
- )
610
- : [];
611
- const now = Date.now();
612
- const hasReadyAt =
613
- typeof body?.readyAt === 'number' && Number.isFinite(body.readyAt);
614
- const readyAt = hasReadyAt ? Math.max(0, body.readyAt as number) : now;
615
- const ready = body?.ready === true || hasReadyAt;
616
- const ttlMs =
617
- typeof body?.ttlMs === 'number' &&
618
- Number.isFinite(body.ttlMs) &&
619
- body.ttlMs > 0
620
- ? Math.min(body.ttlMs, WORKFLOW_POOL_DEFAULT_TTL_MS)
621
- : WORKFLOW_POOL_DEFAULT_TTL_MS;
622
- await this.state.blockConcurrencyWhile(async () => {
623
- await this.gcWorkflowPool(now, version);
624
- const writes: Record<string, WorkflowPoolEntry> = {};
625
- const keys = ids.map((id) => `${WORKFLOW_POOL_KEY_PREFIX}${id}`);
626
- const existing = (await this.state.storage.get<WorkflowPoolEntry>(
627
- keys,
628
- )) as Map<string, WorkflowPoolEntry>;
629
- for (const id of ids) {
630
- const key = `${WORKFLOW_POOL_KEY_PREFIX}${id}`;
631
- const existingEntry = existing.get(key);
632
- const existingReadyAt =
633
- existingEntry?.version === version &&
634
- existingEntry.expiresAt > now &&
635
- existingEntry.state === 'ready' &&
636
- existingEntry.readyAt !== null
637
- ? existingEntry.readyAt
638
- : null;
639
- const nextReadyAt = ready ? readyAt : existingReadyAt;
640
- writes[key] = {
641
- id,
642
- version,
643
- state: nextReadyAt !== null ? 'ready' : 'warming',
644
- createdAt:
645
- existingEntry?.version === version ? existingEntry.createdAt : now,
646
- readyAt: nextReadyAt,
647
- expiresAt: now + ttlMs,
648
- };
649
- }
650
- if (Object.keys(writes).length > 0) {
651
- await this.state.storage.put(writes);
652
- }
653
- });
654
- return new Response(JSON.stringify({ added: ids.length }), {
655
- headers: { 'content-type': 'application/json' },
656
- });
657
- }
658
-
659
- private async handlePoolClaim(req: Request): Promise<Response> {
660
- const version = this.workflowPoolVersion(req);
661
- if (!version) {
662
- return new Response('version is required', { status: 400 });
663
- }
664
- const body = (await req.json().catch(() => null)) as {
665
- runId?: unknown;
666
- } | null;
667
- const runId = typeof body?.runId === 'string' ? body.runId : '';
668
- if (!runId) {
669
- return new Response('runId is required', { status: 400 });
670
- }
671
- const minReadyAgeMs = this.workflowPoolMinReadyAgeMs(req);
672
- let claimedId: string | null = null;
673
- let counts: WorkflowPoolCounts = { available: 0, warming: 0 };
674
- await this.state.blockConcurrencyWhile(async () => {
675
- const now = Date.now();
676
- await this.gcWorkflowPool(now, version);
677
- const mappingKey = `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`;
678
- const existingMapping =
679
- await this.state.storage.get<WorkflowRunMapping>(mappingKey);
680
- if (existingMapping?.version === version) {
681
- if (
682
- existingMapping.state === 'claimed' ||
683
- existingMapping.state === 'started'
684
- ) {
685
- claimedId = existingMapping.instanceId || null;
686
- }
687
- return;
688
- }
689
- const entries = await this.state.storage.list<WorkflowPoolEntry>({
690
- prefix: WORKFLOW_POOL_KEY_PREFIX,
691
- });
692
- counts = this.countWorkflowPoolEntries(
693
- entries,
694
- version,
695
- now,
696
- minReadyAgeMs,
697
- );
698
- const sorted = [...entries.entries()]
699
- .map(([key, entry]) => ({ key, entry }))
700
- .filter((entry) =>
701
- this.isReadyWorkflowPoolEntry(entry, version, now, minReadyAgeMs),
702
- )
703
- .sort((a, b) => b.entry.readyAt - a.entry.readyAt);
704
- const selected = sorted[0];
705
- if (!selected) return;
706
- claimedId = selected.entry.id;
707
- await this.state.storage.delete(selected.key);
708
- await this.state.storage.put(mappingKey, {
709
- runId,
710
- instanceId: claimedId,
711
- state: 'claimed',
712
- blockedInstanceId: null,
713
- claimedAt: now,
714
- startedAt: null,
715
- version,
716
- createdAt: now,
717
- expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
718
- } satisfies WorkflowRunMapping);
719
- counts = {
720
- available: Math.max(0, counts.available - 1),
721
- warming: counts.warming,
722
- };
723
- });
724
- return new Response(JSON.stringify({ id: claimedId, ...counts }), {
725
- headers: { 'content-type': 'application/json' },
726
- });
727
- }
728
-
729
- private async handlePoolCount(req: Request): Promise<Response> {
730
- const version = this.workflowPoolVersion(req);
731
- if (!version) {
732
- return new Response('version is required', { status: 400 });
733
- }
734
- const minReadyAgeMs = this.workflowPoolMinReadyAgeMs(req);
735
- const now = Date.now();
736
- await this.gcWorkflowPool(now, version);
737
- const entries = await this.state.storage.list<WorkflowPoolEntry>({
738
- prefix: WORKFLOW_POOL_KEY_PREFIX,
739
- });
740
- const counts = this.countWorkflowPoolEntries(
741
- entries,
742
- version,
743
- now,
744
- minReadyAgeMs,
745
- );
746
- return new Response(JSON.stringify(counts), {
747
- headers: { 'content-type': 'application/json' },
748
- });
749
- }
750
-
751
- private countWorkflowPoolEntries(
752
- entries: Map<string, WorkflowPoolEntry>,
753
- version: string,
754
- now: number,
755
- minReadyAgeMs: number,
756
- ): WorkflowPoolCounts {
757
- let available = 0;
758
- let warming = 0;
759
- for (const entry of entries.values()) {
760
- if (entry.version !== version) continue;
761
- if (
762
- entry.state !== 'ready' ||
763
- entry.readyAt === null ||
764
- now - entry.readyAt > WORKFLOW_POOL_READY_MAX_AGE_MS ||
765
- now - entry.readyAt < minReadyAgeMs
766
- ) {
767
- warming += 1;
768
- } else {
769
- available += 1;
770
- }
771
- }
772
- return { available, warming };
773
- }
774
-
775
- private async handlePoolList(req: Request): Promise<Response> {
776
- const version = this.workflowPoolVersion(req);
777
- if (!version) {
778
- return new Response('version is required', { status: 400 });
779
- }
780
- await this.gcWorkflowPool(Date.now(), version);
781
- const entries = await this.state.storage.list<WorkflowPoolEntry>({
782
- prefix: WORKFLOW_POOL_KEY_PREFIX,
783
- });
784
- return new Response(
785
- JSON.stringify({
786
- entries: [...entries.values()]
787
- .filter((entry) => entry.version === version)
788
- .map((entry) => ({
789
- id: entry.id,
790
- state: entry.state,
791
- createdAt: entry.createdAt,
792
- readyAt: entry.readyAt,
793
- expiresAt: entry.expiresAt,
794
- })),
795
- }),
796
- { headers: { 'content-type': 'application/json' } },
797
- );
798
- }
799
-
800
- private async handlePoolPromote(req: Request): Promise<Response> {
801
- const body = (await req.json().catch(() => null)) as {
802
- ids?: unknown;
803
- version?: unknown;
804
- } | null;
805
- const version =
806
- typeof body?.version === 'string' ? body.version.trim() : '';
807
- if (!version) {
808
- return new Response('version is required', { status: 400 });
809
- }
810
- const ids = Array.isArray(body?.ids)
811
- ? body.ids.filter(
812
- (id): id is string => typeof id === 'string' && id.length > 0,
813
- )
814
- : [];
815
- const now = Date.now();
816
- const writes: Record<string, WorkflowPoolEntry> = {};
817
- await this.state.blockConcurrencyWhile(async () => {
818
- await this.gcWorkflowPool(now, version);
819
- for (const id of ids) {
820
- const key = `${WORKFLOW_POOL_KEY_PREFIX}${id}`;
821
- const entry = await this.state.storage.get<WorkflowPoolEntry>(key);
822
- if (!entry || entry.version !== version || entry.expiresAt <= now) {
823
- continue;
824
- }
825
- writes[key] = {
826
- ...entry,
827
- state: 'ready',
828
- readyAt: now,
829
- };
830
- }
831
- if (Object.keys(writes).length > 0) {
832
- await this.state.storage.put(writes);
833
- }
834
- });
835
- return new Response(
836
- JSON.stringify({ promoted: Object.keys(writes).length }),
837
- {
838
- headers: { 'content-type': 'application/json' },
839
- },
840
- );
841
- }
842
-
843
- private async handlePoolReady(req: Request): Promise<Response> {
844
- const body = (await req.json().catch(() => null)) as {
845
- poolId?: unknown;
846
- version?: unknown;
847
- } | null;
848
- const poolId = typeof body?.poolId === 'string' ? body.poolId : '';
849
- const version =
850
- typeof body?.version === 'string' ? body.version.trim() : '';
851
- if (!poolId || !version) {
852
- return new Response('poolId and version are required', { status: 400 });
853
- }
854
- const now = Date.now();
855
- let ready = false;
856
- await this.state.blockConcurrencyWhile(async () => {
857
- await this.gcWorkflowPool(now, version);
858
- const key = `${WORKFLOW_POOL_KEY_PREFIX}${poolId}`;
859
- const entry = await this.state.storage.get<WorkflowPoolEntry>(key);
860
- if (!entry || entry.version !== version || entry.expiresAt <= now) {
861
- return;
862
- }
863
- await this.state.storage.put(key, {
864
- ...entry,
865
- state: 'ready',
866
- readyAt: entry.readyAt ?? now,
867
- });
868
- ready = true;
869
- });
870
- return new Response(JSON.stringify({ ready }), {
871
- headers: { 'content-type': 'application/json' },
872
- });
873
- }
874
-
875
- private async handlePoolDelete(req: Request): Promise<Response> {
876
- const body = (await req.json().catch(() => null)) as {
877
- ids?: unknown;
878
- version?: unknown;
879
- } | null;
880
- const version =
881
- typeof body?.version === 'string' ? body.version.trim() : '';
882
- if (!version) {
883
- return new Response('version is required', { status: 400 });
884
- }
885
- const ids = Array.isArray(body?.ids)
886
- ? body.ids.filter(
887
- (id): id is string => typeof id === 'string' && id.length > 0,
888
- )
889
- : [];
890
- const keys: string[] = [];
891
- await this.state.blockConcurrencyWhile(async () => {
892
- await this.gcWorkflowPool(Date.now(), version);
893
- for (const id of ids) {
894
- const key = `${WORKFLOW_POOL_KEY_PREFIX}${id}`;
895
- const entry = await this.state.storage.get<WorkflowPoolEntry>(key);
896
- if (entry?.version === version) keys.push(key);
897
- }
898
- if (keys.length > 0) {
899
- await this.state.storage.delete(keys);
900
- }
901
- });
902
- return new Response(JSON.stringify({ deleted: keys.length }), {
903
- headers: { 'content-type': 'application/json' },
904
- });
905
- }
906
-
907
- private async handlePoolMapRun(req: Request): Promise<Response> {
574
+ private async handleRunRetryStatePut(req: Request): Promise<Response> {
908
575
  const body = (await req.json().catch(() => null)) as {
909
576
  runId?: unknown;
910
- instanceId?: unknown;
911
- version?: unknown;
912
- started?: unknown;
577
+ params?: unknown;
578
+ ttlMs?: unknown;
913
579
  } | null;
914
580
  const runId = typeof body?.runId === 'string' ? body.runId : '';
915
- const instanceId =
916
- typeof body?.instanceId === 'string' ? body.instanceId : '';
917
- const version =
918
- typeof body?.version === 'string' ? body.version.trim() : '';
919
- const started = body?.started === true;
920
- if (!runId || !instanceId || !version) {
921
- return new Response('runId, instanceId, and version are required', {
581
+ if (!runId || !body || (!('params' in body) && !('paramsRef' in body))) {
582
+ return new Response('runId and params or paramsRef are required', {
922
583
  status: 400,
923
584
  });
924
585
  }
925
586
  const now = Date.now();
926
- let mapped = true;
927
- const key = `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`;
587
+ const ttlMs =
588
+ typeof body.ttlMs === 'number' && Number.isFinite(body.ttlMs)
589
+ ? Math.max(60_000, Math.min(body.ttlMs, WORKFLOW_RUN_STATE_TTL_MS))
590
+ : WORKFLOW_RUN_STATE_TTL_MS;
591
+ const key = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
928
592
  await this.state.blockConcurrencyWhile(async () => {
929
- const existing = await this.state.storage.get<WorkflowRunMapping>(key);
930
- if (
931
- existing?.version === version &&
932
- existing.state === 'blocked' &&
933
- existing.blockedInstanceId === instanceId
934
- ) {
935
- mapped = false;
936
- return;
937
- }
938
- const alreadyStarted =
939
- existing?.version === version &&
940
- existing.instanceId === instanceId &&
941
- existing.state === 'started' &&
942
- typeof existing.startedAt === 'number';
943
- const startedAt = started || alreadyStarted ? now : null;
944
- await this.state.storage.put(key, {
593
+ const existing = await this.state.storage.get<WorkflowRunRetryState>(key);
594
+ const retryState = {
945
595
  runId,
946
- instanceId,
947
- state: startedAt !== null ? 'started' : 'claimed',
948
- blockedInstanceId: null,
949
- claimedAt:
950
- existing?.version === version &&
951
- typeof existing.claimedAt === 'number'
952
- ? existing.claimedAt
953
- : now,
954
- startedAt,
955
- version,
956
- createdAt:
957
- existing?.version === version && existing.createdAt
958
- ? existing.createdAt
959
- : now,
960
- expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
961
- } satisfies WorkflowRunMapping);
962
- });
963
- return new Response(JSON.stringify({ mapped }), {
964
- headers: { 'content-type': 'application/json' },
965
- });
966
- }
967
-
968
- private async handlePoolBlockRun(req: Request): Promise<Response> {
969
- const body = (await req.json().catch(() => null)) as {
970
- runId?: unknown;
971
- instanceId?: unknown;
972
- version?: unknown;
973
- } | null;
974
- const runId = typeof body?.runId === 'string' ? body.runId : '';
975
- const instanceId =
976
- typeof body?.instanceId === 'string' ? body.instanceId : '';
977
- const version =
978
- typeof body?.version === 'string' ? body.version.trim() : '';
979
- if (!runId || !instanceId || !version) {
980
- return new Response('runId, instanceId, and version are required', {
981
- status: 400,
982
- });
983
- }
984
- const now = Date.now();
985
- const key = `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`;
986
- let started = false;
987
- await this.state.blockConcurrencyWhile(async () => {
988
- const existing = await this.state.storage.get<WorkflowRunMapping>(key);
989
- if (
990
- existing?.version === version &&
991
- existing.instanceId === instanceId &&
992
- existing.state === 'started' &&
993
- typeof existing.startedAt === 'number'
994
- ) {
995
- started = true;
996
- return;
596
+ params: 'params' in body ? body.params : null,
597
+ paramsRef: 'paramsRef' in body ? body.paramsRef : null,
598
+ paramsBytes:
599
+ typeof (body as { paramsBytes?: unknown }).paramsBytes === 'number' &&
600
+ Number.isFinite((body as { paramsBytes?: number }).paramsBytes)
601
+ ? (body as { paramsBytes: number }).paramsBytes
602
+ : undefined,
603
+ retryAttempts:
604
+ existing?.runId === runId &&
605
+ typeof existing.retryAttempts === 'number'
606
+ ? existing.retryAttempts
607
+ : 0,
608
+ updatedAt: now,
609
+ expiresAt: now + ttlMs,
610
+ } satisfies WorkflowRunRetryState;
611
+ const bytes = jsonByteLength(retryState);
612
+ if (bytes > WORKFLOW_RUN_RETRY_STATE_MAX_BYTES) {
613
+ throw new Error(
614
+ `workflow retry state too large: ${bytes} bytes exceeds ${WORKFLOW_RUN_RETRY_STATE_MAX_BYTES}`,
615
+ );
997
616
  }
998
- await this.state.storage.put(key, {
999
- runId,
1000
- instanceId: '',
1001
- state: 'blocked',
1002
- blockedInstanceId: instanceId,
1003
- claimedAt:
1004
- existing?.version === version &&
1005
- typeof existing.claimedAt === 'number'
1006
- ? existing.claimedAt
1007
- : now,
1008
- startedAt: null,
1009
- version,
1010
- createdAt: now,
1011
- expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
1012
- } satisfies WorkflowRunMapping);
617
+ await this.state.storage.put(key, retryState);
1013
618
  });
1014
- if (started) {
1015
- return new Response(JSON.stringify({ blocked: false, started: true }), {
1016
- headers: { 'content-type': 'application/json' },
1017
- });
1018
- }
1019
- return new Response(JSON.stringify({ blocked: true, started: false }), {
619
+ return new Response(JSON.stringify({ ok: true }), {
1020
620
  headers: { 'content-type': 'application/json' },
1021
621
  });
1022
622
  }
1023
623
 
1024
- private async handlePoolResolveRun(req: Request): Promise<Response> {
1025
- const url = new URL(req.url);
1026
- const runId = url.searchParams.get('runId') ?? '';
1027
- const version = url.searchParams.get('version')?.trim() ?? '';
1028
- if (!runId) {
1029
- return new Response('runId is required', { status: 400 });
1030
- }
1031
- if (!version) {
1032
- return new Response('version is required', { status: 400 });
1033
- }
1034
- await this.gcWorkflowPool(Date.now(), version);
1035
- const mapping = await this.state.storage.get<WorkflowRunMapping>(
1036
- `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`,
1037
- );
1038
- if (mapping && mapping.version !== version) {
1039
- await this.state.storage.delete(
1040
- `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`,
1041
- );
1042
- return new Response(JSON.stringify({ instanceId: null }), {
1043
- headers: { 'content-type': 'application/json' },
1044
- });
1045
- }
1046
- return new Response(
1047
- JSON.stringify({
1048
- instanceId: mapping?.instanceId || null,
1049
- startedAt: mapping?.startedAt ?? null,
1050
- }),
1051
- { headers: { 'content-type': 'application/json' } },
1052
- );
1053
- }
1054
-
1055
- private async handleRunRetryStatePut(req: Request): Promise<Response> {
624
+ private async handleRunLaunchStatePut(req: Request): Promise<Response> {
1056
625
  const body = (await req.json().catch(() => null)) as {
1057
626
  runId?: unknown;
1058
627
  params?: unknown;
1059
- ttlMs?: unknown;
628
+ paramsRef?: unknown;
629
+ paramsBytes?: unknown;
630
+ sessions?: unknown;
631
+ retryTtlMs?: unknown;
632
+ dbSessionsTtlMs?: unknown;
1060
633
  } | null;
1061
634
  const runId = typeof body?.runId === 'string' ? body.runId : '';
1062
- if (!runId || !body || !('params' in body)) {
1063
- return new Response('runId and params are required', { status: 400 });
635
+ const sessions = Array.isArray(body?.sessions)
636
+ ? (body.sessions as PreloadedRuntimeDbSession[])
637
+ : null;
638
+ if (
639
+ !runId ||
640
+ !body ||
641
+ (!('params' in body) && !('paramsRef' in body)) ||
642
+ !sessions
643
+ ) {
644
+ return new Response(
645
+ 'runId, params or paramsRef, and sessions are required',
646
+ {
647
+ status: 400,
648
+ },
649
+ );
1064
650
  }
651
+ assertEncryptedDbSessionsForStorage(sessions);
1065
652
  const now = Date.now();
1066
- const ttlMs =
1067
- typeof body.ttlMs === 'number' && Number.isFinite(body.ttlMs)
1068
- ? Math.max(
1069
- 60_000,
1070
- Math.min(body.ttlMs, WORKFLOW_POOL_RUN_MAPPING_TTL_MS),
653
+ const retryTtlMs =
654
+ typeof body.retryTtlMs === 'number' && Number.isFinite(body.retryTtlMs)
655
+ ? Math.max(60_000, Math.min(body.retryTtlMs, WORKFLOW_RUN_STATE_TTL_MS))
656
+ : WORKFLOW_RUN_STATE_TTL_MS;
657
+ const dbSessionsTtlMs =
658
+ typeof body.dbSessionsTtlMs === 'number' &&
659
+ Number.isFinite(body.dbSessionsTtlMs) &&
660
+ body.dbSessionsTtlMs > 0
661
+ ? Math.min(
662
+ Math.max(Math.floor(body.dbSessionsTtlMs), 60_000),
663
+ 30 * 60_000,
1071
664
  )
1072
- : WORKFLOW_POOL_RUN_MAPPING_TTL_MS;
1073
- const key = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
665
+ : WORKFLOW_DB_SESSIONS_TTL_MS;
666
+ const retryKey = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
667
+ const dbSessionsState: WorkflowDbSessionsState = {
668
+ runId,
669
+ sessions,
670
+ storedAt: now,
671
+ expiresAt: now + dbSessionsTtlMs,
672
+ };
673
+ let retryStateExpiresAt = now + retryTtlMs;
1074
674
  await this.state.blockConcurrencyWhile(async () => {
1075
- const existing = await this.state.storage.get<WorkflowRunRetryState>(key);
675
+ const existing =
676
+ await this.state.storage.get<WorkflowRunRetryState>(retryKey);
1076
677
  const retryState = {
1077
678
  runId,
1078
- params: body.params,
679
+ params: 'params' in body ? body.params : null,
680
+ paramsRef: 'paramsRef' in body ? body.paramsRef : null,
681
+ paramsBytes:
682
+ typeof body.paramsBytes === 'number' &&
683
+ Number.isFinite(body.paramsBytes)
684
+ ? body.paramsBytes
685
+ : undefined,
1079
686
  retryAttempts:
1080
687
  existing?.runId === runId &&
1081
688
  typeof existing.retryAttempts === 'number'
1082
689
  ? existing.retryAttempts
1083
690
  : 0,
1084
691
  updatedAt: now,
1085
- expiresAt: now + ttlMs,
692
+ expiresAt: retryStateExpiresAt,
1086
693
  } satisfies WorkflowRunRetryState;
1087
694
  const bytes = jsonByteLength(retryState);
1088
695
  if (bytes > WORKFLOW_RUN_RETRY_STATE_MAX_BYTES) {
@@ -1090,11 +697,19 @@ export class PlayDedup implements DurableObject {
1090
697
  `workflow retry state too large: ${bytes} bytes exceeds ${WORKFLOW_RUN_RETRY_STATE_MAX_BYTES}`,
1091
698
  );
1092
699
  }
1093
- await this.state.storage.put(key, retryState);
1094
- });
1095
- return new Response(JSON.stringify({ ok: true }), {
1096
- headers: { 'content-type': 'application/json' },
700
+ retryStateExpiresAt = retryState.expiresAt;
701
+ await this.state.storage.put(retryKey, retryState);
702
+ await this.state.storage.put(WORKFLOW_DB_SESSIONS_KEY, dbSessionsState);
1097
703
  });
704
+ return new Response(
705
+ JSON.stringify({
706
+ ok: true,
707
+ sessionCount: sessions.length,
708
+ retryExpiresAt: retryStateExpiresAt,
709
+ dbSessionsExpiresAt: dbSessionsState.expiresAt,
710
+ }),
711
+ { headers: { 'content-type': 'application/json' } },
712
+ );
1098
713
  }
1099
714
 
1100
715
  private async handleRunRetryClaim(req: Request): Promise<Response> {
@@ -1128,6 +743,8 @@ export class PlayDedup implements DurableObject {
1128
743
  claimed: false,
1129
744
  attempts: existing.retryAttempts,
1130
745
  params: existing.params,
746
+ paramsRef: existing.paramsRef ?? null,
747
+ paramsBytes: existing.paramsBytes ?? null,
1131
748
  };
1132
749
  return;
1133
750
  }
@@ -1136,12 +753,14 @@ export class PlayDedup implements DurableObject {
1136
753
  ...existing,
1137
754
  retryAttempts: nextAttempts,
1138
755
  updatedAt: now,
1139
- expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
756
+ expiresAt: now + WORKFLOW_RUN_STATE_TTL_MS,
1140
757
  } satisfies WorkflowRunRetryState);
1141
758
  response = {
1142
759
  claimed: true,
1143
760
  attempts: nextAttempts,
1144
761
  params: existing.params,
762
+ paramsRef: existing.paramsRef ?? null,
763
+ paramsBytes: existing.paramsBytes ?? null,
1145
764
  };
1146
765
  });
1147
766
  return new Response(JSON.stringify(response), {
@@ -1149,6 +768,59 @@ export class PlayDedup implements DurableObject {
1149
768
  });
1150
769
  }
1151
770
 
771
+ private async handleWorkflowInstancePut(req: Request): Promise<Response> {
772
+ const body = (await req.json().catch(() => null)) as {
773
+ runId?: unknown;
774
+ instanceId?: unknown;
775
+ ttlMs?: unknown;
776
+ } | null;
777
+ const runId = typeof body?.runId === 'string' ? body.runId : '';
778
+ const instanceId =
779
+ typeof body?.instanceId === 'string' ? body.instanceId : '';
780
+ if (!runId || !instanceId) {
781
+ return new Response('runId and instanceId are required', { status: 400 });
782
+ }
783
+ const now = Date.now();
784
+ const ttlMs =
785
+ typeof body?.ttlMs === 'number' &&
786
+ Number.isFinite(body.ttlMs) &&
787
+ body.ttlMs > 0
788
+ ? Math.max(60_000, Math.min(body.ttlMs, WORKFLOW_RUN_STATE_TTL_MS))
789
+ : WORKFLOW_RUN_STATE_TTL_MS;
790
+ await this.state.storage.put('workflow-instance', {
791
+ runId,
792
+ instanceId,
793
+ updatedAt: now,
794
+ expiresAt: now + ttlMs,
795
+ } satisfies WorkflowInstanceState);
796
+ return new Response(JSON.stringify({ ok: true }), {
797
+ headers: { 'content-type': 'application/json' },
798
+ });
799
+ }
800
+
801
+ private async handleWorkflowInstanceGet(req: Request): Promise<Response> {
802
+ const runId = new URL(req.url).searchParams.get('runId') ?? '';
803
+ if (!runId) {
804
+ return new Response('runId is required', { status: 400 });
805
+ }
806
+ const state =
807
+ await this.state.storage.get<WorkflowInstanceState>('workflow-instance');
808
+ if (!state || state.runId !== runId) {
809
+ return new Response(JSON.stringify({ instanceId: null }), {
810
+ headers: { 'content-type': 'application/json' },
811
+ });
812
+ }
813
+ if (state.expiresAt <= Date.now()) {
814
+ await this.state.storage.delete('workflow-instance');
815
+ return new Response(JSON.stringify({ instanceId: null }), {
816
+ headers: { 'content-type': 'application/json' },
817
+ });
818
+ }
819
+ return new Response(JSON.stringify({ instanceId: state.instanceId }), {
820
+ headers: { 'content-type': 'application/json' },
821
+ });
822
+ }
823
+
1152
824
  private async handleDbSessionsPut(req: Request): Promise<Response> {
1153
825
  const body = (await req.json().catch(() => null)) as {
1154
826
  runId?: unknown;
@@ -1220,36 +892,6 @@ export class PlayDedup implements DurableObject {
1220
892
  );
1221
893
  }
1222
894
 
1223
- private async handlePoolClear(req: Request): Promise<Response> {
1224
- const version = this.workflowPoolVersion(req);
1225
- const [pool, mappings, retries] = await Promise.all([
1226
- this.state.storage.list<WorkflowPoolEntry>({
1227
- prefix: WORKFLOW_POOL_KEY_PREFIX,
1228
- }),
1229
- this.state.storage.list<WorkflowRunMapping>({
1230
- prefix: WORKFLOW_POOL_RUN_KEY_PREFIX,
1231
- }),
1232
- this.state.storage.list<WorkflowRunRetryState>({
1233
- prefix: WORKFLOW_RUN_RETRY_KEY_PREFIX,
1234
- }),
1235
- ]);
1236
- const keys = [
1237
- ...[...pool.entries()]
1238
- .filter(([, entry]) => !version || entry.version === version)
1239
- .map(([key]) => key),
1240
- ...[...mappings.entries()]
1241
- .filter(([, entry]) => !version || entry.version === version)
1242
- .map(([key]) => key),
1243
- ...[...retries.keys()],
1244
- ];
1245
- if (keys.length > 0) {
1246
- await this.state.storage.delete(keys);
1247
- }
1248
- return new Response(JSON.stringify({ deleted: keys.length }), {
1249
- headers: { 'content-type': 'application/json' },
1250
- });
1251
- }
1252
-
1253
895
  private async handleTraceAdd(req: Request): Promise<Response> {
1254
896
  const body = (await req
1255
897
  .json()
@@ -1365,6 +1007,226 @@ export class PlayDedup implements DurableObject {
1365
1007
  });
1366
1008
  }
1367
1009
 
1010
+ private async handleChildTerminalSet(req: Request): Promise<Response> {
1011
+ const body = (await req.json().catch(() => null)) as {
1012
+ eventKey?: unknown;
1013
+ data?: unknown;
1014
+ storedAt?: unknown;
1015
+ } | null;
1016
+ const eventKey =
1017
+ typeof body?.eventKey === 'string' && body.eventKey.trim()
1018
+ ? body.eventKey.trim()
1019
+ : '';
1020
+ if (!eventKey) {
1021
+ return new Response('eventKey is required', { status: 400 });
1022
+ }
1023
+ const state: CoordinatorChildTerminalState = {
1024
+ eventKey,
1025
+ data: body?.data ?? null,
1026
+ storedAt:
1027
+ typeof body?.storedAt === 'number' && Number.isFinite(body.storedAt)
1028
+ ? body.storedAt
1029
+ : Date.now(),
1030
+ };
1031
+ await this.state.storage.put(this.childTerminalKey(eventKey), state);
1032
+ this.wakeChildTerminalWaiters(eventKey);
1033
+ return new Response('{}', {
1034
+ headers: { 'content-type': 'application/json' },
1035
+ });
1036
+ }
1037
+
1038
+ private async handleChildTerminalGet(req: Request): Promise<Response> {
1039
+ const eventKey = new URL(req.url).searchParams.get('eventKey')?.trim();
1040
+ if (!eventKey) {
1041
+ return new Response('eventKey is required', { status: 400 });
1042
+ }
1043
+ const state = await this.readChildTerminalState(eventKey);
1044
+ return new Response(JSON.stringify({ state: state ?? null }), {
1045
+ headers: { 'content-type': 'application/json' },
1046
+ });
1047
+ }
1048
+
1049
+ private async handleChildTerminalAwait(req: Request): Promise<Response> {
1050
+ const url = new URL(req.url);
1051
+ const eventKey = url.searchParams.get('eventKey')?.trim();
1052
+ if (!eventKey) {
1053
+ return new Response('eventKey is required', { status: 400 });
1054
+ }
1055
+ const timeoutMs = Math.min(
1056
+ Math.max(Number(url.searchParams.get('timeoutMs') ?? '0'), 0),
1057
+ 30_000,
1058
+ );
1059
+ let state = await this.readChildTerminalState(eventKey);
1060
+ if (!state && timeoutMs > 0) {
1061
+ await new Promise<void>((resolve) => {
1062
+ let settled = false;
1063
+ let timeout: ReturnType<typeof setTimeout>;
1064
+ const finish = () => {
1065
+ if (settled) return;
1066
+ settled = true;
1067
+ this.childTerminalWaiters.get(eventKey)?.delete(finish);
1068
+ if (this.childTerminalWaiters.get(eventKey)?.size === 0) {
1069
+ this.childTerminalWaiters.delete(eventKey);
1070
+ }
1071
+ clearTimeout(timeout);
1072
+ resolve();
1073
+ };
1074
+ timeout = setTimeout(finish, timeoutMs);
1075
+ if (!this.childTerminalWaiters.has(eventKey)) {
1076
+ this.childTerminalWaiters.set(eventKey, new Set());
1077
+ }
1078
+ this.childTerminalWaiters.get(eventKey)!.add(finish);
1079
+ });
1080
+ state = await this.readChildTerminalState(eventKey);
1081
+ }
1082
+ return new Response(JSON.stringify({ state: state ?? null }), {
1083
+ headers: { 'content-type': 'application/json' },
1084
+ });
1085
+ }
1086
+
1087
+ /**
1088
+ * Lease up to `requested` request-window permits for `bucketId` under all
1089
+ * `rules`. Single-threaded DO access means this is the same sliding-window
1090
+ * math as InMemoryRateStateBackend.acquire, generalized to grant a BLOCK of
1091
+ * permits at once so the runtime amortizes the round-trip across many calls.
1092
+ *
1093
+ * Concurrency rules (`maxConcurrency`) are NOT enforced here: simultaneous
1094
+ * in-flight tracking across isolates needs a reliable release signal, which a
1095
+ * dying isolate cannot guarantee. The Governor's global tool-concurrency
1096
+ * semaphore is the cross-call concurrency backstop; this DO owns the
1097
+ * cross-isolate REQUEST RATE, which is the throughput governor.
1098
+ */
1099
+ private async handleRateAcquire(req: Request): Promise<Response> {
1100
+ const body = (await req
1101
+ .json()
1102
+ .catch(() => null)) as RateAcquireRequest | null;
1103
+ if (
1104
+ !body ||
1105
+ typeof body.bucketId !== 'string' ||
1106
+ !body.bucketId.trim() ||
1107
+ !Array.isArray(body.rules)
1108
+ ) {
1109
+ return new Response('bucketId and rules are required', { status: 400 });
1110
+ }
1111
+ const requested =
1112
+ typeof body.requested === 'number' && Number.isFinite(body.requested)
1113
+ ? Math.max(1, Math.floor(body.requested))
1114
+ : 1;
1115
+ const result = this.computeRateAcquire(
1116
+ body.bucketId,
1117
+ body.rules,
1118
+ requested,
1119
+ );
1120
+ return new Response(JSON.stringify(result), {
1121
+ headers: { 'content-type': 'application/json' },
1122
+ });
1123
+ }
1124
+
1125
+ private computeRateAcquire(
1126
+ bucketId: string,
1127
+ rules: RateRule[],
1128
+ requested: number,
1129
+ ): RateAcquireResponse {
1130
+ const now = Date.now();
1131
+ const usableRules = rules.filter(
1132
+ (rule) =>
1133
+ rule &&
1134
+ typeof rule.ruleId === 'string' &&
1135
+ typeof rule.requestsPerWindow === 'number' &&
1136
+ rule.requestsPerWindow > 0 &&
1137
+ typeof rule.windowMs === 'number',
1138
+ );
1139
+ if (usableRules.length === 0) {
1140
+ return { granted: requested, waitMs: 0 };
1141
+ }
1142
+
1143
+ let waitMs = 0;
1144
+ const cooldownUntil = this.rateCooldownUntil.get(bucketId) ?? 0;
1145
+ if (cooldownUntil > now) waitMs = Math.max(waitMs, cooldownUntil - now);
1146
+
1147
+ // The grant is the min remaining capacity across every rule's window. A
1148
+ // single round-trip can never debit more than the tightest rule allows.
1149
+ let grantable = requested;
1150
+ for (const rule of usableRules) {
1151
+ const state = this.getRateWindowState(bucketId, rule, now);
1152
+ this.resetExpiredRateWindow(state, rule.windowMs, now);
1153
+ const remaining = rule.requestsPerWindow - state.startedInWindow;
1154
+ grantable = Math.min(grantable, Math.max(0, remaining));
1155
+ if (remaining <= 0) {
1156
+ waitMs = Math.max(
1157
+ waitMs,
1158
+ rule.windowMs > 0
1159
+ ? state.windowStartedAt + rule.windowMs - now
1160
+ : RATE_MIN_WAIT_MS,
1161
+ );
1162
+ }
1163
+ }
1164
+
1165
+ if (waitMs > 0 || grantable <= 0) {
1166
+ return { granted: 0, waitMs: Math.max(RATE_MIN_WAIT_MS, waitMs) };
1167
+ }
1168
+
1169
+ for (const rule of usableRules) {
1170
+ const state = this.getRateWindowState(bucketId, rule, now);
1171
+ state.startedInWindow += grantable;
1172
+ }
1173
+ return { granted: grantable, waitMs: 0 };
1174
+ }
1175
+
1176
+ private async handleRatePenalize(req: Request): Promise<Response> {
1177
+ const body = (await req
1178
+ .json()
1179
+ .catch(() => null)) as RatePenalizeRequest | null;
1180
+ if (!body || typeof body.bucketId !== 'string' || !body.bucketId.trim()) {
1181
+ return new Response('bucketId is required', { status: 400 });
1182
+ }
1183
+ const cooldownMs =
1184
+ typeof body.cooldownMs === 'number' && Number.isFinite(body.cooldownMs)
1185
+ ? Math.floor(body.cooldownMs)
1186
+ : 0;
1187
+ if (cooldownMs > 0) {
1188
+ const until = Date.now() + cooldownMs;
1189
+ const existing = this.rateCooldownUntil.get(body.bucketId) ?? 0;
1190
+ this.rateCooldownUntil.set(body.bucketId, Math.max(existing, until));
1191
+ }
1192
+ return new Response('{}', {
1193
+ headers: { 'content-type': 'application/json' },
1194
+ });
1195
+ }
1196
+
1197
+ private getRateWindowState(
1198
+ bucketId: string,
1199
+ rule: RateRule,
1200
+ now: number,
1201
+ ): RateRuleWindowState {
1202
+ const key = RATE_STATE_KEY(bucketId, rule.ruleId);
1203
+ const existing = this.rateWindows.get(key);
1204
+ if (existing) return existing;
1205
+ const created: RateRuleWindowState = {
1206
+ windowStartedAt: now,
1207
+ startedInWindow: 0,
1208
+ };
1209
+ this.rateWindows.set(key, created);
1210
+ return created;
1211
+ }
1212
+
1213
+ private resetExpiredRateWindow(
1214
+ state: RateRuleWindowState,
1215
+ windowMs: number,
1216
+ now: number,
1217
+ ): void {
1218
+ if (windowMs <= 0) {
1219
+ state.windowStartedAt = now;
1220
+ state.startedInWindow = 0;
1221
+ return;
1222
+ }
1223
+ if (now - state.windowStartedAt < windowMs) return;
1224
+ const elapsed = Math.floor((now - state.windowStartedAt) / windowMs);
1225
+ state.windowStartedAt += elapsed * windowMs;
1226
+ state.startedInWindow = 0;
1227
+ if (now - state.windowStartedAt >= windowMs) state.windowStartedAt = now;
1228
+ }
1229
+
1368
1230
  private async storeRunEvent(
1369
1231
  input: CoordinatorRunEventInput,
1370
1232
  ): Promise<CoordinatorRunEvent> {
@@ -1380,12 +1242,9 @@ export class PlayDedup implements DurableObject {
1380
1242
  [COORDINATOR_RUN_EVENT_SEQUENCE_KEY]: seq,
1381
1243
  [this.runEventKey(event)]: event,
1382
1244
  });
1383
- const entries = await this.state.storage.list<CoordinatorRunEvent>({
1384
- prefix: COORDINATOR_RUN_EVENT_KEY_PREFIX,
1385
- });
1386
- const overflow = entries.size - COORDINATOR_RUN_EVENT_MAX_ENTRIES;
1387
- if (overflow > 0) {
1388
- await this.state.storage.delete([...entries.keys()].slice(0, overflow));
1245
+ const pruneSeq = seq - COORDINATOR_RUN_EVENT_MAX_ENTRIES;
1246
+ if (pruneSeq > 0) {
1247
+ await this.state.storage.delete(this.runEventKeyForSeq(pruneSeq));
1389
1248
  }
1390
1249
  });
1391
1250
  if (!event) {
@@ -1482,15 +1341,26 @@ export class PlayDedup implements DurableObject {
1482
1341
  Math.max(Number(url.searchParams.get('timeoutMs') ?? '0'), 0),
1483
1342
  30_000,
1484
1343
  );
1485
- const readEvents = async () => {
1486
- const entries = await this.state.storage.list<CoordinatorRunEvent>({
1487
- prefix: COORDINATOR_RUN_EVENT_KEY_PREFIX,
1488
- });
1489
- return [...entries.values()]
1490
- .filter((event) => event.seq > afterSeq)
1491
- .sort((left, right) => left.seq - right.seq);
1344
+ const readLatestSeq = async () =>
1345
+ (await this.state.storage.get<number>(
1346
+ COORDINATOR_RUN_EVENT_SEQUENCE_KEY,
1347
+ )) ?? afterSeq;
1348
+ const readEvents = async (latestSeq: number) => {
1349
+ const firstAvailableSeq = Math.max(
1350
+ 1,
1351
+ latestSeq - COORDINATOR_RUN_EVENT_MAX_ENTRIES + 1,
1352
+ );
1353
+ const startSeq = Math.max(afterSeq + 1, firstAvailableSeq);
1354
+ if (latestSeq < startSeq) return [];
1355
+ const keys: string[] = [];
1356
+ for (let seq = startSeq; seq <= latestSeq; seq += 1) {
1357
+ keys.push(this.runEventKeyForSeq(seq));
1358
+ }
1359
+ const entries = await this.state.storage.get<CoordinatorRunEvent>(keys);
1360
+ return [...entries.values()].sort((left, right) => left.seq - right.seq);
1492
1361
  };
1493
- let events = await readEvents();
1362
+ let latestSeq = await readLatestSeq();
1363
+ let events = await readEvents(latestSeq);
1494
1364
  if (events.length === 0 && timeoutMs > 0) {
1495
1365
  await new Promise<void>((resolve) => {
1496
1366
  let settled = false;
@@ -1505,12 +1375,9 @@ export class PlayDedup implements DurableObject {
1505
1375
  const timeout = setTimeout(finish, timeoutMs);
1506
1376
  this.runEventWaiters.add(onEvent);
1507
1377
  });
1508
- events = await readEvents();
1378
+ latestSeq = await readLatestSeq();
1379
+ events = await readEvents(latestSeq);
1509
1380
  }
1510
- const latestSeq =
1511
- (await this.state.storage.get<number>(
1512
- COORDINATOR_RUN_EVENT_SEQUENCE_KEY,
1513
- )) ?? afterSeq;
1514
1381
  return new Response(JSON.stringify({ events, latestSeq }), {
1515
1382
  headers: { 'content-type': 'application/json' },
1516
1383
  });