deepline 0.1.79 → 0.1.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/cli/index.js +76 -42
- package/dist/cli/index.mjs +76 -42
- package/dist/index.d.mts +9 -1
- package/dist/index.d.ts +9 -1
- package/dist/index.js +13 -10
- package/dist/index.mjs +13 -10
- package/dist/repo/apps/play-runner-workers/src/child-play-await.ts +192 -0
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +1103 -1617
- package/dist/repo/apps/play-runner-workers/src/dedup-do.ts +506 -654
- package/dist/repo/apps/play-runner-workers/src/entry.ts +1148 -598
- package/dist/repo/apps/play-runner-workers/src/runtime/tool-http-errors.ts +43 -1
- package/dist/repo/apps/play-runner-workers/src/workflow-retry-state.ts +8 -2
- package/dist/repo/sdk/src/client.ts +15 -8
- package/dist/repo/sdk/src/release.ts +2 -2
- package/dist/repo/sdk/src/types.ts +5 -0
- package/dist/repo/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts +231 -0
- package/dist/repo/shared_libs/play-runtime/governor/governor.ts +376 -0
- package/dist/repo/shared_libs/play-runtime/governor/policy.ts +179 -0
- package/dist/repo/shared_libs/play-runtime/governor/rate-state-backend.ts +87 -0
- package/dist/repo/shared_libs/play-runtime/run-failure.ts +12 -0
- package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts +24 -0
- package/dist/repo/shared_libs/play-runtime/submit-limits.ts +35 -0
- package/dist/repo/shared_libs/plays/bundling/index.ts +4 -12
- package/dist/repo/shared_libs/plays/bundling/limits.ts +29 -0
- package/dist/repo/shared_libs/plays/static-pipeline.ts +56 -3
- package/dist/repo/shared_libs/temporal/constants.ts +38 -0
- package/package.json +1 -1
- package/dist/repo/shared_libs/play-runtime/tool-batch-executor.ts +0 -149
|
@@ -85,31 +85,6 @@ type AwaitRequest = {
|
|
|
85
85
|
timeoutMs: number;
|
|
86
86
|
};
|
|
87
87
|
|
|
88
|
-
type WorkflowPoolEntryState = 'warming' | 'ready';
|
|
89
|
-
|
|
90
|
-
type WorkflowPoolEntry = {
|
|
91
|
-
id: string;
|
|
92
|
-
version: string;
|
|
93
|
-
state: WorkflowPoolEntryState;
|
|
94
|
-
createdAt: number;
|
|
95
|
-
readyAt: number | null;
|
|
96
|
-
expiresAt: number;
|
|
97
|
-
};
|
|
98
|
-
|
|
99
|
-
type WorkflowRunMappingState = 'claimed' | 'started' | 'blocked';
|
|
100
|
-
|
|
101
|
-
type WorkflowRunMapping = {
|
|
102
|
-
runId: string;
|
|
103
|
-
instanceId: string;
|
|
104
|
-
state: WorkflowRunMappingState;
|
|
105
|
-
blockedInstanceId?: string | null;
|
|
106
|
-
claimedAt?: number | null;
|
|
107
|
-
startedAt?: number | null;
|
|
108
|
-
version: string;
|
|
109
|
-
createdAt: number;
|
|
110
|
-
expiresAt: number;
|
|
111
|
-
};
|
|
112
|
-
|
|
113
88
|
type WorkflowRunRetryState = {
|
|
114
89
|
runId: string;
|
|
115
90
|
params: unknown;
|
|
@@ -120,6 +95,13 @@ type WorkflowRunRetryState = {
|
|
|
120
95
|
expiresAt: number;
|
|
121
96
|
};
|
|
122
97
|
|
|
98
|
+
type WorkflowInstanceState = {
|
|
99
|
+
runId: string;
|
|
100
|
+
instanceId: string;
|
|
101
|
+
updatedAt: number;
|
|
102
|
+
expiresAt: number;
|
|
103
|
+
};
|
|
104
|
+
|
|
123
105
|
type WorkflowDbSessionsState = {
|
|
124
106
|
runId: string;
|
|
125
107
|
sessions: PreloadedRuntimeDbSession[];
|
|
@@ -150,6 +132,12 @@ type CoordinatorTerminalState = {
|
|
|
150
132
|
completedAt: number;
|
|
151
133
|
};
|
|
152
134
|
|
|
135
|
+
type CoordinatorChildTerminalState = {
|
|
136
|
+
eventKey: string;
|
|
137
|
+
data: unknown;
|
|
138
|
+
storedAt: number;
|
|
139
|
+
};
|
|
140
|
+
|
|
153
141
|
type CoordinatorRunEvent =
|
|
154
142
|
| {
|
|
155
143
|
seq: number;
|
|
@@ -196,31 +184,71 @@ type CoordinatorRunEvent =
|
|
|
196
184
|
type OmitRunEventSequence<T> = T extends unknown ? Omit<T, 'seq'> : never;
|
|
197
185
|
type CoordinatorRunEventInput = OmitRunEventSequence<CoordinatorRunEvent>;
|
|
198
186
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
187
|
+
/**
|
|
188
|
+
* Per-(org,provider) rate-state for the distributed Rate State Backend.
|
|
189
|
+
*
|
|
190
|
+
* One PlayDedup DO instance is addressed per bucket via
|
|
191
|
+
* `idFromName('rate:<orgId>:<provider>')`, so a single instance owns the rate
|
|
192
|
+
* window for that bucket and is single-threaded — which is exactly why the
|
|
193
|
+
* in-process algorithm from `InMemoryRateStateBackend` is correct here. The
|
|
194
|
+
* coordinator RPCs `/rate-acquire` (lease N permits) and `/rate-penalize`
|
|
195
|
+
* (Retry-After cooldown) into it; the esm_workers runtime never blocks per call
|
|
196
|
+
* on a full round-trip because it leases small permit BLOCKS at a time.
|
|
197
|
+
*
|
|
198
|
+
* Rate windows are intentionally kept in DO memory (not durable storage): a
|
|
199
|
+
* window is sub-second-to-seconds ephemeral state, the DO instance outlives any
|
|
200
|
+
* single window, and persisting it would only add storage latency to the hot
|
|
201
|
+
* path. This mirrors the in-memory backend and the Redis sliding-window limiter
|
|
202
|
+
* (`src/lib/redis/customer-rate-limiter.ts`), which also keeps window state in a
|
|
203
|
+
* volatile store with TTL rather than durable rows.
|
|
204
|
+
*/
|
|
205
|
+
type RateRule = {
|
|
206
|
+
ruleId: string;
|
|
207
|
+
requestsPerWindow: number;
|
|
208
|
+
windowMs: number;
|
|
209
|
+
maxConcurrency: number | null;
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
type RateRuleWindowState = {
|
|
213
|
+
windowStartedAt: number;
|
|
214
|
+
startedInWindow: number;
|
|
202
215
|
};
|
|
203
216
|
|
|
204
|
-
type
|
|
205
|
-
|
|
206
|
-
|
|
217
|
+
type RateAcquireRequest = {
|
|
218
|
+
bucketId: string;
|
|
219
|
+
rules: RateRule[];
|
|
220
|
+
/** How many permits the caller wants to lease in this round-trip. */
|
|
221
|
+
requested: number;
|
|
207
222
|
};
|
|
208
223
|
|
|
224
|
+
type RateAcquireResponse = {
|
|
225
|
+
/** Permits actually granted (0..requested). */
|
|
226
|
+
granted: number;
|
|
227
|
+
/** Suggested wait before the next acquire when granted === 0. */
|
|
228
|
+
waitMs: number;
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
type RatePenalizeRequest = {
|
|
232
|
+
bucketId: string;
|
|
233
|
+
cooldownMs: number;
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
const RATE_MIN_WAIT_MS = 10;
|
|
237
|
+
const RATE_STATE_KEY = (bucketId: string, ruleId: string) =>
|
|
238
|
+
`${bucketId}::${ruleId}`;
|
|
239
|
+
|
|
209
240
|
const DEDUP_KEY_PREFIX = 'd:';
|
|
210
|
-
const WORKFLOW_POOL_KEY_PREFIX = 'p:';
|
|
211
|
-
const WORKFLOW_POOL_RUN_KEY_PREFIX = 'm:';
|
|
212
241
|
const WORKFLOW_RUN_RETRY_KEY_PREFIX = 'r:';
|
|
213
242
|
const WORKFLOW_DB_SESSIONS_KEY = 'db-sessions';
|
|
214
243
|
const COORDINATOR_TRACE_KEY_PREFIX = 't:';
|
|
215
244
|
const COORDINATOR_RUN_EVENT_KEY_PREFIX = 'e:';
|
|
216
245
|
const COORDINATOR_TERMINAL_KEY = 'terminal';
|
|
246
|
+
const COORDINATOR_CHILD_TERMINAL_KEY_PREFIX = 'child-terminal:';
|
|
217
247
|
const COORDINATOR_RUN_EVENT_SEQUENCE_KEY = 'event-seq';
|
|
218
248
|
const COORDINATOR_TRACE_MAX_ENTRIES = 200;
|
|
219
249
|
const COORDINATOR_RUN_EVENT_MAX_ENTRIES = 500;
|
|
220
250
|
const FINISH_ALARM_DELAY_MS = 60_000; // self-evict 1 min after finish() called
|
|
221
|
-
const
|
|
222
|
-
const WORKFLOW_POOL_RUN_MAPPING_TTL_MS = 60 * 60 * 1000;
|
|
223
|
-
const WORKFLOW_POOL_READY_MAX_AGE_MS = 7 * 60_000;
|
|
251
|
+
const WORKFLOW_RUN_STATE_TTL_MS = 60 * 60 * 1000;
|
|
224
252
|
const WORKFLOW_RUN_RETRY_STATE_MAX_BYTES = 110_000;
|
|
225
253
|
const WORKFLOW_DB_SESSIONS_TTL_MS = 10 * 60_000;
|
|
226
254
|
|
|
@@ -253,6 +281,13 @@ export class PlayDedup implements DurableObject {
|
|
|
253
281
|
private waiters: Map<string, Set<(value: unknown) => void>> = new Map();
|
|
254
282
|
private runEventWaiters: Set<(value: CoordinatorRunEvent) => void> =
|
|
255
283
|
new Set();
|
|
284
|
+
private childTerminalWaiters: Map<string, Set<() => void>> = new Map();
|
|
285
|
+
// Per-(bucketId, ruleId) request-window state for the rate-state backend.
|
|
286
|
+
// In-memory by design — see the RateRule doc block. Single-threaded DO access
|
|
287
|
+
// makes this the same sliding-window algorithm as InMemoryRateStateBackend.
|
|
288
|
+
private rateWindows: Map<string, RateRuleWindowState> = new Map();
|
|
289
|
+
// Per-bucket Retry-After cooldown floors (penalize()).
|
|
290
|
+
private rateCooldownUntil: Map<string, number> = new Map();
|
|
256
291
|
|
|
257
292
|
constructor(
|
|
258
293
|
private readonly state: DurableObjectState,
|
|
@@ -271,10 +306,35 @@ export class PlayDedup implements DurableObject {
|
|
|
271
306
|
return `${COORDINATOR_RUN_EVENT_KEY_PREFIX}${String(event.seq).padStart(13, '0')}`;
|
|
272
307
|
}
|
|
273
308
|
|
|
309
|
+
private runEventKeyForSeq(seq: number): string {
|
|
310
|
+
return `${COORDINATOR_RUN_EVENT_KEY_PREFIX}${String(seq).padStart(13, '0')}`;
|
|
311
|
+
}
|
|
312
|
+
|
|
274
313
|
private waiterKey(toolId: string, inputHash: string): string {
|
|
275
314
|
return `${toolId}:${inputHash}`;
|
|
276
315
|
}
|
|
277
316
|
|
|
317
|
+
private childTerminalKey(eventKey: string): string {
|
|
318
|
+
return `${COORDINATOR_CHILD_TERMINAL_KEY_PREFIX}${eventKey}`;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
private wakeChildTerminalWaiters(eventKey: string): void {
|
|
322
|
+
const waiters = this.childTerminalWaiters.get(eventKey);
|
|
323
|
+
if (!waiters) return;
|
|
324
|
+
for (const resolve of waiters) {
|
|
325
|
+
resolve();
|
|
326
|
+
}
|
|
327
|
+
this.childTerminalWaiters.delete(eventKey);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
private async readChildTerminalState(
|
|
331
|
+
eventKey: string,
|
|
332
|
+
): Promise<CoordinatorChildTerminalState | undefined> {
|
|
333
|
+
return await this.state.storage.get<CoordinatorChildTerminalState>(
|
|
334
|
+
this.childTerminalKey(eventKey),
|
|
335
|
+
);
|
|
336
|
+
}
|
|
337
|
+
|
|
278
338
|
async fetch(req: Request): Promise<Response> {
|
|
279
339
|
const url = new URL(req.url);
|
|
280
340
|
try {
|
|
@@ -289,36 +349,20 @@ export class PlayDedup implements DurableObject {
|
|
|
289
349
|
return await this.handleFinish(req);
|
|
290
350
|
case '/debug':
|
|
291
351
|
return await this.handleDebug(req);
|
|
292
|
-
case '/pool-add':
|
|
293
|
-
return await this.handlePoolAdd(req);
|
|
294
|
-
case '/pool-claim':
|
|
295
|
-
return await this.handlePoolClaim(req);
|
|
296
|
-
case '/pool-count':
|
|
297
|
-
return await this.handlePoolCount(req);
|
|
298
|
-
case '/pool-list':
|
|
299
|
-
return await this.handlePoolList(req);
|
|
300
|
-
case '/pool-promote':
|
|
301
|
-
return await this.handlePoolPromote(req);
|
|
302
|
-
case '/pool-ready':
|
|
303
|
-
return await this.handlePoolReady(req);
|
|
304
|
-
case '/pool-delete':
|
|
305
|
-
return await this.handlePoolDelete(req);
|
|
306
|
-
case '/pool-map-run':
|
|
307
|
-
return await this.handlePoolMapRun(req);
|
|
308
|
-
case '/pool-block-run':
|
|
309
|
-
return await this.handlePoolBlockRun(req);
|
|
310
|
-
case '/pool-resolve-run':
|
|
311
|
-
return await this.handlePoolResolveRun(req);
|
|
312
352
|
case '/run-retry-state-put':
|
|
313
353
|
return await this.handleRunRetryStatePut(req);
|
|
354
|
+
case '/run-launch-state-put':
|
|
355
|
+
return await this.handleRunLaunchStatePut(req);
|
|
356
|
+
case '/workflow-instance-put':
|
|
357
|
+
return await this.handleWorkflowInstancePut(req);
|
|
358
|
+
case '/workflow-instance-get':
|
|
359
|
+
return await this.handleWorkflowInstanceGet(req);
|
|
314
360
|
case '/run-retry-claim':
|
|
315
361
|
return await this.handleRunRetryClaim(req);
|
|
316
362
|
case '/db-sessions-put':
|
|
317
363
|
return await this.handleDbSessionsPut(req);
|
|
318
364
|
case '/db-sessions-get':
|
|
319
365
|
return await this.handleDbSessionsGet(req);
|
|
320
|
-
case '/pool-clear':
|
|
321
|
-
return await this.handlePoolClear(req);
|
|
322
366
|
case '/trace-add':
|
|
323
367
|
return await this.handleTraceAdd(req);
|
|
324
368
|
case '/trace-list':
|
|
@@ -331,6 +375,16 @@ export class PlayDedup implements DurableObject {
|
|
|
331
375
|
return await this.handleTerminalSet(req);
|
|
332
376
|
case '/terminal-get':
|
|
333
377
|
return await this.handleTerminalGet();
|
|
378
|
+
case '/child-terminal-set':
|
|
379
|
+
return await this.handleChildTerminalSet(req);
|
|
380
|
+
case '/child-terminal-get':
|
|
381
|
+
return await this.handleChildTerminalGet(req);
|
|
382
|
+
case '/child-terminal-await':
|
|
383
|
+
return await this.handleChildTerminalAwait(req);
|
|
384
|
+
case '/rate-acquire':
|
|
385
|
+
return await this.handleRateAcquire(req);
|
|
386
|
+
case '/rate-penalize':
|
|
387
|
+
return await this.handleRatePenalize(req);
|
|
334
388
|
default:
|
|
335
389
|
return new Response('not found', { status: 404 });
|
|
336
390
|
}
|
|
@@ -345,10 +399,11 @@ export class PlayDedup implements DurableObject {
|
|
|
345
399
|
|
|
346
400
|
async alarm(): Promise<void> {
|
|
347
401
|
// Fired after /finish was called. Evict storage and let the DO instance
|
|
348
|
-
// be garbage-collected by
|
|
402
|
+
// be garbage-collected by Cloudflare.
|
|
349
403
|
await this.state.storage.deleteAll();
|
|
350
404
|
this.waiters.clear();
|
|
351
405
|
this.runEventWaiters.clear();
|
|
406
|
+
this.childTerminalWaiters.clear();
|
|
352
407
|
}
|
|
353
408
|
|
|
354
409
|
private async handleLookupOrClaim(req: Request): Promise<Response> {
|
|
@@ -516,544 +571,6 @@ export class PlayDedup implements DurableObject {
|
|
|
516
571
|
);
|
|
517
572
|
}
|
|
518
573
|
|
|
519
|
-
private workflowPoolVersion(req: Request): string {
|
|
520
|
-
return new URL(req.url).searchParams.get('version')?.trim() ?? '';
|
|
521
|
-
}
|
|
522
|
-
|
|
523
|
-
private workflowPoolMinReadyAgeMs(req: Request): number {
|
|
524
|
-
const raw = Number(new URL(req.url).searchParams.get('minReadyAgeMs') ?? 0);
|
|
525
|
-
if (!Number.isFinite(raw) || raw <= 0) {
|
|
526
|
-
return 0;
|
|
527
|
-
}
|
|
528
|
-
return Math.min(Math.floor(raw), WORKFLOW_POOL_DEFAULT_TTL_MS);
|
|
529
|
-
}
|
|
530
|
-
|
|
531
|
-
private isReadyWorkflowPoolEntry(
|
|
532
|
-
value: { key: string; entry: WorkflowPoolEntry | undefined },
|
|
533
|
-
version: string,
|
|
534
|
-
now: number,
|
|
535
|
-
minReadyAgeMs = 0,
|
|
536
|
-
): value is ReadyWorkflowPoolEntryRecord {
|
|
537
|
-
return (
|
|
538
|
-
value.entry !== undefined &&
|
|
539
|
-
value.entry.version === version &&
|
|
540
|
-
value.entry.expiresAt > now &&
|
|
541
|
-
value.entry.state === 'ready' &&
|
|
542
|
-
value.entry.readyAt !== null &&
|
|
543
|
-
now - value.entry.readyAt <= WORKFLOW_POOL_READY_MAX_AGE_MS &&
|
|
544
|
-
now - value.entry.readyAt >= minReadyAgeMs
|
|
545
|
-
);
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
private async gcWorkflowPool(
|
|
549
|
-
now = Date.now(),
|
|
550
|
-
version?: string,
|
|
551
|
-
): Promise<void> {
|
|
552
|
-
const [pool, mappings, retries] = await Promise.all([
|
|
553
|
-
this.state.storage.list<WorkflowPoolEntry>({
|
|
554
|
-
prefix: WORKFLOW_POOL_KEY_PREFIX,
|
|
555
|
-
}),
|
|
556
|
-
this.state.storage.list<WorkflowRunMapping>({
|
|
557
|
-
prefix: WORKFLOW_POOL_RUN_KEY_PREFIX,
|
|
558
|
-
}),
|
|
559
|
-
this.state.storage.list<WorkflowRunRetryState>({
|
|
560
|
-
prefix: WORKFLOW_RUN_RETRY_KEY_PREFIX,
|
|
561
|
-
}),
|
|
562
|
-
]);
|
|
563
|
-
const expiredKeys: string[] = [];
|
|
564
|
-
for (const [key, entry] of pool) {
|
|
565
|
-
if (
|
|
566
|
-
!entry ||
|
|
567
|
-
entry.expiresAt <= now ||
|
|
568
|
-
(entry.state === 'ready' &&
|
|
569
|
-
entry.readyAt !== null &&
|
|
570
|
-
now - entry.readyAt > WORKFLOW_POOL_READY_MAX_AGE_MS) ||
|
|
571
|
-
(version && entry.version !== version)
|
|
572
|
-
) {
|
|
573
|
-
expiredKeys.push(key);
|
|
574
|
-
}
|
|
575
|
-
}
|
|
576
|
-
for (const [key, mapping] of mappings) {
|
|
577
|
-
if (
|
|
578
|
-
!mapping ||
|
|
579
|
-
mapping.expiresAt <= now ||
|
|
580
|
-
(version && mapping.version !== version)
|
|
581
|
-
) {
|
|
582
|
-
expiredKeys.push(key);
|
|
583
|
-
}
|
|
584
|
-
}
|
|
585
|
-
for (const [key, retryState] of retries) {
|
|
586
|
-
if (!retryState || retryState.expiresAt <= now) {
|
|
587
|
-
expiredKeys.push(key);
|
|
588
|
-
}
|
|
589
|
-
}
|
|
590
|
-
if (expiredKeys.length > 0) {
|
|
591
|
-
await this.state.storage.delete(expiredKeys);
|
|
592
|
-
}
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
private async handlePoolAdd(req: Request): Promise<Response> {
|
|
596
|
-
const body = (await req.json().catch(() => null)) as {
|
|
597
|
-
ids?: unknown;
|
|
598
|
-
ttlMs?: unknown;
|
|
599
|
-
version?: unknown;
|
|
600
|
-
readyAt?: unknown;
|
|
601
|
-
ready?: unknown;
|
|
602
|
-
} | null;
|
|
603
|
-
const version =
|
|
604
|
-
typeof body?.version === 'string' ? body.version.trim() : '';
|
|
605
|
-
if (!version) {
|
|
606
|
-
return new Response('version is required', { status: 400 });
|
|
607
|
-
}
|
|
608
|
-
const ids = Array.isArray(body?.ids)
|
|
609
|
-
? body.ids.filter(
|
|
610
|
-
(id): id is string => typeof id === 'string' && id.length > 0,
|
|
611
|
-
)
|
|
612
|
-
: [];
|
|
613
|
-
const now = Date.now();
|
|
614
|
-
const hasReadyAt =
|
|
615
|
-
typeof body?.readyAt === 'number' && Number.isFinite(body.readyAt);
|
|
616
|
-
const readyAt = hasReadyAt ? Math.max(0, body.readyAt as number) : now;
|
|
617
|
-
const ready = body?.ready === true || hasReadyAt;
|
|
618
|
-
const ttlMs =
|
|
619
|
-
typeof body?.ttlMs === 'number' &&
|
|
620
|
-
Number.isFinite(body.ttlMs) &&
|
|
621
|
-
body.ttlMs > 0
|
|
622
|
-
? Math.min(body.ttlMs, WORKFLOW_POOL_DEFAULT_TTL_MS)
|
|
623
|
-
: WORKFLOW_POOL_DEFAULT_TTL_MS;
|
|
624
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
625
|
-
await this.gcWorkflowPool(now, version);
|
|
626
|
-
const writes: Record<string, WorkflowPoolEntry> = {};
|
|
627
|
-
const keys = ids.map((id) => `${WORKFLOW_POOL_KEY_PREFIX}${id}`);
|
|
628
|
-
const existing = (await this.state.storage.get<WorkflowPoolEntry>(
|
|
629
|
-
keys,
|
|
630
|
-
)) as Map<string, WorkflowPoolEntry>;
|
|
631
|
-
for (const id of ids) {
|
|
632
|
-
const key = `${WORKFLOW_POOL_KEY_PREFIX}${id}`;
|
|
633
|
-
const existingEntry = existing.get(key);
|
|
634
|
-
const existingReadyAt =
|
|
635
|
-
existingEntry?.version === version &&
|
|
636
|
-
existingEntry.expiresAt > now &&
|
|
637
|
-
existingEntry.state === 'ready' &&
|
|
638
|
-
existingEntry.readyAt !== null
|
|
639
|
-
? existingEntry.readyAt
|
|
640
|
-
: null;
|
|
641
|
-
const nextReadyAt = ready ? readyAt : existingReadyAt;
|
|
642
|
-
writes[key] = {
|
|
643
|
-
id,
|
|
644
|
-
version,
|
|
645
|
-
state: nextReadyAt !== null ? 'ready' : 'warming',
|
|
646
|
-
createdAt:
|
|
647
|
-
existingEntry?.version === version ? existingEntry.createdAt : now,
|
|
648
|
-
readyAt: nextReadyAt,
|
|
649
|
-
expiresAt: now + ttlMs,
|
|
650
|
-
};
|
|
651
|
-
}
|
|
652
|
-
if (Object.keys(writes).length > 0) {
|
|
653
|
-
await this.state.storage.put(writes);
|
|
654
|
-
}
|
|
655
|
-
});
|
|
656
|
-
return new Response(JSON.stringify({ added: ids.length }), {
|
|
657
|
-
headers: { 'content-type': 'application/json' },
|
|
658
|
-
});
|
|
659
|
-
}
|
|
660
|
-
|
|
661
|
-
private async handlePoolClaim(req: Request): Promise<Response> {
|
|
662
|
-
const version = this.workflowPoolVersion(req);
|
|
663
|
-
if (!version) {
|
|
664
|
-
return new Response('version is required', { status: 400 });
|
|
665
|
-
}
|
|
666
|
-
const body = (await req.json().catch(() => null)) as {
|
|
667
|
-
runId?: unknown;
|
|
668
|
-
} | null;
|
|
669
|
-
const runId = typeof body?.runId === 'string' ? body.runId : '';
|
|
670
|
-
if (!runId) {
|
|
671
|
-
return new Response('runId is required', { status: 400 });
|
|
672
|
-
}
|
|
673
|
-
const minReadyAgeMs = this.workflowPoolMinReadyAgeMs(req);
|
|
674
|
-
let claimedId: string | null = null;
|
|
675
|
-
let counts: WorkflowPoolCounts = { available: 0, warming: 0 };
|
|
676
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
677
|
-
const now = Date.now();
|
|
678
|
-
await this.gcWorkflowPool(now, version);
|
|
679
|
-
const mappingKey = `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`;
|
|
680
|
-
const existingMapping =
|
|
681
|
-
await this.state.storage.get<WorkflowRunMapping>(mappingKey);
|
|
682
|
-
if (existingMapping?.version === version) {
|
|
683
|
-
if (
|
|
684
|
-
existingMapping.state === 'claimed' ||
|
|
685
|
-
existingMapping.state === 'started'
|
|
686
|
-
) {
|
|
687
|
-
claimedId = existingMapping.instanceId || null;
|
|
688
|
-
}
|
|
689
|
-
return;
|
|
690
|
-
}
|
|
691
|
-
const entries = await this.state.storage.list<WorkflowPoolEntry>({
|
|
692
|
-
prefix: WORKFLOW_POOL_KEY_PREFIX,
|
|
693
|
-
});
|
|
694
|
-
counts = this.countWorkflowPoolEntries(
|
|
695
|
-
entries,
|
|
696
|
-
version,
|
|
697
|
-
now,
|
|
698
|
-
minReadyAgeMs,
|
|
699
|
-
);
|
|
700
|
-
const sorted = [...entries.entries()]
|
|
701
|
-
.map(([key, entry]) => ({ key, entry }))
|
|
702
|
-
.filter((entry) =>
|
|
703
|
-
this.isReadyWorkflowPoolEntry(entry, version, now, minReadyAgeMs),
|
|
704
|
-
)
|
|
705
|
-
.sort((a, b) => b.entry.readyAt - a.entry.readyAt);
|
|
706
|
-
const selected = sorted[0];
|
|
707
|
-
if (!selected) return;
|
|
708
|
-
claimedId = selected.entry.id;
|
|
709
|
-
await this.state.storage.delete(selected.key);
|
|
710
|
-
await this.state.storage.put(mappingKey, {
|
|
711
|
-
runId,
|
|
712
|
-
instanceId: claimedId,
|
|
713
|
-
state: 'claimed',
|
|
714
|
-
blockedInstanceId: null,
|
|
715
|
-
claimedAt: now,
|
|
716
|
-
startedAt: null,
|
|
717
|
-
version,
|
|
718
|
-
createdAt: now,
|
|
719
|
-
expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
|
|
720
|
-
} satisfies WorkflowRunMapping);
|
|
721
|
-
counts = {
|
|
722
|
-
available: Math.max(0, counts.available - 1),
|
|
723
|
-
warming: counts.warming,
|
|
724
|
-
};
|
|
725
|
-
});
|
|
726
|
-
return new Response(JSON.stringify({ id: claimedId, ...counts }), {
|
|
727
|
-
headers: { 'content-type': 'application/json' },
|
|
728
|
-
});
|
|
729
|
-
}
|
|
730
|
-
|
|
731
|
-
private async handlePoolCount(req: Request): Promise<Response> {
|
|
732
|
-
const version = this.workflowPoolVersion(req);
|
|
733
|
-
if (!version) {
|
|
734
|
-
return new Response('version is required', { status: 400 });
|
|
735
|
-
}
|
|
736
|
-
const minReadyAgeMs = this.workflowPoolMinReadyAgeMs(req);
|
|
737
|
-
const now = Date.now();
|
|
738
|
-
await this.gcWorkflowPool(now, version);
|
|
739
|
-
const entries = await this.state.storage.list<WorkflowPoolEntry>({
|
|
740
|
-
prefix: WORKFLOW_POOL_KEY_PREFIX,
|
|
741
|
-
});
|
|
742
|
-
const counts = this.countWorkflowPoolEntries(
|
|
743
|
-
entries,
|
|
744
|
-
version,
|
|
745
|
-
now,
|
|
746
|
-
minReadyAgeMs,
|
|
747
|
-
);
|
|
748
|
-
return new Response(JSON.stringify(counts), {
|
|
749
|
-
headers: { 'content-type': 'application/json' },
|
|
750
|
-
});
|
|
751
|
-
}
|
|
752
|
-
|
|
753
|
-
private countWorkflowPoolEntries(
|
|
754
|
-
entries: Map<string, WorkflowPoolEntry>,
|
|
755
|
-
version: string,
|
|
756
|
-
now: number,
|
|
757
|
-
minReadyAgeMs: number,
|
|
758
|
-
): WorkflowPoolCounts {
|
|
759
|
-
let available = 0;
|
|
760
|
-
let warming = 0;
|
|
761
|
-
for (const entry of entries.values()) {
|
|
762
|
-
if (entry.version !== version) continue;
|
|
763
|
-
if (
|
|
764
|
-
entry.state !== 'ready' ||
|
|
765
|
-
entry.readyAt === null ||
|
|
766
|
-
now - entry.readyAt > WORKFLOW_POOL_READY_MAX_AGE_MS ||
|
|
767
|
-
now - entry.readyAt < minReadyAgeMs
|
|
768
|
-
) {
|
|
769
|
-
warming += 1;
|
|
770
|
-
} else {
|
|
771
|
-
available += 1;
|
|
772
|
-
}
|
|
773
|
-
}
|
|
774
|
-
return { available, warming };
|
|
775
|
-
}
|
|
776
|
-
|
|
777
|
-
private async handlePoolList(req: Request): Promise<Response> {
|
|
778
|
-
const version = this.workflowPoolVersion(req);
|
|
779
|
-
if (!version) {
|
|
780
|
-
return new Response('version is required', { status: 400 });
|
|
781
|
-
}
|
|
782
|
-
await this.gcWorkflowPool(Date.now(), version);
|
|
783
|
-
const entries = await this.state.storage.list<WorkflowPoolEntry>({
|
|
784
|
-
prefix: WORKFLOW_POOL_KEY_PREFIX,
|
|
785
|
-
});
|
|
786
|
-
return new Response(
|
|
787
|
-
JSON.stringify({
|
|
788
|
-
entries: [...entries.values()]
|
|
789
|
-
.filter((entry) => entry.version === version)
|
|
790
|
-
.map((entry) => ({
|
|
791
|
-
id: entry.id,
|
|
792
|
-
state: entry.state,
|
|
793
|
-
createdAt: entry.createdAt,
|
|
794
|
-
readyAt: entry.readyAt,
|
|
795
|
-
expiresAt: entry.expiresAt,
|
|
796
|
-
})),
|
|
797
|
-
}),
|
|
798
|
-
{ headers: { 'content-type': 'application/json' } },
|
|
799
|
-
);
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
private async handlePoolPromote(req: Request): Promise<Response> {
|
|
803
|
-
const body = (await req.json().catch(() => null)) as {
|
|
804
|
-
ids?: unknown;
|
|
805
|
-
version?: unknown;
|
|
806
|
-
} | null;
|
|
807
|
-
const version =
|
|
808
|
-
typeof body?.version === 'string' ? body.version.trim() : '';
|
|
809
|
-
if (!version) {
|
|
810
|
-
return new Response('version is required', { status: 400 });
|
|
811
|
-
}
|
|
812
|
-
const ids = Array.isArray(body?.ids)
|
|
813
|
-
? body.ids.filter(
|
|
814
|
-
(id): id is string => typeof id === 'string' && id.length > 0,
|
|
815
|
-
)
|
|
816
|
-
: [];
|
|
817
|
-
const now = Date.now();
|
|
818
|
-
const writes: Record<string, WorkflowPoolEntry> = {};
|
|
819
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
820
|
-
await this.gcWorkflowPool(now, version);
|
|
821
|
-
for (const id of ids) {
|
|
822
|
-
const key = `${WORKFLOW_POOL_KEY_PREFIX}${id}`;
|
|
823
|
-
const entry = await this.state.storage.get<WorkflowPoolEntry>(key);
|
|
824
|
-
if (!entry || entry.version !== version || entry.expiresAt <= now) {
|
|
825
|
-
continue;
|
|
826
|
-
}
|
|
827
|
-
writes[key] = {
|
|
828
|
-
...entry,
|
|
829
|
-
state: 'ready',
|
|
830
|
-
readyAt: now,
|
|
831
|
-
};
|
|
832
|
-
}
|
|
833
|
-
if (Object.keys(writes).length > 0) {
|
|
834
|
-
await this.state.storage.put(writes);
|
|
835
|
-
}
|
|
836
|
-
});
|
|
837
|
-
return new Response(
|
|
838
|
-
JSON.stringify({ promoted: Object.keys(writes).length }),
|
|
839
|
-
{
|
|
840
|
-
headers: { 'content-type': 'application/json' },
|
|
841
|
-
},
|
|
842
|
-
);
|
|
843
|
-
}
|
|
844
|
-
|
|
845
|
-
private async handlePoolReady(req: Request): Promise<Response> {
|
|
846
|
-
const body = (await req.json().catch(() => null)) as {
|
|
847
|
-
poolId?: unknown;
|
|
848
|
-
version?: unknown;
|
|
849
|
-
} | null;
|
|
850
|
-
const poolId = typeof body?.poolId === 'string' ? body.poolId : '';
|
|
851
|
-
const version =
|
|
852
|
-
typeof body?.version === 'string' ? body.version.trim() : '';
|
|
853
|
-
if (!poolId || !version) {
|
|
854
|
-
return new Response('poolId and version are required', { status: 400 });
|
|
855
|
-
}
|
|
856
|
-
const now = Date.now();
|
|
857
|
-
let ready = false;
|
|
858
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
859
|
-
await this.gcWorkflowPool(now, version);
|
|
860
|
-
const key = `${WORKFLOW_POOL_KEY_PREFIX}${poolId}`;
|
|
861
|
-
const entry = await this.state.storage.get<WorkflowPoolEntry>(key);
|
|
862
|
-
if (!entry || entry.version !== version || entry.expiresAt <= now) {
|
|
863
|
-
return;
|
|
864
|
-
}
|
|
865
|
-
await this.state.storage.put(key, {
|
|
866
|
-
...entry,
|
|
867
|
-
state: 'ready',
|
|
868
|
-
readyAt: entry.readyAt ?? now,
|
|
869
|
-
});
|
|
870
|
-
ready = true;
|
|
871
|
-
});
|
|
872
|
-
return new Response(JSON.stringify({ ready }), {
|
|
873
|
-
headers: { 'content-type': 'application/json' },
|
|
874
|
-
});
|
|
875
|
-
}
|
|
876
|
-
|
|
877
|
-
private async handlePoolDelete(req: Request): Promise<Response> {
|
|
878
|
-
const body = (await req.json().catch(() => null)) as {
|
|
879
|
-
ids?: unknown;
|
|
880
|
-
version?: unknown;
|
|
881
|
-
} | null;
|
|
882
|
-
const version =
|
|
883
|
-
typeof body?.version === 'string' ? body.version.trim() : '';
|
|
884
|
-
if (!version) {
|
|
885
|
-
return new Response('version is required', { status: 400 });
|
|
886
|
-
}
|
|
887
|
-
const ids = Array.isArray(body?.ids)
|
|
888
|
-
? body.ids.filter(
|
|
889
|
-
(id): id is string => typeof id === 'string' && id.length > 0,
|
|
890
|
-
)
|
|
891
|
-
: [];
|
|
892
|
-
const keys: string[] = [];
|
|
893
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
894
|
-
await this.gcWorkflowPool(Date.now(), version);
|
|
895
|
-
for (const id of ids) {
|
|
896
|
-
const key = `${WORKFLOW_POOL_KEY_PREFIX}${id}`;
|
|
897
|
-
const entry = await this.state.storage.get<WorkflowPoolEntry>(key);
|
|
898
|
-
if (entry?.version === version) keys.push(key);
|
|
899
|
-
}
|
|
900
|
-
if (keys.length > 0) {
|
|
901
|
-
await this.state.storage.delete(keys);
|
|
902
|
-
}
|
|
903
|
-
});
|
|
904
|
-
return new Response(JSON.stringify({ deleted: keys.length }), {
|
|
905
|
-
headers: { 'content-type': 'application/json' },
|
|
906
|
-
});
|
|
907
|
-
}
|
|
908
|
-
|
|
909
|
-
private async handlePoolMapRun(req: Request): Promise<Response> {
|
|
910
|
-
const body = (await req.json().catch(() => null)) as {
|
|
911
|
-
runId?: unknown;
|
|
912
|
-
instanceId?: unknown;
|
|
913
|
-
version?: unknown;
|
|
914
|
-
started?: unknown;
|
|
915
|
-
} | null;
|
|
916
|
-
const runId = typeof body?.runId === 'string' ? body.runId : '';
|
|
917
|
-
const instanceId =
|
|
918
|
-
typeof body?.instanceId === 'string' ? body.instanceId : '';
|
|
919
|
-
const version =
|
|
920
|
-
typeof body?.version === 'string' ? body.version.trim() : '';
|
|
921
|
-
const started = body?.started === true;
|
|
922
|
-
if (!runId || !instanceId || !version) {
|
|
923
|
-
return new Response('runId, instanceId, and version are required', {
|
|
924
|
-
status: 400,
|
|
925
|
-
});
|
|
926
|
-
}
|
|
927
|
-
const now = Date.now();
|
|
928
|
-
let mapped = true;
|
|
929
|
-
const key = `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`;
|
|
930
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
931
|
-
const existing = await this.state.storage.get<WorkflowRunMapping>(key);
|
|
932
|
-
if (
|
|
933
|
-
existing?.version === version &&
|
|
934
|
-
existing.state === 'blocked' &&
|
|
935
|
-
existing.blockedInstanceId === instanceId
|
|
936
|
-
) {
|
|
937
|
-
mapped = false;
|
|
938
|
-
return;
|
|
939
|
-
}
|
|
940
|
-
const alreadyStarted =
|
|
941
|
-
existing?.version === version &&
|
|
942
|
-
existing.instanceId === instanceId &&
|
|
943
|
-
existing.state === 'started' &&
|
|
944
|
-
typeof existing.startedAt === 'number';
|
|
945
|
-
const startedAt = started || alreadyStarted ? now : null;
|
|
946
|
-
await this.state.storage.put(key, {
|
|
947
|
-
runId,
|
|
948
|
-
instanceId,
|
|
949
|
-
state: startedAt !== null ? 'started' : 'claimed',
|
|
950
|
-
blockedInstanceId: null,
|
|
951
|
-
claimedAt:
|
|
952
|
-
existing?.version === version &&
|
|
953
|
-
typeof existing.claimedAt === 'number'
|
|
954
|
-
? existing.claimedAt
|
|
955
|
-
: now,
|
|
956
|
-
startedAt,
|
|
957
|
-
version,
|
|
958
|
-
createdAt:
|
|
959
|
-
existing?.version === version && existing.createdAt
|
|
960
|
-
? existing.createdAt
|
|
961
|
-
: now,
|
|
962
|
-
expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
|
|
963
|
-
} satisfies WorkflowRunMapping);
|
|
964
|
-
});
|
|
965
|
-
return new Response(JSON.stringify({ mapped }), {
|
|
966
|
-
headers: { 'content-type': 'application/json' },
|
|
967
|
-
});
|
|
968
|
-
}
|
|
969
|
-
|
|
970
|
-
private async handlePoolBlockRun(req: Request): Promise<Response> {
|
|
971
|
-
const body = (await req.json().catch(() => null)) as {
|
|
972
|
-
runId?: unknown;
|
|
973
|
-
instanceId?: unknown;
|
|
974
|
-
version?: unknown;
|
|
975
|
-
} | null;
|
|
976
|
-
const runId = typeof body?.runId === 'string' ? body.runId : '';
|
|
977
|
-
const instanceId =
|
|
978
|
-
typeof body?.instanceId === 'string' ? body.instanceId : '';
|
|
979
|
-
const version =
|
|
980
|
-
typeof body?.version === 'string' ? body.version.trim() : '';
|
|
981
|
-
if (!runId || !instanceId || !version) {
|
|
982
|
-
return new Response('runId, instanceId, and version are required', {
|
|
983
|
-
status: 400,
|
|
984
|
-
});
|
|
985
|
-
}
|
|
986
|
-
const now = Date.now();
|
|
987
|
-
const key = `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`;
|
|
988
|
-
let started = false;
|
|
989
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
990
|
-
const existing = await this.state.storage.get<WorkflowRunMapping>(key);
|
|
991
|
-
if (
|
|
992
|
-
existing?.version === version &&
|
|
993
|
-
existing.instanceId === instanceId &&
|
|
994
|
-
existing.state === 'started' &&
|
|
995
|
-
typeof existing.startedAt === 'number'
|
|
996
|
-
) {
|
|
997
|
-
started = true;
|
|
998
|
-
return;
|
|
999
|
-
}
|
|
1000
|
-
await this.state.storage.put(key, {
|
|
1001
|
-
runId,
|
|
1002
|
-
instanceId: '',
|
|
1003
|
-
state: 'blocked',
|
|
1004
|
-
blockedInstanceId: instanceId,
|
|
1005
|
-
claimedAt:
|
|
1006
|
-
existing?.version === version &&
|
|
1007
|
-
typeof existing.claimedAt === 'number'
|
|
1008
|
-
? existing.claimedAt
|
|
1009
|
-
: now,
|
|
1010
|
-
startedAt: null,
|
|
1011
|
-
version,
|
|
1012
|
-
createdAt: now,
|
|
1013
|
-
expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
|
|
1014
|
-
} satisfies WorkflowRunMapping);
|
|
1015
|
-
});
|
|
1016
|
-
if (started) {
|
|
1017
|
-
return new Response(JSON.stringify({ blocked: false, started: true }), {
|
|
1018
|
-
headers: { 'content-type': 'application/json' },
|
|
1019
|
-
});
|
|
1020
|
-
}
|
|
1021
|
-
return new Response(JSON.stringify({ blocked: true, started: false }), {
|
|
1022
|
-
headers: { 'content-type': 'application/json' },
|
|
1023
|
-
});
|
|
1024
|
-
}
|
|
1025
|
-
|
|
1026
|
-
private async handlePoolResolveRun(req: Request): Promise<Response> {
|
|
1027
|
-
const url = new URL(req.url);
|
|
1028
|
-
const runId = url.searchParams.get('runId') ?? '';
|
|
1029
|
-
const version = url.searchParams.get('version')?.trim() ?? '';
|
|
1030
|
-
if (!runId) {
|
|
1031
|
-
return new Response('runId is required', { status: 400 });
|
|
1032
|
-
}
|
|
1033
|
-
if (!version) {
|
|
1034
|
-
return new Response('version is required', { status: 400 });
|
|
1035
|
-
}
|
|
1036
|
-
await this.gcWorkflowPool(Date.now(), version);
|
|
1037
|
-
const mapping = await this.state.storage.get<WorkflowRunMapping>(
|
|
1038
|
-
`${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`,
|
|
1039
|
-
);
|
|
1040
|
-
if (mapping && mapping.version !== version) {
|
|
1041
|
-
await this.state.storage.delete(
|
|
1042
|
-
`${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`,
|
|
1043
|
-
);
|
|
1044
|
-
return new Response(JSON.stringify({ instanceId: null }), {
|
|
1045
|
-
headers: { 'content-type': 'application/json' },
|
|
1046
|
-
});
|
|
1047
|
-
}
|
|
1048
|
-
return new Response(
|
|
1049
|
-
JSON.stringify({
|
|
1050
|
-
instanceId: mapping?.instanceId || null,
|
|
1051
|
-
startedAt: mapping?.startedAt ?? null,
|
|
1052
|
-
}),
|
|
1053
|
-
{ headers: { 'content-type': 'application/json' } },
|
|
1054
|
-
);
|
|
1055
|
-
}
|
|
1056
|
-
|
|
1057
574
|
private async handleRunRetryStatePut(req: Request): Promise<Response> {
|
|
1058
575
|
const body = (await req.json().catch(() => null)) as {
|
|
1059
576
|
runId?: unknown;
|
|
@@ -1069,11 +586,8 @@ export class PlayDedup implements DurableObject {
|
|
|
1069
586
|
const now = Date.now();
|
|
1070
587
|
const ttlMs =
|
|
1071
588
|
typeof body.ttlMs === 'number' && Number.isFinite(body.ttlMs)
|
|
1072
|
-
? Math.max(
|
|
1073
|
-
|
|
1074
|
-
Math.min(body.ttlMs, WORKFLOW_POOL_RUN_MAPPING_TTL_MS),
|
|
1075
|
-
)
|
|
1076
|
-
: WORKFLOW_POOL_RUN_MAPPING_TTL_MS;
|
|
589
|
+
? Math.max(60_000, Math.min(body.ttlMs, WORKFLOW_RUN_STATE_TTL_MS))
|
|
590
|
+
: WORKFLOW_RUN_STATE_TTL_MS;
|
|
1077
591
|
const key = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
|
|
1078
592
|
await this.state.blockConcurrencyWhile(async () => {
|
|
1079
593
|
const existing = await this.state.storage.get<WorkflowRunRetryState>(key);
|
|
@@ -1082,8 +596,7 @@ export class PlayDedup implements DurableObject {
|
|
|
1082
596
|
params: 'params' in body ? body.params : null,
|
|
1083
597
|
paramsRef: 'paramsRef' in body ? body.paramsRef : null,
|
|
1084
598
|
paramsBytes:
|
|
1085
|
-
typeof (body as { paramsBytes?: unknown }).paramsBytes ===
|
|
1086
|
-
'number' &&
|
|
599
|
+
typeof (body as { paramsBytes?: unknown }).paramsBytes === 'number' &&
|
|
1087
600
|
Number.isFinite((body as { paramsBytes?: number }).paramsBytes)
|
|
1088
601
|
? (body as { paramsBytes: number }).paramsBytes
|
|
1089
602
|
: undefined,
|
|
@@ -1108,6 +621,97 @@ export class PlayDedup implements DurableObject {
|
|
|
1108
621
|
});
|
|
1109
622
|
}
|
|
1110
623
|
|
|
624
|
+
private async handleRunLaunchStatePut(req: Request): Promise<Response> {
|
|
625
|
+
const body = (await req.json().catch(() => null)) as {
|
|
626
|
+
runId?: unknown;
|
|
627
|
+
params?: unknown;
|
|
628
|
+
paramsRef?: unknown;
|
|
629
|
+
paramsBytes?: unknown;
|
|
630
|
+
sessions?: unknown;
|
|
631
|
+
retryTtlMs?: unknown;
|
|
632
|
+
dbSessionsTtlMs?: unknown;
|
|
633
|
+
} | null;
|
|
634
|
+
const runId = typeof body?.runId === 'string' ? body.runId : '';
|
|
635
|
+
const sessions = Array.isArray(body?.sessions)
|
|
636
|
+
? (body.sessions as PreloadedRuntimeDbSession[])
|
|
637
|
+
: null;
|
|
638
|
+
if (
|
|
639
|
+
!runId ||
|
|
640
|
+
!body ||
|
|
641
|
+
(!('params' in body) && !('paramsRef' in body)) ||
|
|
642
|
+
!sessions
|
|
643
|
+
) {
|
|
644
|
+
return new Response(
|
|
645
|
+
'runId, params or paramsRef, and sessions are required',
|
|
646
|
+
{
|
|
647
|
+
status: 400,
|
|
648
|
+
},
|
|
649
|
+
);
|
|
650
|
+
}
|
|
651
|
+
assertEncryptedDbSessionsForStorage(sessions);
|
|
652
|
+
const now = Date.now();
|
|
653
|
+
const retryTtlMs =
|
|
654
|
+
typeof body.retryTtlMs === 'number' && Number.isFinite(body.retryTtlMs)
|
|
655
|
+
? Math.max(60_000, Math.min(body.retryTtlMs, WORKFLOW_RUN_STATE_TTL_MS))
|
|
656
|
+
: WORKFLOW_RUN_STATE_TTL_MS;
|
|
657
|
+
const dbSessionsTtlMs =
|
|
658
|
+
typeof body.dbSessionsTtlMs === 'number' &&
|
|
659
|
+
Number.isFinite(body.dbSessionsTtlMs) &&
|
|
660
|
+
body.dbSessionsTtlMs > 0
|
|
661
|
+
? Math.min(
|
|
662
|
+
Math.max(Math.floor(body.dbSessionsTtlMs), 60_000),
|
|
663
|
+
30 * 60_000,
|
|
664
|
+
)
|
|
665
|
+
: WORKFLOW_DB_SESSIONS_TTL_MS;
|
|
666
|
+
const retryKey = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
|
|
667
|
+
const dbSessionsState: WorkflowDbSessionsState = {
|
|
668
|
+
runId,
|
|
669
|
+
sessions,
|
|
670
|
+
storedAt: now,
|
|
671
|
+
expiresAt: now + dbSessionsTtlMs,
|
|
672
|
+
};
|
|
673
|
+
let retryStateExpiresAt = now + retryTtlMs;
|
|
674
|
+
await this.state.blockConcurrencyWhile(async () => {
|
|
675
|
+
const existing =
|
|
676
|
+
await this.state.storage.get<WorkflowRunRetryState>(retryKey);
|
|
677
|
+
const retryState = {
|
|
678
|
+
runId,
|
|
679
|
+
params: 'params' in body ? body.params : null,
|
|
680
|
+
paramsRef: 'paramsRef' in body ? body.paramsRef : null,
|
|
681
|
+
paramsBytes:
|
|
682
|
+
typeof body.paramsBytes === 'number' &&
|
|
683
|
+
Number.isFinite(body.paramsBytes)
|
|
684
|
+
? body.paramsBytes
|
|
685
|
+
: undefined,
|
|
686
|
+
retryAttempts:
|
|
687
|
+
existing?.runId === runId &&
|
|
688
|
+
typeof existing.retryAttempts === 'number'
|
|
689
|
+
? existing.retryAttempts
|
|
690
|
+
: 0,
|
|
691
|
+
updatedAt: now,
|
|
692
|
+
expiresAt: retryStateExpiresAt,
|
|
693
|
+
} satisfies WorkflowRunRetryState;
|
|
694
|
+
const bytes = jsonByteLength(retryState);
|
|
695
|
+
if (bytes > WORKFLOW_RUN_RETRY_STATE_MAX_BYTES) {
|
|
696
|
+
throw new Error(
|
|
697
|
+
`workflow retry state too large: ${bytes} bytes exceeds ${WORKFLOW_RUN_RETRY_STATE_MAX_BYTES}`,
|
|
698
|
+
);
|
|
699
|
+
}
|
|
700
|
+
retryStateExpiresAt = retryState.expiresAt;
|
|
701
|
+
await this.state.storage.put(retryKey, retryState);
|
|
702
|
+
await this.state.storage.put(WORKFLOW_DB_SESSIONS_KEY, dbSessionsState);
|
|
703
|
+
});
|
|
704
|
+
return new Response(
|
|
705
|
+
JSON.stringify({
|
|
706
|
+
ok: true,
|
|
707
|
+
sessionCount: sessions.length,
|
|
708
|
+
retryExpiresAt: retryStateExpiresAt,
|
|
709
|
+
dbSessionsExpiresAt: dbSessionsState.expiresAt,
|
|
710
|
+
}),
|
|
711
|
+
{ headers: { 'content-type': 'application/json' } },
|
|
712
|
+
);
|
|
713
|
+
}
|
|
714
|
+
|
|
1111
715
|
private async handleRunRetryClaim(req: Request): Promise<Response> {
|
|
1112
716
|
const body = (await req.json().catch(() => null)) as {
|
|
1113
717
|
runId?: unknown;
|
|
@@ -1149,7 +753,7 @@ export class PlayDedup implements DurableObject {
|
|
|
1149
753
|
...existing,
|
|
1150
754
|
retryAttempts: nextAttempts,
|
|
1151
755
|
updatedAt: now,
|
|
1152
|
-
expiresAt: now +
|
|
756
|
+
expiresAt: now + WORKFLOW_RUN_STATE_TTL_MS,
|
|
1153
757
|
} satisfies WorkflowRunRetryState);
|
|
1154
758
|
response = {
|
|
1155
759
|
claimed: true,
|
|
@@ -1164,6 +768,59 @@ export class PlayDedup implements DurableObject {
|
|
|
1164
768
|
});
|
|
1165
769
|
}
|
|
1166
770
|
|
|
771
|
+
private async handleWorkflowInstancePut(req: Request): Promise<Response> {
|
|
772
|
+
const body = (await req.json().catch(() => null)) as {
|
|
773
|
+
runId?: unknown;
|
|
774
|
+
instanceId?: unknown;
|
|
775
|
+
ttlMs?: unknown;
|
|
776
|
+
} | null;
|
|
777
|
+
const runId = typeof body?.runId === 'string' ? body.runId : '';
|
|
778
|
+
const instanceId =
|
|
779
|
+
typeof body?.instanceId === 'string' ? body.instanceId : '';
|
|
780
|
+
if (!runId || !instanceId) {
|
|
781
|
+
return new Response('runId and instanceId are required', { status: 400 });
|
|
782
|
+
}
|
|
783
|
+
const now = Date.now();
|
|
784
|
+
const ttlMs =
|
|
785
|
+
typeof body?.ttlMs === 'number' &&
|
|
786
|
+
Number.isFinite(body.ttlMs) &&
|
|
787
|
+
body.ttlMs > 0
|
|
788
|
+
? Math.max(60_000, Math.min(body.ttlMs, WORKFLOW_RUN_STATE_TTL_MS))
|
|
789
|
+
: WORKFLOW_RUN_STATE_TTL_MS;
|
|
790
|
+
await this.state.storage.put('workflow-instance', {
|
|
791
|
+
runId,
|
|
792
|
+
instanceId,
|
|
793
|
+
updatedAt: now,
|
|
794
|
+
expiresAt: now + ttlMs,
|
|
795
|
+
} satisfies WorkflowInstanceState);
|
|
796
|
+
return new Response(JSON.stringify({ ok: true }), {
|
|
797
|
+
headers: { 'content-type': 'application/json' },
|
|
798
|
+
});
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
private async handleWorkflowInstanceGet(req: Request): Promise<Response> {
|
|
802
|
+
const runId = new URL(req.url).searchParams.get('runId') ?? '';
|
|
803
|
+
if (!runId) {
|
|
804
|
+
return new Response('runId is required', { status: 400 });
|
|
805
|
+
}
|
|
806
|
+
const state =
|
|
807
|
+
await this.state.storage.get<WorkflowInstanceState>('workflow-instance');
|
|
808
|
+
if (!state || state.runId !== runId) {
|
|
809
|
+
return new Response(JSON.stringify({ instanceId: null }), {
|
|
810
|
+
headers: { 'content-type': 'application/json' },
|
|
811
|
+
});
|
|
812
|
+
}
|
|
813
|
+
if (state.expiresAt <= Date.now()) {
|
|
814
|
+
await this.state.storage.delete('workflow-instance');
|
|
815
|
+
return new Response(JSON.stringify({ instanceId: null }), {
|
|
816
|
+
headers: { 'content-type': 'application/json' },
|
|
817
|
+
});
|
|
818
|
+
}
|
|
819
|
+
return new Response(JSON.stringify({ instanceId: state.instanceId }), {
|
|
820
|
+
headers: { 'content-type': 'application/json' },
|
|
821
|
+
});
|
|
822
|
+
}
|
|
823
|
+
|
|
1167
824
|
private async handleDbSessionsPut(req: Request): Promise<Response> {
|
|
1168
825
|
const body = (await req.json().catch(() => null)) as {
|
|
1169
826
|
runId?: unknown;
|
|
@@ -1235,36 +892,6 @@ export class PlayDedup implements DurableObject {
|
|
|
1235
892
|
);
|
|
1236
893
|
}
|
|
1237
894
|
|
|
1238
|
-
private async handlePoolClear(req: Request): Promise<Response> {
|
|
1239
|
-
const version = this.workflowPoolVersion(req);
|
|
1240
|
-
const [pool, mappings, retries] = await Promise.all([
|
|
1241
|
-
this.state.storage.list<WorkflowPoolEntry>({
|
|
1242
|
-
prefix: WORKFLOW_POOL_KEY_PREFIX,
|
|
1243
|
-
}),
|
|
1244
|
-
this.state.storage.list<WorkflowRunMapping>({
|
|
1245
|
-
prefix: WORKFLOW_POOL_RUN_KEY_PREFIX,
|
|
1246
|
-
}),
|
|
1247
|
-
this.state.storage.list<WorkflowRunRetryState>({
|
|
1248
|
-
prefix: WORKFLOW_RUN_RETRY_KEY_PREFIX,
|
|
1249
|
-
}),
|
|
1250
|
-
]);
|
|
1251
|
-
const keys = [
|
|
1252
|
-
...[...pool.entries()]
|
|
1253
|
-
.filter(([, entry]) => !version || entry.version === version)
|
|
1254
|
-
.map(([key]) => key),
|
|
1255
|
-
...[...mappings.entries()]
|
|
1256
|
-
.filter(([, entry]) => !version || entry.version === version)
|
|
1257
|
-
.map(([key]) => key),
|
|
1258
|
-
...[...retries.keys()],
|
|
1259
|
-
];
|
|
1260
|
-
if (keys.length > 0) {
|
|
1261
|
-
await this.state.storage.delete(keys);
|
|
1262
|
-
}
|
|
1263
|
-
return new Response(JSON.stringify({ deleted: keys.length }), {
|
|
1264
|
-
headers: { 'content-type': 'application/json' },
|
|
1265
|
-
});
|
|
1266
|
-
}
|
|
1267
|
-
|
|
1268
895
|
private async handleTraceAdd(req: Request): Promise<Response> {
|
|
1269
896
|
const body = (await req
|
|
1270
897
|
.json()
|
|
@@ -1380,6 +1007,226 @@ export class PlayDedup implements DurableObject {
|
|
|
1380
1007
|
});
|
|
1381
1008
|
}
|
|
1382
1009
|
|
|
1010
|
+
private async handleChildTerminalSet(req: Request): Promise<Response> {
|
|
1011
|
+
const body = (await req.json().catch(() => null)) as {
|
|
1012
|
+
eventKey?: unknown;
|
|
1013
|
+
data?: unknown;
|
|
1014
|
+
storedAt?: unknown;
|
|
1015
|
+
} | null;
|
|
1016
|
+
const eventKey =
|
|
1017
|
+
typeof body?.eventKey === 'string' && body.eventKey.trim()
|
|
1018
|
+
? body.eventKey.trim()
|
|
1019
|
+
: '';
|
|
1020
|
+
if (!eventKey) {
|
|
1021
|
+
return new Response('eventKey is required', { status: 400 });
|
|
1022
|
+
}
|
|
1023
|
+
const state: CoordinatorChildTerminalState = {
|
|
1024
|
+
eventKey,
|
|
1025
|
+
data: body?.data ?? null,
|
|
1026
|
+
storedAt:
|
|
1027
|
+
typeof body?.storedAt === 'number' && Number.isFinite(body.storedAt)
|
|
1028
|
+
? body.storedAt
|
|
1029
|
+
: Date.now(),
|
|
1030
|
+
};
|
|
1031
|
+
await this.state.storage.put(this.childTerminalKey(eventKey), state);
|
|
1032
|
+
this.wakeChildTerminalWaiters(eventKey);
|
|
1033
|
+
return new Response('{}', {
|
|
1034
|
+
headers: { 'content-type': 'application/json' },
|
|
1035
|
+
});
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
private async handleChildTerminalGet(req: Request): Promise<Response> {
|
|
1039
|
+
const eventKey = new URL(req.url).searchParams.get('eventKey')?.trim();
|
|
1040
|
+
if (!eventKey) {
|
|
1041
|
+
return new Response('eventKey is required', { status: 400 });
|
|
1042
|
+
}
|
|
1043
|
+
const state = await this.readChildTerminalState(eventKey);
|
|
1044
|
+
return new Response(JSON.stringify({ state: state ?? null }), {
|
|
1045
|
+
headers: { 'content-type': 'application/json' },
|
|
1046
|
+
});
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
private async handleChildTerminalAwait(req: Request): Promise<Response> {
|
|
1050
|
+
const url = new URL(req.url);
|
|
1051
|
+
const eventKey = url.searchParams.get('eventKey')?.trim();
|
|
1052
|
+
if (!eventKey) {
|
|
1053
|
+
return new Response('eventKey is required', { status: 400 });
|
|
1054
|
+
}
|
|
1055
|
+
const timeoutMs = Math.min(
|
|
1056
|
+
Math.max(Number(url.searchParams.get('timeoutMs') ?? '0'), 0),
|
|
1057
|
+
30_000,
|
|
1058
|
+
);
|
|
1059
|
+
let state = await this.readChildTerminalState(eventKey);
|
|
1060
|
+
if (!state && timeoutMs > 0) {
|
|
1061
|
+
await new Promise<void>((resolve) => {
|
|
1062
|
+
let settled = false;
|
|
1063
|
+
let timeout: ReturnType<typeof setTimeout>;
|
|
1064
|
+
const finish = () => {
|
|
1065
|
+
if (settled) return;
|
|
1066
|
+
settled = true;
|
|
1067
|
+
this.childTerminalWaiters.get(eventKey)?.delete(finish);
|
|
1068
|
+
if (this.childTerminalWaiters.get(eventKey)?.size === 0) {
|
|
1069
|
+
this.childTerminalWaiters.delete(eventKey);
|
|
1070
|
+
}
|
|
1071
|
+
clearTimeout(timeout);
|
|
1072
|
+
resolve();
|
|
1073
|
+
};
|
|
1074
|
+
timeout = setTimeout(finish, timeoutMs);
|
|
1075
|
+
if (!this.childTerminalWaiters.has(eventKey)) {
|
|
1076
|
+
this.childTerminalWaiters.set(eventKey, new Set());
|
|
1077
|
+
}
|
|
1078
|
+
this.childTerminalWaiters.get(eventKey)!.add(finish);
|
|
1079
|
+
});
|
|
1080
|
+
state = await this.readChildTerminalState(eventKey);
|
|
1081
|
+
}
|
|
1082
|
+
return new Response(JSON.stringify({ state: state ?? null }), {
|
|
1083
|
+
headers: { 'content-type': 'application/json' },
|
|
1084
|
+
});
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
/**
|
|
1088
|
+
* Lease up to `requested` request-window permits for `bucketId` under all
|
|
1089
|
+
* `rules`. Single-threaded DO access means this is the same sliding-window
|
|
1090
|
+
* math as InMemoryRateStateBackend.acquire, generalized to grant a BLOCK of
|
|
1091
|
+
* permits at once so the runtime amortizes the round-trip across many calls.
|
|
1092
|
+
*
|
|
1093
|
+
* Concurrency rules (`maxConcurrency`) are NOT enforced here: simultaneous
|
|
1094
|
+
* in-flight tracking across isolates needs a reliable release signal, which a
|
|
1095
|
+
* dying isolate cannot guarantee. The Governor's global tool-concurrency
|
|
1096
|
+
* semaphore is the cross-call concurrency backstop; this DO owns the
|
|
1097
|
+
* cross-isolate REQUEST RATE, which is the throughput governor.
|
|
1098
|
+
*/
|
|
1099
|
+
private async handleRateAcquire(req: Request): Promise<Response> {
|
|
1100
|
+
const body = (await req
|
|
1101
|
+
.json()
|
|
1102
|
+
.catch(() => null)) as RateAcquireRequest | null;
|
|
1103
|
+
if (
|
|
1104
|
+
!body ||
|
|
1105
|
+
typeof body.bucketId !== 'string' ||
|
|
1106
|
+
!body.bucketId.trim() ||
|
|
1107
|
+
!Array.isArray(body.rules)
|
|
1108
|
+
) {
|
|
1109
|
+
return new Response('bucketId and rules are required', { status: 400 });
|
|
1110
|
+
}
|
|
1111
|
+
const requested =
|
|
1112
|
+
typeof body.requested === 'number' && Number.isFinite(body.requested)
|
|
1113
|
+
? Math.max(1, Math.floor(body.requested))
|
|
1114
|
+
: 1;
|
|
1115
|
+
const result = this.computeRateAcquire(
|
|
1116
|
+
body.bucketId,
|
|
1117
|
+
body.rules,
|
|
1118
|
+
requested,
|
|
1119
|
+
);
|
|
1120
|
+
return new Response(JSON.stringify(result), {
|
|
1121
|
+
headers: { 'content-type': 'application/json' },
|
|
1122
|
+
});
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
private computeRateAcquire(
|
|
1126
|
+
bucketId: string,
|
|
1127
|
+
rules: RateRule[],
|
|
1128
|
+
requested: number,
|
|
1129
|
+
): RateAcquireResponse {
|
|
1130
|
+
const now = Date.now();
|
|
1131
|
+
const usableRules = rules.filter(
|
|
1132
|
+
(rule) =>
|
|
1133
|
+
rule &&
|
|
1134
|
+
typeof rule.ruleId === 'string' &&
|
|
1135
|
+
typeof rule.requestsPerWindow === 'number' &&
|
|
1136
|
+
rule.requestsPerWindow > 0 &&
|
|
1137
|
+
typeof rule.windowMs === 'number',
|
|
1138
|
+
);
|
|
1139
|
+
if (usableRules.length === 0) {
|
|
1140
|
+
return { granted: requested, waitMs: 0 };
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
let waitMs = 0;
|
|
1144
|
+
const cooldownUntil = this.rateCooldownUntil.get(bucketId) ?? 0;
|
|
1145
|
+
if (cooldownUntil > now) waitMs = Math.max(waitMs, cooldownUntil - now);
|
|
1146
|
+
|
|
1147
|
+
// The grant is the min remaining capacity across every rule's window. A
|
|
1148
|
+
// single round-trip can never debit more than the tightest rule allows.
|
|
1149
|
+
let grantable = requested;
|
|
1150
|
+
for (const rule of usableRules) {
|
|
1151
|
+
const state = this.getRateWindowState(bucketId, rule, now);
|
|
1152
|
+
this.resetExpiredRateWindow(state, rule.windowMs, now);
|
|
1153
|
+
const remaining = rule.requestsPerWindow - state.startedInWindow;
|
|
1154
|
+
grantable = Math.min(grantable, Math.max(0, remaining));
|
|
1155
|
+
if (remaining <= 0) {
|
|
1156
|
+
waitMs = Math.max(
|
|
1157
|
+
waitMs,
|
|
1158
|
+
rule.windowMs > 0
|
|
1159
|
+
? state.windowStartedAt + rule.windowMs - now
|
|
1160
|
+
: RATE_MIN_WAIT_MS,
|
|
1161
|
+
);
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
if (waitMs > 0 || grantable <= 0) {
|
|
1166
|
+
return { granted: 0, waitMs: Math.max(RATE_MIN_WAIT_MS, waitMs) };
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
for (const rule of usableRules) {
|
|
1170
|
+
const state = this.getRateWindowState(bucketId, rule, now);
|
|
1171
|
+
state.startedInWindow += grantable;
|
|
1172
|
+
}
|
|
1173
|
+
return { granted: grantable, waitMs: 0 };
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
private async handleRatePenalize(req: Request): Promise<Response> {
|
|
1177
|
+
const body = (await req
|
|
1178
|
+
.json()
|
|
1179
|
+
.catch(() => null)) as RatePenalizeRequest | null;
|
|
1180
|
+
if (!body || typeof body.bucketId !== 'string' || !body.bucketId.trim()) {
|
|
1181
|
+
return new Response('bucketId is required', { status: 400 });
|
|
1182
|
+
}
|
|
1183
|
+
const cooldownMs =
|
|
1184
|
+
typeof body.cooldownMs === 'number' && Number.isFinite(body.cooldownMs)
|
|
1185
|
+
? Math.floor(body.cooldownMs)
|
|
1186
|
+
: 0;
|
|
1187
|
+
if (cooldownMs > 0) {
|
|
1188
|
+
const until = Date.now() + cooldownMs;
|
|
1189
|
+
const existing = this.rateCooldownUntil.get(body.bucketId) ?? 0;
|
|
1190
|
+
this.rateCooldownUntil.set(body.bucketId, Math.max(existing, until));
|
|
1191
|
+
}
|
|
1192
|
+
return new Response('{}', {
|
|
1193
|
+
headers: { 'content-type': 'application/json' },
|
|
1194
|
+
});
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
private getRateWindowState(
|
|
1198
|
+
bucketId: string,
|
|
1199
|
+
rule: RateRule,
|
|
1200
|
+
now: number,
|
|
1201
|
+
): RateRuleWindowState {
|
|
1202
|
+
const key = RATE_STATE_KEY(bucketId, rule.ruleId);
|
|
1203
|
+
const existing = this.rateWindows.get(key);
|
|
1204
|
+
if (existing) return existing;
|
|
1205
|
+
const created: RateRuleWindowState = {
|
|
1206
|
+
windowStartedAt: now,
|
|
1207
|
+
startedInWindow: 0,
|
|
1208
|
+
};
|
|
1209
|
+
this.rateWindows.set(key, created);
|
|
1210
|
+
return created;
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
private resetExpiredRateWindow(
|
|
1214
|
+
state: RateRuleWindowState,
|
|
1215
|
+
windowMs: number,
|
|
1216
|
+
now: number,
|
|
1217
|
+
): void {
|
|
1218
|
+
if (windowMs <= 0) {
|
|
1219
|
+
state.windowStartedAt = now;
|
|
1220
|
+
state.startedInWindow = 0;
|
|
1221
|
+
return;
|
|
1222
|
+
}
|
|
1223
|
+
if (now - state.windowStartedAt < windowMs) return;
|
|
1224
|
+
const elapsed = Math.floor((now - state.windowStartedAt) / windowMs);
|
|
1225
|
+
state.windowStartedAt += elapsed * windowMs;
|
|
1226
|
+
state.startedInWindow = 0;
|
|
1227
|
+
if (now - state.windowStartedAt >= windowMs) state.windowStartedAt = now;
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1383
1230
|
private async storeRunEvent(
|
|
1384
1231
|
input: CoordinatorRunEventInput,
|
|
1385
1232
|
): Promise<CoordinatorRunEvent> {
|
|
@@ -1395,12 +1242,9 @@ export class PlayDedup implements DurableObject {
|
|
|
1395
1242
|
[COORDINATOR_RUN_EVENT_SEQUENCE_KEY]: seq,
|
|
1396
1243
|
[this.runEventKey(event)]: event,
|
|
1397
1244
|
});
|
|
1398
|
-
const
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
const overflow = entries.size - COORDINATOR_RUN_EVENT_MAX_ENTRIES;
|
|
1402
|
-
if (overflow > 0) {
|
|
1403
|
-
await this.state.storage.delete([...entries.keys()].slice(0, overflow));
|
|
1245
|
+
const pruneSeq = seq - COORDINATOR_RUN_EVENT_MAX_ENTRIES;
|
|
1246
|
+
if (pruneSeq > 0) {
|
|
1247
|
+
await this.state.storage.delete(this.runEventKeyForSeq(pruneSeq));
|
|
1404
1248
|
}
|
|
1405
1249
|
});
|
|
1406
1250
|
if (!event) {
|
|
@@ -1497,15 +1341,26 @@ export class PlayDedup implements DurableObject {
|
|
|
1497
1341
|
Math.max(Number(url.searchParams.get('timeoutMs') ?? '0'), 0),
|
|
1498
1342
|
30_000,
|
|
1499
1343
|
);
|
|
1500
|
-
const
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1344
|
+
const readLatestSeq = async () =>
|
|
1345
|
+
(await this.state.storage.get<number>(
|
|
1346
|
+
COORDINATOR_RUN_EVENT_SEQUENCE_KEY,
|
|
1347
|
+
)) ?? afterSeq;
|
|
1348
|
+
const readEvents = async (latestSeq: number) => {
|
|
1349
|
+
const firstAvailableSeq = Math.max(
|
|
1350
|
+
1,
|
|
1351
|
+
latestSeq - COORDINATOR_RUN_EVENT_MAX_ENTRIES + 1,
|
|
1352
|
+
);
|
|
1353
|
+
const startSeq = Math.max(afterSeq + 1, firstAvailableSeq);
|
|
1354
|
+
if (latestSeq < startSeq) return [];
|
|
1355
|
+
const keys: string[] = [];
|
|
1356
|
+
for (let seq = startSeq; seq <= latestSeq; seq += 1) {
|
|
1357
|
+
keys.push(this.runEventKeyForSeq(seq));
|
|
1358
|
+
}
|
|
1359
|
+
const entries = await this.state.storage.get<CoordinatorRunEvent>(keys);
|
|
1360
|
+
return [...entries.values()].sort((left, right) => left.seq - right.seq);
|
|
1507
1361
|
};
|
|
1508
|
-
let
|
|
1362
|
+
let latestSeq = await readLatestSeq();
|
|
1363
|
+
let events = await readEvents(latestSeq);
|
|
1509
1364
|
if (events.length === 0 && timeoutMs > 0) {
|
|
1510
1365
|
await new Promise<void>((resolve) => {
|
|
1511
1366
|
let settled = false;
|
|
@@ -1520,12 +1375,9 @@ export class PlayDedup implements DurableObject {
|
|
|
1520
1375
|
const timeout = setTimeout(finish, timeoutMs);
|
|
1521
1376
|
this.runEventWaiters.add(onEvent);
|
|
1522
1377
|
});
|
|
1523
|
-
|
|
1378
|
+
latestSeq = await readLatestSeq();
|
|
1379
|
+
events = await readEvents(latestSeq);
|
|
1524
1380
|
}
|
|
1525
|
-
const latestSeq =
|
|
1526
|
-
(await this.state.storage.get<number>(
|
|
1527
|
-
COORDINATOR_RUN_EVENT_SEQUENCE_KEY,
|
|
1528
|
-
)) ?? afterSeq;
|
|
1529
1381
|
return new Response(JSON.stringify({ events, latestSeq }), {
|
|
1530
1382
|
headers: { 'content-type': 'application/json' },
|
|
1531
1383
|
});
|