deepline 0.1.78 → 0.1.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +69 -37
- package/dist/cli/index.mjs +69 -37
- package/dist/index.d.mts +32 -1
- package/dist/index.d.ts +32 -1
- package/dist/index.js +7 -4
- package/dist/index.mjs +7 -4
- package/dist/repo/apps/play-runner-workers/src/child-play-await.ts +192 -0
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +1320 -1644
- package/dist/repo/apps/play-runner-workers/src/dedup-do.ts +515 -648
- package/dist/repo/apps/play-runner-workers/src/entry.ts +896 -354
- package/dist/repo/apps/play-runner-workers/src/workflow-retry-state.ts +209 -0
- package/dist/repo/sdk/src/client.ts +9 -2
- package/dist/repo/sdk/src/release.ts +2 -2
- package/dist/repo/sdk/src/types.ts +5 -0
- package/dist/repo/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts +231 -0
- package/dist/repo/shared_libs/play-runtime/governor/governor.ts +376 -0
- package/dist/repo/shared_libs/play-runtime/governor/policy.ts +179 -0
- package/dist/repo/shared_libs/play-runtime/governor/rate-state-backend.ts +87 -0
- package/dist/repo/shared_libs/play-runtime/run-failure.ts +12 -0
- package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts +24 -0
- package/dist/repo/shared_libs/play-runtime/submit-limits.ts +35 -0
- package/dist/repo/shared_libs/plays/bundling/index.ts +4 -12
- package/dist/repo/shared_libs/plays/bundling/limits.ts +29 -0
- package/dist/repo/shared_libs/plays/static-pipeline.ts +314 -1
- package/dist/repo/shared_libs/temporal/constants.ts +38 -0
- package/package.json +1 -1
|
@@ -85,35 +85,19 @@ type AwaitRequest = {
|
|
|
85
85
|
timeoutMs: number;
|
|
86
86
|
};
|
|
87
87
|
|
|
88
|
-
type
|
|
89
|
-
|
|
90
|
-
type WorkflowPoolEntry = {
|
|
91
|
-
id: string;
|
|
92
|
-
version: string;
|
|
93
|
-
state: WorkflowPoolEntryState;
|
|
94
|
-
createdAt: number;
|
|
95
|
-
readyAt: number | null;
|
|
96
|
-
expiresAt: number;
|
|
97
|
-
};
|
|
98
|
-
|
|
99
|
-
type WorkflowRunMappingState = 'claimed' | 'started' | 'blocked';
|
|
100
|
-
|
|
101
|
-
type WorkflowRunMapping = {
|
|
88
|
+
type WorkflowRunRetryState = {
|
|
102
89
|
runId: string;
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
version: string;
|
|
109
|
-
createdAt: number;
|
|
90
|
+
params: unknown;
|
|
91
|
+
paramsRef?: unknown;
|
|
92
|
+
paramsBytes?: number;
|
|
93
|
+
retryAttempts: number;
|
|
94
|
+
updatedAt: number;
|
|
110
95
|
expiresAt: number;
|
|
111
96
|
};
|
|
112
97
|
|
|
113
|
-
type
|
|
98
|
+
type WorkflowInstanceState = {
|
|
114
99
|
runId: string;
|
|
115
|
-
|
|
116
|
-
retryAttempts: number;
|
|
100
|
+
instanceId: string;
|
|
117
101
|
updatedAt: number;
|
|
118
102
|
expiresAt: number;
|
|
119
103
|
};
|
|
@@ -148,6 +132,12 @@ type CoordinatorTerminalState = {
|
|
|
148
132
|
completedAt: number;
|
|
149
133
|
};
|
|
150
134
|
|
|
135
|
+
type CoordinatorChildTerminalState = {
|
|
136
|
+
eventKey: string;
|
|
137
|
+
data: unknown;
|
|
138
|
+
storedAt: number;
|
|
139
|
+
};
|
|
140
|
+
|
|
151
141
|
type CoordinatorRunEvent =
|
|
152
142
|
| {
|
|
153
143
|
seq: number;
|
|
@@ -194,31 +184,71 @@ type CoordinatorRunEvent =
|
|
|
194
184
|
type OmitRunEventSequence<T> = T extends unknown ? Omit<T, 'seq'> : never;
|
|
195
185
|
type CoordinatorRunEventInput = OmitRunEventSequence<CoordinatorRunEvent>;
|
|
196
186
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
187
|
+
/**
|
|
188
|
+
* Per-(org,provider) rate-state for the distributed Rate State Backend.
|
|
189
|
+
*
|
|
190
|
+
* One PlayDedup DO instance is addressed per bucket via
|
|
191
|
+
* `idFromName('rate:<orgId>:<provider>')`, so a single instance owns the rate
|
|
192
|
+
* window for that bucket and is single-threaded — which is exactly why the
|
|
193
|
+
* in-process algorithm from `InMemoryRateStateBackend` is correct here. The
|
|
194
|
+
* coordinator RPCs `/rate-acquire` (lease N permits) and `/rate-penalize`
|
|
195
|
+
* (Retry-After cooldown) into it; the esm_workers runtime never blocks per call
|
|
196
|
+
* on a full round-trip because it leases small permit BLOCKS at a time.
|
|
197
|
+
*
|
|
198
|
+
* Rate windows are intentionally kept in DO memory (not durable storage): a
|
|
199
|
+
* window is sub-second-to-seconds ephemeral state, the DO instance outlives any
|
|
200
|
+
* single window, and persisting it would only add storage latency to the hot
|
|
201
|
+
* path. This mirrors the in-memory backend and the Redis sliding-window limiter
|
|
202
|
+
* (`src/lib/redis/customer-rate-limiter.ts`), which also keeps window state in a
|
|
203
|
+
* volatile store with TTL rather than durable rows.
|
|
204
|
+
*/
|
|
205
|
+
type RateRule = {
|
|
206
|
+
ruleId: string;
|
|
207
|
+
requestsPerWindow: number;
|
|
208
|
+
windowMs: number;
|
|
209
|
+
maxConcurrency: number | null;
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
type RateRuleWindowState = {
|
|
213
|
+
windowStartedAt: number;
|
|
214
|
+
startedInWindow: number;
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
type RateAcquireRequest = {
|
|
218
|
+
bucketId: string;
|
|
219
|
+
rules: RateRule[];
|
|
220
|
+
/** How many permits the caller wants to lease in this round-trip. */
|
|
221
|
+
requested: number;
|
|
222
|
+
};
|
|
223
|
+
|
|
224
|
+
type RateAcquireResponse = {
|
|
225
|
+
/** Permits actually granted (0..requested). */
|
|
226
|
+
granted: number;
|
|
227
|
+
/** Suggested wait before the next acquire when granted === 0. */
|
|
228
|
+
waitMs: number;
|
|
200
229
|
};
|
|
201
230
|
|
|
202
|
-
type
|
|
203
|
-
|
|
204
|
-
|
|
231
|
+
type RatePenalizeRequest = {
|
|
232
|
+
bucketId: string;
|
|
233
|
+
cooldownMs: number;
|
|
205
234
|
};
|
|
206
235
|
|
|
236
|
+
const RATE_MIN_WAIT_MS = 10;
|
|
237
|
+
const RATE_STATE_KEY = (bucketId: string, ruleId: string) =>
|
|
238
|
+
`${bucketId}::${ruleId}`;
|
|
239
|
+
|
|
207
240
|
const DEDUP_KEY_PREFIX = 'd:';
|
|
208
|
-
const WORKFLOW_POOL_KEY_PREFIX = 'p:';
|
|
209
|
-
const WORKFLOW_POOL_RUN_KEY_PREFIX = 'm:';
|
|
210
241
|
const WORKFLOW_RUN_RETRY_KEY_PREFIX = 'r:';
|
|
211
242
|
const WORKFLOW_DB_SESSIONS_KEY = 'db-sessions';
|
|
212
243
|
const COORDINATOR_TRACE_KEY_PREFIX = 't:';
|
|
213
244
|
const COORDINATOR_RUN_EVENT_KEY_PREFIX = 'e:';
|
|
214
245
|
const COORDINATOR_TERMINAL_KEY = 'terminal';
|
|
246
|
+
const COORDINATOR_CHILD_TERMINAL_KEY_PREFIX = 'child-terminal:';
|
|
215
247
|
const COORDINATOR_RUN_EVENT_SEQUENCE_KEY = 'event-seq';
|
|
216
248
|
const COORDINATOR_TRACE_MAX_ENTRIES = 200;
|
|
217
249
|
const COORDINATOR_RUN_EVENT_MAX_ENTRIES = 500;
|
|
218
250
|
const FINISH_ALARM_DELAY_MS = 60_000; // self-evict 1 min after finish() called
|
|
219
|
-
const
|
|
220
|
-
const WORKFLOW_POOL_RUN_MAPPING_TTL_MS = 60 * 60 * 1000;
|
|
221
|
-
const WORKFLOW_POOL_READY_MAX_AGE_MS = 7 * 60_000;
|
|
251
|
+
const WORKFLOW_RUN_STATE_TTL_MS = 60 * 60 * 1000;
|
|
222
252
|
const WORKFLOW_RUN_RETRY_STATE_MAX_BYTES = 110_000;
|
|
223
253
|
const WORKFLOW_DB_SESSIONS_TTL_MS = 10 * 60_000;
|
|
224
254
|
|
|
@@ -251,6 +281,13 @@ export class PlayDedup implements DurableObject {
|
|
|
251
281
|
private waiters: Map<string, Set<(value: unknown) => void>> = new Map();
|
|
252
282
|
private runEventWaiters: Set<(value: CoordinatorRunEvent) => void> =
|
|
253
283
|
new Set();
|
|
284
|
+
private childTerminalWaiters: Map<string, Set<() => void>> = new Map();
|
|
285
|
+
// Per-(bucketId, ruleId) request-window state for the rate-state backend.
|
|
286
|
+
// In-memory by design — see the RateRule doc block. Single-threaded DO access
|
|
287
|
+
// makes this the same sliding-window algorithm as InMemoryRateStateBackend.
|
|
288
|
+
private rateWindows: Map<string, RateRuleWindowState> = new Map();
|
|
289
|
+
// Per-bucket Retry-After cooldown floors (penalize()).
|
|
290
|
+
private rateCooldownUntil: Map<string, number> = new Map();
|
|
254
291
|
|
|
255
292
|
constructor(
|
|
256
293
|
private readonly state: DurableObjectState,
|
|
@@ -269,10 +306,35 @@ export class PlayDedup implements DurableObject {
|
|
|
269
306
|
return `${COORDINATOR_RUN_EVENT_KEY_PREFIX}${String(event.seq).padStart(13, '0')}`;
|
|
270
307
|
}
|
|
271
308
|
|
|
309
|
+
private runEventKeyForSeq(seq: number): string {
|
|
310
|
+
return `${COORDINATOR_RUN_EVENT_KEY_PREFIX}${String(seq).padStart(13, '0')}`;
|
|
311
|
+
}
|
|
312
|
+
|
|
272
313
|
private waiterKey(toolId: string, inputHash: string): string {
|
|
273
314
|
return `${toolId}:${inputHash}`;
|
|
274
315
|
}
|
|
275
316
|
|
|
317
|
+
private childTerminalKey(eventKey: string): string {
|
|
318
|
+
return `${COORDINATOR_CHILD_TERMINAL_KEY_PREFIX}${eventKey}`;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
private wakeChildTerminalWaiters(eventKey: string): void {
|
|
322
|
+
const waiters = this.childTerminalWaiters.get(eventKey);
|
|
323
|
+
if (!waiters) return;
|
|
324
|
+
for (const resolve of waiters) {
|
|
325
|
+
resolve();
|
|
326
|
+
}
|
|
327
|
+
this.childTerminalWaiters.delete(eventKey);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
private async readChildTerminalState(
|
|
331
|
+
eventKey: string,
|
|
332
|
+
): Promise<CoordinatorChildTerminalState | undefined> {
|
|
333
|
+
return await this.state.storage.get<CoordinatorChildTerminalState>(
|
|
334
|
+
this.childTerminalKey(eventKey),
|
|
335
|
+
);
|
|
336
|
+
}
|
|
337
|
+
|
|
276
338
|
async fetch(req: Request): Promise<Response> {
|
|
277
339
|
const url = new URL(req.url);
|
|
278
340
|
try {
|
|
@@ -287,36 +349,20 @@ export class PlayDedup implements DurableObject {
|
|
|
287
349
|
return await this.handleFinish(req);
|
|
288
350
|
case '/debug':
|
|
289
351
|
return await this.handleDebug(req);
|
|
290
|
-
case '/pool-add':
|
|
291
|
-
return await this.handlePoolAdd(req);
|
|
292
|
-
case '/pool-claim':
|
|
293
|
-
return await this.handlePoolClaim(req);
|
|
294
|
-
case '/pool-count':
|
|
295
|
-
return await this.handlePoolCount(req);
|
|
296
|
-
case '/pool-list':
|
|
297
|
-
return await this.handlePoolList(req);
|
|
298
|
-
case '/pool-promote':
|
|
299
|
-
return await this.handlePoolPromote(req);
|
|
300
|
-
case '/pool-ready':
|
|
301
|
-
return await this.handlePoolReady(req);
|
|
302
|
-
case '/pool-delete':
|
|
303
|
-
return await this.handlePoolDelete(req);
|
|
304
|
-
case '/pool-map-run':
|
|
305
|
-
return await this.handlePoolMapRun(req);
|
|
306
|
-
case '/pool-block-run':
|
|
307
|
-
return await this.handlePoolBlockRun(req);
|
|
308
|
-
case '/pool-resolve-run':
|
|
309
|
-
return await this.handlePoolResolveRun(req);
|
|
310
352
|
case '/run-retry-state-put':
|
|
311
353
|
return await this.handleRunRetryStatePut(req);
|
|
354
|
+
case '/run-launch-state-put':
|
|
355
|
+
return await this.handleRunLaunchStatePut(req);
|
|
356
|
+
case '/workflow-instance-put':
|
|
357
|
+
return await this.handleWorkflowInstancePut(req);
|
|
358
|
+
case '/workflow-instance-get':
|
|
359
|
+
return await this.handleWorkflowInstanceGet(req);
|
|
312
360
|
case '/run-retry-claim':
|
|
313
361
|
return await this.handleRunRetryClaim(req);
|
|
314
362
|
case '/db-sessions-put':
|
|
315
363
|
return await this.handleDbSessionsPut(req);
|
|
316
364
|
case '/db-sessions-get':
|
|
317
365
|
return await this.handleDbSessionsGet(req);
|
|
318
|
-
case '/pool-clear':
|
|
319
|
-
return await this.handlePoolClear(req);
|
|
320
366
|
case '/trace-add':
|
|
321
367
|
return await this.handleTraceAdd(req);
|
|
322
368
|
case '/trace-list':
|
|
@@ -329,6 +375,16 @@ export class PlayDedup implements DurableObject {
|
|
|
329
375
|
return await this.handleTerminalSet(req);
|
|
330
376
|
case '/terminal-get':
|
|
331
377
|
return await this.handleTerminalGet();
|
|
378
|
+
case '/child-terminal-set':
|
|
379
|
+
return await this.handleChildTerminalSet(req);
|
|
380
|
+
case '/child-terminal-get':
|
|
381
|
+
return await this.handleChildTerminalGet(req);
|
|
382
|
+
case '/child-terminal-await':
|
|
383
|
+
return await this.handleChildTerminalAwait(req);
|
|
384
|
+
case '/rate-acquire':
|
|
385
|
+
return await this.handleRateAcquire(req);
|
|
386
|
+
case '/rate-penalize':
|
|
387
|
+
return await this.handleRatePenalize(req);
|
|
332
388
|
default:
|
|
333
389
|
return new Response('not found', { status: 404 });
|
|
334
390
|
}
|
|
@@ -343,10 +399,11 @@ export class PlayDedup implements DurableObject {
|
|
|
343
399
|
|
|
344
400
|
async alarm(): Promise<void> {
|
|
345
401
|
// Fired after /finish was called. Evict storage and let the DO instance
|
|
346
|
-
// be garbage-collected by
|
|
402
|
+
// be garbage-collected by Cloudflare.
|
|
347
403
|
await this.state.storage.deleteAll();
|
|
348
404
|
this.waiters.clear();
|
|
349
405
|
this.runEventWaiters.clear();
|
|
406
|
+
this.childTerminalWaiters.clear();
|
|
350
407
|
}
|
|
351
408
|
|
|
352
409
|
private async handleLookupOrClaim(req: Request): Promise<Response> {
|
|
@@ -514,575 +571,125 @@ export class PlayDedup implements DurableObject {
|
|
|
514
571
|
);
|
|
515
572
|
}
|
|
516
573
|
|
|
517
|
-
private
|
|
518
|
-
return new URL(req.url).searchParams.get('version')?.trim() ?? '';
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
private workflowPoolMinReadyAgeMs(req: Request): number {
|
|
522
|
-
const raw = Number(new URL(req.url).searchParams.get('minReadyAgeMs') ?? 0);
|
|
523
|
-
if (!Number.isFinite(raw) || raw <= 0) {
|
|
524
|
-
return 0;
|
|
525
|
-
}
|
|
526
|
-
return Math.min(Math.floor(raw), WORKFLOW_POOL_DEFAULT_TTL_MS);
|
|
527
|
-
}
|
|
528
|
-
|
|
529
|
-
private isReadyWorkflowPoolEntry(
|
|
530
|
-
value: { key: string; entry: WorkflowPoolEntry | undefined },
|
|
531
|
-
version: string,
|
|
532
|
-
now: number,
|
|
533
|
-
minReadyAgeMs = 0,
|
|
534
|
-
): value is ReadyWorkflowPoolEntryRecord {
|
|
535
|
-
return (
|
|
536
|
-
value.entry !== undefined &&
|
|
537
|
-
value.entry.version === version &&
|
|
538
|
-
value.entry.expiresAt > now &&
|
|
539
|
-
value.entry.state === 'ready' &&
|
|
540
|
-
value.entry.readyAt !== null &&
|
|
541
|
-
now - value.entry.readyAt <= WORKFLOW_POOL_READY_MAX_AGE_MS &&
|
|
542
|
-
now - value.entry.readyAt >= minReadyAgeMs
|
|
543
|
-
);
|
|
544
|
-
}
|
|
545
|
-
|
|
546
|
-
private async gcWorkflowPool(
|
|
547
|
-
now = Date.now(),
|
|
548
|
-
version?: string,
|
|
549
|
-
): Promise<void> {
|
|
550
|
-
const [pool, mappings, retries] = await Promise.all([
|
|
551
|
-
this.state.storage.list<WorkflowPoolEntry>({
|
|
552
|
-
prefix: WORKFLOW_POOL_KEY_PREFIX,
|
|
553
|
-
}),
|
|
554
|
-
this.state.storage.list<WorkflowRunMapping>({
|
|
555
|
-
prefix: WORKFLOW_POOL_RUN_KEY_PREFIX,
|
|
556
|
-
}),
|
|
557
|
-
this.state.storage.list<WorkflowRunRetryState>({
|
|
558
|
-
prefix: WORKFLOW_RUN_RETRY_KEY_PREFIX,
|
|
559
|
-
}),
|
|
560
|
-
]);
|
|
561
|
-
const expiredKeys: string[] = [];
|
|
562
|
-
for (const [key, entry] of pool) {
|
|
563
|
-
if (
|
|
564
|
-
!entry ||
|
|
565
|
-
entry.expiresAt <= now ||
|
|
566
|
-
(entry.state === 'ready' &&
|
|
567
|
-
entry.readyAt !== null &&
|
|
568
|
-
now - entry.readyAt > WORKFLOW_POOL_READY_MAX_AGE_MS) ||
|
|
569
|
-
(version && entry.version !== version)
|
|
570
|
-
) {
|
|
571
|
-
expiredKeys.push(key);
|
|
572
|
-
}
|
|
573
|
-
}
|
|
574
|
-
for (const [key, mapping] of mappings) {
|
|
575
|
-
if (
|
|
576
|
-
!mapping ||
|
|
577
|
-
mapping.expiresAt <= now ||
|
|
578
|
-
(version && mapping.version !== version)
|
|
579
|
-
) {
|
|
580
|
-
expiredKeys.push(key);
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
for (const [key, retryState] of retries) {
|
|
584
|
-
if (!retryState || retryState.expiresAt <= now) {
|
|
585
|
-
expiredKeys.push(key);
|
|
586
|
-
}
|
|
587
|
-
}
|
|
588
|
-
if (expiredKeys.length > 0) {
|
|
589
|
-
await this.state.storage.delete(expiredKeys);
|
|
590
|
-
}
|
|
591
|
-
}
|
|
592
|
-
|
|
593
|
-
private async handlePoolAdd(req: Request): Promise<Response> {
|
|
594
|
-
const body = (await req.json().catch(() => null)) as {
|
|
595
|
-
ids?: unknown;
|
|
596
|
-
ttlMs?: unknown;
|
|
597
|
-
version?: unknown;
|
|
598
|
-
readyAt?: unknown;
|
|
599
|
-
ready?: unknown;
|
|
600
|
-
} | null;
|
|
601
|
-
const version =
|
|
602
|
-
typeof body?.version === 'string' ? body.version.trim() : '';
|
|
603
|
-
if (!version) {
|
|
604
|
-
return new Response('version is required', { status: 400 });
|
|
605
|
-
}
|
|
606
|
-
const ids = Array.isArray(body?.ids)
|
|
607
|
-
? body.ids.filter(
|
|
608
|
-
(id): id is string => typeof id === 'string' && id.length > 0,
|
|
609
|
-
)
|
|
610
|
-
: [];
|
|
611
|
-
const now = Date.now();
|
|
612
|
-
const hasReadyAt =
|
|
613
|
-
typeof body?.readyAt === 'number' && Number.isFinite(body.readyAt);
|
|
614
|
-
const readyAt = hasReadyAt ? Math.max(0, body.readyAt as number) : now;
|
|
615
|
-
const ready = body?.ready === true || hasReadyAt;
|
|
616
|
-
const ttlMs =
|
|
617
|
-
typeof body?.ttlMs === 'number' &&
|
|
618
|
-
Number.isFinite(body.ttlMs) &&
|
|
619
|
-
body.ttlMs > 0
|
|
620
|
-
? Math.min(body.ttlMs, WORKFLOW_POOL_DEFAULT_TTL_MS)
|
|
621
|
-
: WORKFLOW_POOL_DEFAULT_TTL_MS;
|
|
622
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
623
|
-
await this.gcWorkflowPool(now, version);
|
|
624
|
-
const writes: Record<string, WorkflowPoolEntry> = {};
|
|
625
|
-
const keys = ids.map((id) => `${WORKFLOW_POOL_KEY_PREFIX}${id}`);
|
|
626
|
-
const existing = (await this.state.storage.get<WorkflowPoolEntry>(
|
|
627
|
-
keys,
|
|
628
|
-
)) as Map<string, WorkflowPoolEntry>;
|
|
629
|
-
for (const id of ids) {
|
|
630
|
-
const key = `${WORKFLOW_POOL_KEY_PREFIX}${id}`;
|
|
631
|
-
const existingEntry = existing.get(key);
|
|
632
|
-
const existingReadyAt =
|
|
633
|
-
existingEntry?.version === version &&
|
|
634
|
-
existingEntry.expiresAt > now &&
|
|
635
|
-
existingEntry.state === 'ready' &&
|
|
636
|
-
existingEntry.readyAt !== null
|
|
637
|
-
? existingEntry.readyAt
|
|
638
|
-
: null;
|
|
639
|
-
const nextReadyAt = ready ? readyAt : existingReadyAt;
|
|
640
|
-
writes[key] = {
|
|
641
|
-
id,
|
|
642
|
-
version,
|
|
643
|
-
state: nextReadyAt !== null ? 'ready' : 'warming',
|
|
644
|
-
createdAt:
|
|
645
|
-
existingEntry?.version === version ? existingEntry.createdAt : now,
|
|
646
|
-
readyAt: nextReadyAt,
|
|
647
|
-
expiresAt: now + ttlMs,
|
|
648
|
-
};
|
|
649
|
-
}
|
|
650
|
-
if (Object.keys(writes).length > 0) {
|
|
651
|
-
await this.state.storage.put(writes);
|
|
652
|
-
}
|
|
653
|
-
});
|
|
654
|
-
return new Response(JSON.stringify({ added: ids.length }), {
|
|
655
|
-
headers: { 'content-type': 'application/json' },
|
|
656
|
-
});
|
|
657
|
-
}
|
|
658
|
-
|
|
659
|
-
private async handlePoolClaim(req: Request): Promise<Response> {
|
|
660
|
-
const version = this.workflowPoolVersion(req);
|
|
661
|
-
if (!version) {
|
|
662
|
-
return new Response('version is required', { status: 400 });
|
|
663
|
-
}
|
|
664
|
-
const body = (await req.json().catch(() => null)) as {
|
|
665
|
-
runId?: unknown;
|
|
666
|
-
} | null;
|
|
667
|
-
const runId = typeof body?.runId === 'string' ? body.runId : '';
|
|
668
|
-
if (!runId) {
|
|
669
|
-
return new Response('runId is required', { status: 400 });
|
|
670
|
-
}
|
|
671
|
-
const minReadyAgeMs = this.workflowPoolMinReadyAgeMs(req);
|
|
672
|
-
let claimedId: string | null = null;
|
|
673
|
-
let counts: WorkflowPoolCounts = { available: 0, warming: 0 };
|
|
674
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
675
|
-
const now = Date.now();
|
|
676
|
-
await this.gcWorkflowPool(now, version);
|
|
677
|
-
const mappingKey = `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`;
|
|
678
|
-
const existingMapping =
|
|
679
|
-
await this.state.storage.get<WorkflowRunMapping>(mappingKey);
|
|
680
|
-
if (existingMapping?.version === version) {
|
|
681
|
-
if (
|
|
682
|
-
existingMapping.state === 'claimed' ||
|
|
683
|
-
existingMapping.state === 'started'
|
|
684
|
-
) {
|
|
685
|
-
claimedId = existingMapping.instanceId || null;
|
|
686
|
-
}
|
|
687
|
-
return;
|
|
688
|
-
}
|
|
689
|
-
const entries = await this.state.storage.list<WorkflowPoolEntry>({
|
|
690
|
-
prefix: WORKFLOW_POOL_KEY_PREFIX,
|
|
691
|
-
});
|
|
692
|
-
counts = this.countWorkflowPoolEntries(
|
|
693
|
-
entries,
|
|
694
|
-
version,
|
|
695
|
-
now,
|
|
696
|
-
minReadyAgeMs,
|
|
697
|
-
);
|
|
698
|
-
const sorted = [...entries.entries()]
|
|
699
|
-
.map(([key, entry]) => ({ key, entry }))
|
|
700
|
-
.filter((entry) =>
|
|
701
|
-
this.isReadyWorkflowPoolEntry(entry, version, now, minReadyAgeMs),
|
|
702
|
-
)
|
|
703
|
-
.sort((a, b) => b.entry.readyAt - a.entry.readyAt);
|
|
704
|
-
const selected = sorted[0];
|
|
705
|
-
if (!selected) return;
|
|
706
|
-
claimedId = selected.entry.id;
|
|
707
|
-
await this.state.storage.delete(selected.key);
|
|
708
|
-
await this.state.storage.put(mappingKey, {
|
|
709
|
-
runId,
|
|
710
|
-
instanceId: claimedId,
|
|
711
|
-
state: 'claimed',
|
|
712
|
-
blockedInstanceId: null,
|
|
713
|
-
claimedAt: now,
|
|
714
|
-
startedAt: null,
|
|
715
|
-
version,
|
|
716
|
-
createdAt: now,
|
|
717
|
-
expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
|
|
718
|
-
} satisfies WorkflowRunMapping);
|
|
719
|
-
counts = {
|
|
720
|
-
available: Math.max(0, counts.available - 1),
|
|
721
|
-
warming: counts.warming,
|
|
722
|
-
};
|
|
723
|
-
});
|
|
724
|
-
return new Response(JSON.stringify({ id: claimedId, ...counts }), {
|
|
725
|
-
headers: { 'content-type': 'application/json' },
|
|
726
|
-
});
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
private async handlePoolCount(req: Request): Promise<Response> {
|
|
730
|
-
const version = this.workflowPoolVersion(req);
|
|
731
|
-
if (!version) {
|
|
732
|
-
return new Response('version is required', { status: 400 });
|
|
733
|
-
}
|
|
734
|
-
const minReadyAgeMs = this.workflowPoolMinReadyAgeMs(req);
|
|
735
|
-
const now = Date.now();
|
|
736
|
-
await this.gcWorkflowPool(now, version);
|
|
737
|
-
const entries = await this.state.storage.list<WorkflowPoolEntry>({
|
|
738
|
-
prefix: WORKFLOW_POOL_KEY_PREFIX,
|
|
739
|
-
});
|
|
740
|
-
const counts = this.countWorkflowPoolEntries(
|
|
741
|
-
entries,
|
|
742
|
-
version,
|
|
743
|
-
now,
|
|
744
|
-
minReadyAgeMs,
|
|
745
|
-
);
|
|
746
|
-
return new Response(JSON.stringify(counts), {
|
|
747
|
-
headers: { 'content-type': 'application/json' },
|
|
748
|
-
});
|
|
749
|
-
}
|
|
750
|
-
|
|
751
|
-
private countWorkflowPoolEntries(
|
|
752
|
-
entries: Map<string, WorkflowPoolEntry>,
|
|
753
|
-
version: string,
|
|
754
|
-
now: number,
|
|
755
|
-
minReadyAgeMs: number,
|
|
756
|
-
): WorkflowPoolCounts {
|
|
757
|
-
let available = 0;
|
|
758
|
-
let warming = 0;
|
|
759
|
-
for (const entry of entries.values()) {
|
|
760
|
-
if (entry.version !== version) continue;
|
|
761
|
-
if (
|
|
762
|
-
entry.state !== 'ready' ||
|
|
763
|
-
entry.readyAt === null ||
|
|
764
|
-
now - entry.readyAt > WORKFLOW_POOL_READY_MAX_AGE_MS ||
|
|
765
|
-
now - entry.readyAt < minReadyAgeMs
|
|
766
|
-
) {
|
|
767
|
-
warming += 1;
|
|
768
|
-
} else {
|
|
769
|
-
available += 1;
|
|
770
|
-
}
|
|
771
|
-
}
|
|
772
|
-
return { available, warming };
|
|
773
|
-
}
|
|
774
|
-
|
|
775
|
-
private async handlePoolList(req: Request): Promise<Response> {
|
|
776
|
-
const version = this.workflowPoolVersion(req);
|
|
777
|
-
if (!version) {
|
|
778
|
-
return new Response('version is required', { status: 400 });
|
|
779
|
-
}
|
|
780
|
-
await this.gcWorkflowPool(Date.now(), version);
|
|
781
|
-
const entries = await this.state.storage.list<WorkflowPoolEntry>({
|
|
782
|
-
prefix: WORKFLOW_POOL_KEY_PREFIX,
|
|
783
|
-
});
|
|
784
|
-
return new Response(
|
|
785
|
-
JSON.stringify({
|
|
786
|
-
entries: [...entries.values()]
|
|
787
|
-
.filter((entry) => entry.version === version)
|
|
788
|
-
.map((entry) => ({
|
|
789
|
-
id: entry.id,
|
|
790
|
-
state: entry.state,
|
|
791
|
-
createdAt: entry.createdAt,
|
|
792
|
-
readyAt: entry.readyAt,
|
|
793
|
-
expiresAt: entry.expiresAt,
|
|
794
|
-
})),
|
|
795
|
-
}),
|
|
796
|
-
{ headers: { 'content-type': 'application/json' } },
|
|
797
|
-
);
|
|
798
|
-
}
|
|
799
|
-
|
|
800
|
-
private async handlePoolPromote(req: Request): Promise<Response> {
|
|
801
|
-
const body = (await req.json().catch(() => null)) as {
|
|
802
|
-
ids?: unknown;
|
|
803
|
-
version?: unknown;
|
|
804
|
-
} | null;
|
|
805
|
-
const version =
|
|
806
|
-
typeof body?.version === 'string' ? body.version.trim() : '';
|
|
807
|
-
if (!version) {
|
|
808
|
-
return new Response('version is required', { status: 400 });
|
|
809
|
-
}
|
|
810
|
-
const ids = Array.isArray(body?.ids)
|
|
811
|
-
? body.ids.filter(
|
|
812
|
-
(id): id is string => typeof id === 'string' && id.length > 0,
|
|
813
|
-
)
|
|
814
|
-
: [];
|
|
815
|
-
const now = Date.now();
|
|
816
|
-
const writes: Record<string, WorkflowPoolEntry> = {};
|
|
817
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
818
|
-
await this.gcWorkflowPool(now, version);
|
|
819
|
-
for (const id of ids) {
|
|
820
|
-
const key = `${WORKFLOW_POOL_KEY_PREFIX}${id}`;
|
|
821
|
-
const entry = await this.state.storage.get<WorkflowPoolEntry>(key);
|
|
822
|
-
if (!entry || entry.version !== version || entry.expiresAt <= now) {
|
|
823
|
-
continue;
|
|
824
|
-
}
|
|
825
|
-
writes[key] = {
|
|
826
|
-
...entry,
|
|
827
|
-
state: 'ready',
|
|
828
|
-
readyAt: now,
|
|
829
|
-
};
|
|
830
|
-
}
|
|
831
|
-
if (Object.keys(writes).length > 0) {
|
|
832
|
-
await this.state.storage.put(writes);
|
|
833
|
-
}
|
|
834
|
-
});
|
|
835
|
-
return new Response(
|
|
836
|
-
JSON.stringify({ promoted: Object.keys(writes).length }),
|
|
837
|
-
{
|
|
838
|
-
headers: { 'content-type': 'application/json' },
|
|
839
|
-
},
|
|
840
|
-
);
|
|
841
|
-
}
|
|
842
|
-
|
|
843
|
-
private async handlePoolReady(req: Request): Promise<Response> {
|
|
844
|
-
const body = (await req.json().catch(() => null)) as {
|
|
845
|
-
poolId?: unknown;
|
|
846
|
-
version?: unknown;
|
|
847
|
-
} | null;
|
|
848
|
-
const poolId = typeof body?.poolId === 'string' ? body.poolId : '';
|
|
849
|
-
const version =
|
|
850
|
-
typeof body?.version === 'string' ? body.version.trim() : '';
|
|
851
|
-
if (!poolId || !version) {
|
|
852
|
-
return new Response('poolId and version are required', { status: 400 });
|
|
853
|
-
}
|
|
854
|
-
const now = Date.now();
|
|
855
|
-
let ready = false;
|
|
856
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
857
|
-
await this.gcWorkflowPool(now, version);
|
|
858
|
-
const key = `${WORKFLOW_POOL_KEY_PREFIX}${poolId}`;
|
|
859
|
-
const entry = await this.state.storage.get<WorkflowPoolEntry>(key);
|
|
860
|
-
if (!entry || entry.version !== version || entry.expiresAt <= now) {
|
|
861
|
-
return;
|
|
862
|
-
}
|
|
863
|
-
await this.state.storage.put(key, {
|
|
864
|
-
...entry,
|
|
865
|
-
state: 'ready',
|
|
866
|
-
readyAt: entry.readyAt ?? now,
|
|
867
|
-
});
|
|
868
|
-
ready = true;
|
|
869
|
-
});
|
|
870
|
-
return new Response(JSON.stringify({ ready }), {
|
|
871
|
-
headers: { 'content-type': 'application/json' },
|
|
872
|
-
});
|
|
873
|
-
}
|
|
874
|
-
|
|
875
|
-
private async handlePoolDelete(req: Request): Promise<Response> {
|
|
876
|
-
const body = (await req.json().catch(() => null)) as {
|
|
877
|
-
ids?: unknown;
|
|
878
|
-
version?: unknown;
|
|
879
|
-
} | null;
|
|
880
|
-
const version =
|
|
881
|
-
typeof body?.version === 'string' ? body.version.trim() : '';
|
|
882
|
-
if (!version) {
|
|
883
|
-
return new Response('version is required', { status: 400 });
|
|
884
|
-
}
|
|
885
|
-
const ids = Array.isArray(body?.ids)
|
|
886
|
-
? body.ids.filter(
|
|
887
|
-
(id): id is string => typeof id === 'string' && id.length > 0,
|
|
888
|
-
)
|
|
889
|
-
: [];
|
|
890
|
-
const keys: string[] = [];
|
|
891
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
892
|
-
await this.gcWorkflowPool(Date.now(), version);
|
|
893
|
-
for (const id of ids) {
|
|
894
|
-
const key = `${WORKFLOW_POOL_KEY_PREFIX}${id}`;
|
|
895
|
-
const entry = await this.state.storage.get<WorkflowPoolEntry>(key);
|
|
896
|
-
if (entry?.version === version) keys.push(key);
|
|
897
|
-
}
|
|
898
|
-
if (keys.length > 0) {
|
|
899
|
-
await this.state.storage.delete(keys);
|
|
900
|
-
}
|
|
901
|
-
});
|
|
902
|
-
return new Response(JSON.stringify({ deleted: keys.length }), {
|
|
903
|
-
headers: { 'content-type': 'application/json' },
|
|
904
|
-
});
|
|
905
|
-
}
|
|
906
|
-
|
|
907
|
-
private async handlePoolMapRun(req: Request): Promise<Response> {
|
|
574
|
+
private async handleRunRetryStatePut(req: Request): Promise<Response> {
|
|
908
575
|
const body = (await req.json().catch(() => null)) as {
|
|
909
576
|
runId?: unknown;
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
started?: unknown;
|
|
577
|
+
params?: unknown;
|
|
578
|
+
ttlMs?: unknown;
|
|
913
579
|
} | null;
|
|
914
580
|
const runId = typeof body?.runId === 'string' ? body.runId : '';
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
const version =
|
|
918
|
-
typeof body?.version === 'string' ? body.version.trim() : '';
|
|
919
|
-
const started = body?.started === true;
|
|
920
|
-
if (!runId || !instanceId || !version) {
|
|
921
|
-
return new Response('runId, instanceId, and version are required', {
|
|
581
|
+
if (!runId || !body || (!('params' in body) && !('paramsRef' in body))) {
|
|
582
|
+
return new Response('runId and params or paramsRef are required', {
|
|
922
583
|
status: 400,
|
|
923
584
|
});
|
|
924
585
|
}
|
|
925
586
|
const now = Date.now();
|
|
926
|
-
|
|
927
|
-
|
|
587
|
+
const ttlMs =
|
|
588
|
+
typeof body.ttlMs === 'number' && Number.isFinite(body.ttlMs)
|
|
589
|
+
? Math.max(60_000, Math.min(body.ttlMs, WORKFLOW_RUN_STATE_TTL_MS))
|
|
590
|
+
: WORKFLOW_RUN_STATE_TTL_MS;
|
|
591
|
+
const key = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
|
|
928
592
|
await this.state.blockConcurrencyWhile(async () => {
|
|
929
|
-
const existing = await this.state.storage.get<
|
|
930
|
-
|
|
931
|
-
existing?.version === version &&
|
|
932
|
-
existing.state === 'blocked' &&
|
|
933
|
-
existing.blockedInstanceId === instanceId
|
|
934
|
-
) {
|
|
935
|
-
mapped = false;
|
|
936
|
-
return;
|
|
937
|
-
}
|
|
938
|
-
const alreadyStarted =
|
|
939
|
-
existing?.version === version &&
|
|
940
|
-
existing.instanceId === instanceId &&
|
|
941
|
-
existing.state === 'started' &&
|
|
942
|
-
typeof existing.startedAt === 'number';
|
|
943
|
-
const startedAt = started || alreadyStarted ? now : null;
|
|
944
|
-
await this.state.storage.put(key, {
|
|
593
|
+
const existing = await this.state.storage.get<WorkflowRunRetryState>(key);
|
|
594
|
+
const retryState = {
|
|
945
595
|
runId,
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
}
|
|
967
|
-
|
|
968
|
-
private async handlePoolBlockRun(req: Request): Promise<Response> {
|
|
969
|
-
const body = (await req.json().catch(() => null)) as {
|
|
970
|
-
runId?: unknown;
|
|
971
|
-
instanceId?: unknown;
|
|
972
|
-
version?: unknown;
|
|
973
|
-
} | null;
|
|
974
|
-
const runId = typeof body?.runId === 'string' ? body.runId : '';
|
|
975
|
-
const instanceId =
|
|
976
|
-
typeof body?.instanceId === 'string' ? body.instanceId : '';
|
|
977
|
-
const version =
|
|
978
|
-
typeof body?.version === 'string' ? body.version.trim() : '';
|
|
979
|
-
if (!runId || !instanceId || !version) {
|
|
980
|
-
return new Response('runId, instanceId, and version are required', {
|
|
981
|
-
status: 400,
|
|
982
|
-
});
|
|
983
|
-
}
|
|
984
|
-
const now = Date.now();
|
|
985
|
-
const key = `${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`;
|
|
986
|
-
let started = false;
|
|
987
|
-
await this.state.blockConcurrencyWhile(async () => {
|
|
988
|
-
const existing = await this.state.storage.get<WorkflowRunMapping>(key);
|
|
989
|
-
if (
|
|
990
|
-
existing?.version === version &&
|
|
991
|
-
existing.instanceId === instanceId &&
|
|
992
|
-
existing.state === 'started' &&
|
|
993
|
-
typeof existing.startedAt === 'number'
|
|
994
|
-
) {
|
|
995
|
-
started = true;
|
|
996
|
-
return;
|
|
596
|
+
params: 'params' in body ? body.params : null,
|
|
597
|
+
paramsRef: 'paramsRef' in body ? body.paramsRef : null,
|
|
598
|
+
paramsBytes:
|
|
599
|
+
typeof (body as { paramsBytes?: unknown }).paramsBytes === 'number' &&
|
|
600
|
+
Number.isFinite((body as { paramsBytes?: number }).paramsBytes)
|
|
601
|
+
? (body as { paramsBytes: number }).paramsBytes
|
|
602
|
+
: undefined,
|
|
603
|
+
retryAttempts:
|
|
604
|
+
existing?.runId === runId &&
|
|
605
|
+
typeof existing.retryAttempts === 'number'
|
|
606
|
+
? existing.retryAttempts
|
|
607
|
+
: 0,
|
|
608
|
+
updatedAt: now,
|
|
609
|
+
expiresAt: now + ttlMs,
|
|
610
|
+
} satisfies WorkflowRunRetryState;
|
|
611
|
+
const bytes = jsonByteLength(retryState);
|
|
612
|
+
if (bytes > WORKFLOW_RUN_RETRY_STATE_MAX_BYTES) {
|
|
613
|
+
throw new Error(
|
|
614
|
+
`workflow retry state too large: ${bytes} bytes exceeds ${WORKFLOW_RUN_RETRY_STATE_MAX_BYTES}`,
|
|
615
|
+
);
|
|
997
616
|
}
|
|
998
|
-
await this.state.storage.put(key,
|
|
999
|
-
runId,
|
|
1000
|
-
instanceId: '',
|
|
1001
|
-
state: 'blocked',
|
|
1002
|
-
blockedInstanceId: instanceId,
|
|
1003
|
-
claimedAt:
|
|
1004
|
-
existing?.version === version &&
|
|
1005
|
-
typeof existing.claimedAt === 'number'
|
|
1006
|
-
? existing.claimedAt
|
|
1007
|
-
: now,
|
|
1008
|
-
startedAt: null,
|
|
1009
|
-
version,
|
|
1010
|
-
createdAt: now,
|
|
1011
|
-
expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
|
|
1012
|
-
} satisfies WorkflowRunMapping);
|
|
617
|
+
await this.state.storage.put(key, retryState);
|
|
1013
618
|
});
|
|
1014
|
-
|
|
1015
|
-
return new Response(JSON.stringify({ blocked: false, started: true }), {
|
|
1016
|
-
headers: { 'content-type': 'application/json' },
|
|
1017
|
-
});
|
|
1018
|
-
}
|
|
1019
|
-
return new Response(JSON.stringify({ blocked: true, started: false }), {
|
|
619
|
+
return new Response(JSON.stringify({ ok: true }), {
|
|
1020
620
|
headers: { 'content-type': 'application/json' },
|
|
1021
621
|
});
|
|
1022
622
|
}
|
|
1023
623
|
|
|
1024
|
-
private async
|
|
1025
|
-
const url = new URL(req.url);
|
|
1026
|
-
const runId = url.searchParams.get('runId') ?? '';
|
|
1027
|
-
const version = url.searchParams.get('version')?.trim() ?? '';
|
|
1028
|
-
if (!runId) {
|
|
1029
|
-
return new Response('runId is required', { status: 400 });
|
|
1030
|
-
}
|
|
1031
|
-
if (!version) {
|
|
1032
|
-
return new Response('version is required', { status: 400 });
|
|
1033
|
-
}
|
|
1034
|
-
await this.gcWorkflowPool(Date.now(), version);
|
|
1035
|
-
const mapping = await this.state.storage.get<WorkflowRunMapping>(
|
|
1036
|
-
`${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`,
|
|
1037
|
-
);
|
|
1038
|
-
if (mapping && mapping.version !== version) {
|
|
1039
|
-
await this.state.storage.delete(
|
|
1040
|
-
`${WORKFLOW_POOL_RUN_KEY_PREFIX}${runId}`,
|
|
1041
|
-
);
|
|
1042
|
-
return new Response(JSON.stringify({ instanceId: null }), {
|
|
1043
|
-
headers: { 'content-type': 'application/json' },
|
|
1044
|
-
});
|
|
1045
|
-
}
|
|
1046
|
-
return new Response(
|
|
1047
|
-
JSON.stringify({
|
|
1048
|
-
instanceId: mapping?.instanceId || null,
|
|
1049
|
-
startedAt: mapping?.startedAt ?? null,
|
|
1050
|
-
}),
|
|
1051
|
-
{ headers: { 'content-type': 'application/json' } },
|
|
1052
|
-
);
|
|
1053
|
-
}
|
|
1054
|
-
|
|
1055
|
-
private async handleRunRetryStatePut(req: Request): Promise<Response> {
|
|
624
|
+
private async handleRunLaunchStatePut(req: Request): Promise<Response> {
|
|
1056
625
|
const body = (await req.json().catch(() => null)) as {
|
|
1057
626
|
runId?: unknown;
|
|
1058
627
|
params?: unknown;
|
|
1059
|
-
|
|
628
|
+
paramsRef?: unknown;
|
|
629
|
+
paramsBytes?: unknown;
|
|
630
|
+
sessions?: unknown;
|
|
631
|
+
retryTtlMs?: unknown;
|
|
632
|
+
dbSessionsTtlMs?: unknown;
|
|
1060
633
|
} | null;
|
|
1061
634
|
const runId = typeof body?.runId === 'string' ? body.runId : '';
|
|
1062
|
-
|
|
1063
|
-
|
|
635
|
+
const sessions = Array.isArray(body?.sessions)
|
|
636
|
+
? (body.sessions as PreloadedRuntimeDbSession[])
|
|
637
|
+
: null;
|
|
638
|
+
if (
|
|
639
|
+
!runId ||
|
|
640
|
+
!body ||
|
|
641
|
+
(!('params' in body) && !('paramsRef' in body)) ||
|
|
642
|
+
!sessions
|
|
643
|
+
) {
|
|
644
|
+
return new Response(
|
|
645
|
+
'runId, params or paramsRef, and sessions are required',
|
|
646
|
+
{
|
|
647
|
+
status: 400,
|
|
648
|
+
},
|
|
649
|
+
);
|
|
1064
650
|
}
|
|
651
|
+
assertEncryptedDbSessionsForStorage(sessions);
|
|
1065
652
|
const now = Date.now();
|
|
1066
|
-
const
|
|
1067
|
-
typeof body.
|
|
1068
|
-
? Math.max(
|
|
1069
|
-
|
|
1070
|
-
|
|
653
|
+
const retryTtlMs =
|
|
654
|
+
typeof body.retryTtlMs === 'number' && Number.isFinite(body.retryTtlMs)
|
|
655
|
+
? Math.max(60_000, Math.min(body.retryTtlMs, WORKFLOW_RUN_STATE_TTL_MS))
|
|
656
|
+
: WORKFLOW_RUN_STATE_TTL_MS;
|
|
657
|
+
const dbSessionsTtlMs =
|
|
658
|
+
typeof body.dbSessionsTtlMs === 'number' &&
|
|
659
|
+
Number.isFinite(body.dbSessionsTtlMs) &&
|
|
660
|
+
body.dbSessionsTtlMs > 0
|
|
661
|
+
? Math.min(
|
|
662
|
+
Math.max(Math.floor(body.dbSessionsTtlMs), 60_000),
|
|
663
|
+
30 * 60_000,
|
|
1071
664
|
)
|
|
1072
|
-
:
|
|
1073
|
-
const
|
|
665
|
+
: WORKFLOW_DB_SESSIONS_TTL_MS;
|
|
666
|
+
const retryKey = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
|
|
667
|
+
const dbSessionsState: WorkflowDbSessionsState = {
|
|
668
|
+
runId,
|
|
669
|
+
sessions,
|
|
670
|
+
storedAt: now,
|
|
671
|
+
expiresAt: now + dbSessionsTtlMs,
|
|
672
|
+
};
|
|
673
|
+
let retryStateExpiresAt = now + retryTtlMs;
|
|
1074
674
|
await this.state.blockConcurrencyWhile(async () => {
|
|
1075
|
-
const existing =
|
|
675
|
+
const existing =
|
|
676
|
+
await this.state.storage.get<WorkflowRunRetryState>(retryKey);
|
|
1076
677
|
const retryState = {
|
|
1077
678
|
runId,
|
|
1078
|
-
params: body.params,
|
|
679
|
+
params: 'params' in body ? body.params : null,
|
|
680
|
+
paramsRef: 'paramsRef' in body ? body.paramsRef : null,
|
|
681
|
+
paramsBytes:
|
|
682
|
+
typeof body.paramsBytes === 'number' &&
|
|
683
|
+
Number.isFinite(body.paramsBytes)
|
|
684
|
+
? body.paramsBytes
|
|
685
|
+
: undefined,
|
|
1079
686
|
retryAttempts:
|
|
1080
687
|
existing?.runId === runId &&
|
|
1081
688
|
typeof existing.retryAttempts === 'number'
|
|
1082
689
|
? existing.retryAttempts
|
|
1083
690
|
: 0,
|
|
1084
691
|
updatedAt: now,
|
|
1085
|
-
expiresAt:
|
|
692
|
+
expiresAt: retryStateExpiresAt,
|
|
1086
693
|
} satisfies WorkflowRunRetryState;
|
|
1087
694
|
const bytes = jsonByteLength(retryState);
|
|
1088
695
|
if (bytes > WORKFLOW_RUN_RETRY_STATE_MAX_BYTES) {
|
|
@@ -1090,11 +697,19 @@ export class PlayDedup implements DurableObject {
|
|
|
1090
697
|
`workflow retry state too large: ${bytes} bytes exceeds ${WORKFLOW_RUN_RETRY_STATE_MAX_BYTES}`,
|
|
1091
698
|
);
|
|
1092
699
|
}
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
headers: { 'content-type': 'application/json' },
|
|
700
|
+
retryStateExpiresAt = retryState.expiresAt;
|
|
701
|
+
await this.state.storage.put(retryKey, retryState);
|
|
702
|
+
await this.state.storage.put(WORKFLOW_DB_SESSIONS_KEY, dbSessionsState);
|
|
1097
703
|
});
|
|
704
|
+
return new Response(
|
|
705
|
+
JSON.stringify({
|
|
706
|
+
ok: true,
|
|
707
|
+
sessionCount: sessions.length,
|
|
708
|
+
retryExpiresAt: retryStateExpiresAt,
|
|
709
|
+
dbSessionsExpiresAt: dbSessionsState.expiresAt,
|
|
710
|
+
}),
|
|
711
|
+
{ headers: { 'content-type': 'application/json' } },
|
|
712
|
+
);
|
|
1098
713
|
}
|
|
1099
714
|
|
|
1100
715
|
private async handleRunRetryClaim(req: Request): Promise<Response> {
|
|
@@ -1128,6 +743,8 @@ export class PlayDedup implements DurableObject {
|
|
|
1128
743
|
claimed: false,
|
|
1129
744
|
attempts: existing.retryAttempts,
|
|
1130
745
|
params: existing.params,
|
|
746
|
+
paramsRef: existing.paramsRef ?? null,
|
|
747
|
+
paramsBytes: existing.paramsBytes ?? null,
|
|
1131
748
|
};
|
|
1132
749
|
return;
|
|
1133
750
|
}
|
|
@@ -1136,12 +753,14 @@ export class PlayDedup implements DurableObject {
|
|
|
1136
753
|
...existing,
|
|
1137
754
|
retryAttempts: nextAttempts,
|
|
1138
755
|
updatedAt: now,
|
|
1139
|
-
expiresAt: now +
|
|
756
|
+
expiresAt: now + WORKFLOW_RUN_STATE_TTL_MS,
|
|
1140
757
|
} satisfies WorkflowRunRetryState);
|
|
1141
758
|
response = {
|
|
1142
759
|
claimed: true,
|
|
1143
760
|
attempts: nextAttempts,
|
|
1144
761
|
params: existing.params,
|
|
762
|
+
paramsRef: existing.paramsRef ?? null,
|
|
763
|
+
paramsBytes: existing.paramsBytes ?? null,
|
|
1145
764
|
};
|
|
1146
765
|
});
|
|
1147
766
|
return new Response(JSON.stringify(response), {
|
|
@@ -1149,6 +768,59 @@ export class PlayDedup implements DurableObject {
|
|
|
1149
768
|
});
|
|
1150
769
|
}
|
|
1151
770
|
|
|
771
|
+
private async handleWorkflowInstancePut(req: Request): Promise<Response> {
|
|
772
|
+
const body = (await req.json().catch(() => null)) as {
|
|
773
|
+
runId?: unknown;
|
|
774
|
+
instanceId?: unknown;
|
|
775
|
+
ttlMs?: unknown;
|
|
776
|
+
} | null;
|
|
777
|
+
const runId = typeof body?.runId === 'string' ? body.runId : '';
|
|
778
|
+
const instanceId =
|
|
779
|
+
typeof body?.instanceId === 'string' ? body.instanceId : '';
|
|
780
|
+
if (!runId || !instanceId) {
|
|
781
|
+
return new Response('runId and instanceId are required', { status: 400 });
|
|
782
|
+
}
|
|
783
|
+
const now = Date.now();
|
|
784
|
+
const ttlMs =
|
|
785
|
+
typeof body?.ttlMs === 'number' &&
|
|
786
|
+
Number.isFinite(body.ttlMs) &&
|
|
787
|
+
body.ttlMs > 0
|
|
788
|
+
? Math.max(60_000, Math.min(body.ttlMs, WORKFLOW_RUN_STATE_TTL_MS))
|
|
789
|
+
: WORKFLOW_RUN_STATE_TTL_MS;
|
|
790
|
+
await this.state.storage.put('workflow-instance', {
|
|
791
|
+
runId,
|
|
792
|
+
instanceId,
|
|
793
|
+
updatedAt: now,
|
|
794
|
+
expiresAt: now + ttlMs,
|
|
795
|
+
} satisfies WorkflowInstanceState);
|
|
796
|
+
return new Response(JSON.stringify({ ok: true }), {
|
|
797
|
+
headers: { 'content-type': 'application/json' },
|
|
798
|
+
});
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
private async handleWorkflowInstanceGet(req: Request): Promise<Response> {
|
|
802
|
+
const runId = new URL(req.url).searchParams.get('runId') ?? '';
|
|
803
|
+
if (!runId) {
|
|
804
|
+
return new Response('runId is required', { status: 400 });
|
|
805
|
+
}
|
|
806
|
+
const state =
|
|
807
|
+
await this.state.storage.get<WorkflowInstanceState>('workflow-instance');
|
|
808
|
+
if (!state || state.runId !== runId) {
|
|
809
|
+
return new Response(JSON.stringify({ instanceId: null }), {
|
|
810
|
+
headers: { 'content-type': 'application/json' },
|
|
811
|
+
});
|
|
812
|
+
}
|
|
813
|
+
if (state.expiresAt <= Date.now()) {
|
|
814
|
+
await this.state.storage.delete('workflow-instance');
|
|
815
|
+
return new Response(JSON.stringify({ instanceId: null }), {
|
|
816
|
+
headers: { 'content-type': 'application/json' },
|
|
817
|
+
});
|
|
818
|
+
}
|
|
819
|
+
return new Response(JSON.stringify({ instanceId: state.instanceId }), {
|
|
820
|
+
headers: { 'content-type': 'application/json' },
|
|
821
|
+
});
|
|
822
|
+
}
|
|
823
|
+
|
|
1152
824
|
private async handleDbSessionsPut(req: Request): Promise<Response> {
|
|
1153
825
|
const body = (await req.json().catch(() => null)) as {
|
|
1154
826
|
runId?: unknown;
|
|
@@ -1220,36 +892,6 @@ export class PlayDedup implements DurableObject {
|
|
|
1220
892
|
);
|
|
1221
893
|
}
|
|
1222
894
|
|
|
1223
|
-
private async handlePoolClear(req: Request): Promise<Response> {
|
|
1224
|
-
const version = this.workflowPoolVersion(req);
|
|
1225
|
-
const [pool, mappings, retries] = await Promise.all([
|
|
1226
|
-
this.state.storage.list<WorkflowPoolEntry>({
|
|
1227
|
-
prefix: WORKFLOW_POOL_KEY_PREFIX,
|
|
1228
|
-
}),
|
|
1229
|
-
this.state.storage.list<WorkflowRunMapping>({
|
|
1230
|
-
prefix: WORKFLOW_POOL_RUN_KEY_PREFIX,
|
|
1231
|
-
}),
|
|
1232
|
-
this.state.storage.list<WorkflowRunRetryState>({
|
|
1233
|
-
prefix: WORKFLOW_RUN_RETRY_KEY_PREFIX,
|
|
1234
|
-
}),
|
|
1235
|
-
]);
|
|
1236
|
-
const keys = [
|
|
1237
|
-
...[...pool.entries()]
|
|
1238
|
-
.filter(([, entry]) => !version || entry.version === version)
|
|
1239
|
-
.map(([key]) => key),
|
|
1240
|
-
...[...mappings.entries()]
|
|
1241
|
-
.filter(([, entry]) => !version || entry.version === version)
|
|
1242
|
-
.map(([key]) => key),
|
|
1243
|
-
...[...retries.keys()],
|
|
1244
|
-
];
|
|
1245
|
-
if (keys.length > 0) {
|
|
1246
|
-
await this.state.storage.delete(keys);
|
|
1247
|
-
}
|
|
1248
|
-
return new Response(JSON.stringify({ deleted: keys.length }), {
|
|
1249
|
-
headers: { 'content-type': 'application/json' },
|
|
1250
|
-
});
|
|
1251
|
-
}
|
|
1252
|
-
|
|
1253
895
|
private async handleTraceAdd(req: Request): Promise<Response> {
|
|
1254
896
|
const body = (await req
|
|
1255
897
|
.json()
|
|
@@ -1365,6 +1007,226 @@ export class PlayDedup implements DurableObject {
|
|
|
1365
1007
|
});
|
|
1366
1008
|
}
|
|
1367
1009
|
|
|
1010
|
+
private async handleChildTerminalSet(req: Request): Promise<Response> {
|
|
1011
|
+
const body = (await req.json().catch(() => null)) as {
|
|
1012
|
+
eventKey?: unknown;
|
|
1013
|
+
data?: unknown;
|
|
1014
|
+
storedAt?: unknown;
|
|
1015
|
+
} | null;
|
|
1016
|
+
const eventKey =
|
|
1017
|
+
typeof body?.eventKey === 'string' && body.eventKey.trim()
|
|
1018
|
+
? body.eventKey.trim()
|
|
1019
|
+
: '';
|
|
1020
|
+
if (!eventKey) {
|
|
1021
|
+
return new Response('eventKey is required', { status: 400 });
|
|
1022
|
+
}
|
|
1023
|
+
const state: CoordinatorChildTerminalState = {
|
|
1024
|
+
eventKey,
|
|
1025
|
+
data: body?.data ?? null,
|
|
1026
|
+
storedAt:
|
|
1027
|
+
typeof body?.storedAt === 'number' && Number.isFinite(body.storedAt)
|
|
1028
|
+
? body.storedAt
|
|
1029
|
+
: Date.now(),
|
|
1030
|
+
};
|
|
1031
|
+
await this.state.storage.put(this.childTerminalKey(eventKey), state);
|
|
1032
|
+
this.wakeChildTerminalWaiters(eventKey);
|
|
1033
|
+
return new Response('{}', {
|
|
1034
|
+
headers: { 'content-type': 'application/json' },
|
|
1035
|
+
});
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
private async handleChildTerminalGet(req: Request): Promise<Response> {
|
|
1039
|
+
const eventKey = new URL(req.url).searchParams.get('eventKey')?.trim();
|
|
1040
|
+
if (!eventKey) {
|
|
1041
|
+
return new Response('eventKey is required', { status: 400 });
|
|
1042
|
+
}
|
|
1043
|
+
const state = await this.readChildTerminalState(eventKey);
|
|
1044
|
+
return new Response(JSON.stringify({ state: state ?? null }), {
|
|
1045
|
+
headers: { 'content-type': 'application/json' },
|
|
1046
|
+
});
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
private async handleChildTerminalAwait(req: Request): Promise<Response> {
|
|
1050
|
+
const url = new URL(req.url);
|
|
1051
|
+
const eventKey = url.searchParams.get('eventKey')?.trim();
|
|
1052
|
+
if (!eventKey) {
|
|
1053
|
+
return new Response('eventKey is required', { status: 400 });
|
|
1054
|
+
}
|
|
1055
|
+
const timeoutMs = Math.min(
|
|
1056
|
+
Math.max(Number(url.searchParams.get('timeoutMs') ?? '0'), 0),
|
|
1057
|
+
30_000,
|
|
1058
|
+
);
|
|
1059
|
+
let state = await this.readChildTerminalState(eventKey);
|
|
1060
|
+
if (!state && timeoutMs > 0) {
|
|
1061
|
+
await new Promise<void>((resolve) => {
|
|
1062
|
+
let settled = false;
|
|
1063
|
+
let timeout: ReturnType<typeof setTimeout>;
|
|
1064
|
+
const finish = () => {
|
|
1065
|
+
if (settled) return;
|
|
1066
|
+
settled = true;
|
|
1067
|
+
this.childTerminalWaiters.get(eventKey)?.delete(finish);
|
|
1068
|
+
if (this.childTerminalWaiters.get(eventKey)?.size === 0) {
|
|
1069
|
+
this.childTerminalWaiters.delete(eventKey);
|
|
1070
|
+
}
|
|
1071
|
+
clearTimeout(timeout);
|
|
1072
|
+
resolve();
|
|
1073
|
+
};
|
|
1074
|
+
timeout = setTimeout(finish, timeoutMs);
|
|
1075
|
+
if (!this.childTerminalWaiters.has(eventKey)) {
|
|
1076
|
+
this.childTerminalWaiters.set(eventKey, new Set());
|
|
1077
|
+
}
|
|
1078
|
+
this.childTerminalWaiters.get(eventKey)!.add(finish);
|
|
1079
|
+
});
|
|
1080
|
+
state = await this.readChildTerminalState(eventKey);
|
|
1081
|
+
}
|
|
1082
|
+
return new Response(JSON.stringify({ state: state ?? null }), {
|
|
1083
|
+
headers: { 'content-type': 'application/json' },
|
|
1084
|
+
});
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
/**
|
|
1088
|
+
* Lease up to `requested` request-window permits for `bucketId` under all
|
|
1089
|
+
* `rules`. Single-threaded DO access means this is the same sliding-window
|
|
1090
|
+
* math as InMemoryRateStateBackend.acquire, generalized to grant a BLOCK of
|
|
1091
|
+
* permits at once so the runtime amortizes the round-trip across many calls.
|
|
1092
|
+
*
|
|
1093
|
+
* Concurrency rules (`maxConcurrency`) are NOT enforced here: simultaneous
|
|
1094
|
+
* in-flight tracking across isolates needs a reliable release signal, which a
|
|
1095
|
+
* dying isolate cannot guarantee. The Governor's global tool-concurrency
|
|
1096
|
+
* semaphore is the cross-call concurrency backstop; this DO owns the
|
|
1097
|
+
* cross-isolate REQUEST RATE, which is the throughput governor.
|
|
1098
|
+
*/
|
|
1099
|
+
private async handleRateAcquire(req: Request): Promise<Response> {
|
|
1100
|
+
const body = (await req
|
|
1101
|
+
.json()
|
|
1102
|
+
.catch(() => null)) as RateAcquireRequest | null;
|
|
1103
|
+
if (
|
|
1104
|
+
!body ||
|
|
1105
|
+
typeof body.bucketId !== 'string' ||
|
|
1106
|
+
!body.bucketId.trim() ||
|
|
1107
|
+
!Array.isArray(body.rules)
|
|
1108
|
+
) {
|
|
1109
|
+
return new Response('bucketId and rules are required', { status: 400 });
|
|
1110
|
+
}
|
|
1111
|
+
const requested =
|
|
1112
|
+
typeof body.requested === 'number' && Number.isFinite(body.requested)
|
|
1113
|
+
? Math.max(1, Math.floor(body.requested))
|
|
1114
|
+
: 1;
|
|
1115
|
+
const result = this.computeRateAcquire(
|
|
1116
|
+
body.bucketId,
|
|
1117
|
+
body.rules,
|
|
1118
|
+
requested,
|
|
1119
|
+
);
|
|
1120
|
+
return new Response(JSON.stringify(result), {
|
|
1121
|
+
headers: { 'content-type': 'application/json' },
|
|
1122
|
+
});
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
private computeRateAcquire(
|
|
1126
|
+
bucketId: string,
|
|
1127
|
+
rules: RateRule[],
|
|
1128
|
+
requested: number,
|
|
1129
|
+
): RateAcquireResponse {
|
|
1130
|
+
const now = Date.now();
|
|
1131
|
+
const usableRules = rules.filter(
|
|
1132
|
+
(rule) =>
|
|
1133
|
+
rule &&
|
|
1134
|
+
typeof rule.ruleId === 'string' &&
|
|
1135
|
+
typeof rule.requestsPerWindow === 'number' &&
|
|
1136
|
+
rule.requestsPerWindow > 0 &&
|
|
1137
|
+
typeof rule.windowMs === 'number',
|
|
1138
|
+
);
|
|
1139
|
+
if (usableRules.length === 0) {
|
|
1140
|
+
return { granted: requested, waitMs: 0 };
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
let waitMs = 0;
|
|
1144
|
+
const cooldownUntil = this.rateCooldownUntil.get(bucketId) ?? 0;
|
|
1145
|
+
if (cooldownUntil > now) waitMs = Math.max(waitMs, cooldownUntil - now);
|
|
1146
|
+
|
|
1147
|
+
// The grant is the min remaining capacity across every rule's window. A
|
|
1148
|
+
// single round-trip can never debit more than the tightest rule allows.
|
|
1149
|
+
let grantable = requested;
|
|
1150
|
+
for (const rule of usableRules) {
|
|
1151
|
+
const state = this.getRateWindowState(bucketId, rule, now);
|
|
1152
|
+
this.resetExpiredRateWindow(state, rule.windowMs, now);
|
|
1153
|
+
const remaining = rule.requestsPerWindow - state.startedInWindow;
|
|
1154
|
+
grantable = Math.min(grantable, Math.max(0, remaining));
|
|
1155
|
+
if (remaining <= 0) {
|
|
1156
|
+
waitMs = Math.max(
|
|
1157
|
+
waitMs,
|
|
1158
|
+
rule.windowMs > 0
|
|
1159
|
+
? state.windowStartedAt + rule.windowMs - now
|
|
1160
|
+
: RATE_MIN_WAIT_MS,
|
|
1161
|
+
);
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
if (waitMs > 0 || grantable <= 0) {
|
|
1166
|
+
return { granted: 0, waitMs: Math.max(RATE_MIN_WAIT_MS, waitMs) };
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
for (const rule of usableRules) {
|
|
1170
|
+
const state = this.getRateWindowState(bucketId, rule, now);
|
|
1171
|
+
state.startedInWindow += grantable;
|
|
1172
|
+
}
|
|
1173
|
+
return { granted: grantable, waitMs: 0 };
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
private async handleRatePenalize(req: Request): Promise<Response> {
|
|
1177
|
+
const body = (await req
|
|
1178
|
+
.json()
|
|
1179
|
+
.catch(() => null)) as RatePenalizeRequest | null;
|
|
1180
|
+
if (!body || typeof body.bucketId !== 'string' || !body.bucketId.trim()) {
|
|
1181
|
+
return new Response('bucketId is required', { status: 400 });
|
|
1182
|
+
}
|
|
1183
|
+
const cooldownMs =
|
|
1184
|
+
typeof body.cooldownMs === 'number' && Number.isFinite(body.cooldownMs)
|
|
1185
|
+
? Math.floor(body.cooldownMs)
|
|
1186
|
+
: 0;
|
|
1187
|
+
if (cooldownMs > 0) {
|
|
1188
|
+
const until = Date.now() + cooldownMs;
|
|
1189
|
+
const existing = this.rateCooldownUntil.get(body.bucketId) ?? 0;
|
|
1190
|
+
this.rateCooldownUntil.set(body.bucketId, Math.max(existing, until));
|
|
1191
|
+
}
|
|
1192
|
+
return new Response('{}', {
|
|
1193
|
+
headers: { 'content-type': 'application/json' },
|
|
1194
|
+
});
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
private getRateWindowState(
|
|
1198
|
+
bucketId: string,
|
|
1199
|
+
rule: RateRule,
|
|
1200
|
+
now: number,
|
|
1201
|
+
): RateRuleWindowState {
|
|
1202
|
+
const key = RATE_STATE_KEY(bucketId, rule.ruleId);
|
|
1203
|
+
const existing = this.rateWindows.get(key);
|
|
1204
|
+
if (existing) return existing;
|
|
1205
|
+
const created: RateRuleWindowState = {
|
|
1206
|
+
windowStartedAt: now,
|
|
1207
|
+
startedInWindow: 0,
|
|
1208
|
+
};
|
|
1209
|
+
this.rateWindows.set(key, created);
|
|
1210
|
+
return created;
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
private resetExpiredRateWindow(
|
|
1214
|
+
state: RateRuleWindowState,
|
|
1215
|
+
windowMs: number,
|
|
1216
|
+
now: number,
|
|
1217
|
+
): void {
|
|
1218
|
+
if (windowMs <= 0) {
|
|
1219
|
+
state.windowStartedAt = now;
|
|
1220
|
+
state.startedInWindow = 0;
|
|
1221
|
+
return;
|
|
1222
|
+
}
|
|
1223
|
+
if (now - state.windowStartedAt < windowMs) return;
|
|
1224
|
+
const elapsed = Math.floor((now - state.windowStartedAt) / windowMs);
|
|
1225
|
+
state.windowStartedAt += elapsed * windowMs;
|
|
1226
|
+
state.startedInWindow = 0;
|
|
1227
|
+
if (now - state.windowStartedAt >= windowMs) state.windowStartedAt = now;
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1368
1230
|
private async storeRunEvent(
|
|
1369
1231
|
input: CoordinatorRunEventInput,
|
|
1370
1232
|
): Promise<CoordinatorRunEvent> {
|
|
@@ -1380,12 +1242,9 @@ export class PlayDedup implements DurableObject {
|
|
|
1380
1242
|
[COORDINATOR_RUN_EVENT_SEQUENCE_KEY]: seq,
|
|
1381
1243
|
[this.runEventKey(event)]: event,
|
|
1382
1244
|
});
|
|
1383
|
-
const
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
const overflow = entries.size - COORDINATOR_RUN_EVENT_MAX_ENTRIES;
|
|
1387
|
-
if (overflow > 0) {
|
|
1388
|
-
await this.state.storage.delete([...entries.keys()].slice(0, overflow));
|
|
1245
|
+
const pruneSeq = seq - COORDINATOR_RUN_EVENT_MAX_ENTRIES;
|
|
1246
|
+
if (pruneSeq > 0) {
|
|
1247
|
+
await this.state.storage.delete(this.runEventKeyForSeq(pruneSeq));
|
|
1389
1248
|
}
|
|
1390
1249
|
});
|
|
1391
1250
|
if (!event) {
|
|
@@ -1482,15 +1341,26 @@ export class PlayDedup implements DurableObject {
|
|
|
1482
1341
|
Math.max(Number(url.searchParams.get('timeoutMs') ?? '0'), 0),
|
|
1483
1342
|
30_000,
|
|
1484
1343
|
);
|
|
1485
|
-
const
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1344
|
+
const readLatestSeq = async () =>
|
|
1345
|
+
(await this.state.storage.get<number>(
|
|
1346
|
+
COORDINATOR_RUN_EVENT_SEQUENCE_KEY,
|
|
1347
|
+
)) ?? afterSeq;
|
|
1348
|
+
const readEvents = async (latestSeq: number) => {
|
|
1349
|
+
const firstAvailableSeq = Math.max(
|
|
1350
|
+
1,
|
|
1351
|
+
latestSeq - COORDINATOR_RUN_EVENT_MAX_ENTRIES + 1,
|
|
1352
|
+
);
|
|
1353
|
+
const startSeq = Math.max(afterSeq + 1, firstAvailableSeq);
|
|
1354
|
+
if (latestSeq < startSeq) return [];
|
|
1355
|
+
const keys: string[] = [];
|
|
1356
|
+
for (let seq = startSeq; seq <= latestSeq; seq += 1) {
|
|
1357
|
+
keys.push(this.runEventKeyForSeq(seq));
|
|
1358
|
+
}
|
|
1359
|
+
const entries = await this.state.storage.get<CoordinatorRunEvent>(keys);
|
|
1360
|
+
return [...entries.values()].sort((left, right) => left.seq - right.seq);
|
|
1492
1361
|
};
|
|
1493
|
-
let
|
|
1362
|
+
let latestSeq = await readLatestSeq();
|
|
1363
|
+
let events = await readEvents(latestSeq);
|
|
1494
1364
|
if (events.length === 0 && timeoutMs > 0) {
|
|
1495
1365
|
await new Promise<void>((resolve) => {
|
|
1496
1366
|
let settled = false;
|
|
@@ -1505,12 +1375,9 @@ export class PlayDedup implements DurableObject {
|
|
|
1505
1375
|
const timeout = setTimeout(finish, timeoutMs);
|
|
1506
1376
|
this.runEventWaiters.add(onEvent);
|
|
1507
1377
|
});
|
|
1508
|
-
|
|
1378
|
+
latestSeq = await readLatestSeq();
|
|
1379
|
+
events = await readEvents(latestSeq);
|
|
1509
1380
|
}
|
|
1510
|
-
const latestSeq =
|
|
1511
|
-
(await this.state.storage.get<number>(
|
|
1512
|
-
COORDINATOR_RUN_EVENT_SEQUENCE_KEY,
|
|
1513
|
-
)) ?? afterSeq;
|
|
1514
1381
|
return new Response(JSON.stringify({ events, latestSeq }), {
|
|
1515
1382
|
headers: { 'content-type': 'application/json' },
|
|
1516
1383
|
});
|