deepline 0.2.55 → 0.2.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +14 -0
  2. package/dist/bundling-sources/sdk/src/http.ts +19 -1
  3. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  4. package/dist/bundling-sources/sdk/src/stream-reconnect.ts +6 -0
  5. package/dist/bundling-sources/sdk/src/types.ts +7 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +146 -12
  7. package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +6 -3
  8. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1631 -748
  9. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +20 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +81 -52
  11. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +146 -6
  12. package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +56 -22
  13. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +14 -11
  14. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +21 -2
  15. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +81 -21
  16. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +7 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +21 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +27 -2
  20. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +49 -10
  21. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +96 -3
  22. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +341 -67
  23. package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +10 -0
  24. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +8 -1
  25. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +17 -2
  26. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver.ts +4 -1
  27. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +17 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +16 -2
  29. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +25 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +3 -0
  31. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +2 -0
  32. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +80 -1
  33. package/dist/bundling-sources/shared_libs/plays/docflow.ts +113 -14
  34. package/dist/bundling-sources/shared_libs/plays/play-exports.ts +53 -4
  35. package/dist/bundling-sources/shared_libs/security/safe-fetch.ts +9 -0
  36. package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +85 -48
  37. package/dist/cli/index.js +429 -54
  38. package/dist/cli/index.mjs +409 -28
  39. package/dist/{compiler-manifest-Bl8kmLx9.d.mts → compiler-manifest-TgaC4DeD.d.mts} +13 -0
  40. package/dist/{compiler-manifest-Bl8kmLx9.d.ts → compiler-manifest-TgaC4DeD.d.ts} +13 -0
  41. package/dist/index.d.mts +21 -3
  42. package/dist/index.d.ts +21 -3
  43. package/dist/index.js +29 -2
  44. package/dist/index.mjs +29 -2
  45. package/dist/install-integrity.json +2 -2
  46. package/dist/plays/bundle-play-file.d.mts +2 -2
  47. package/dist/plays/bundle-play-file.d.ts +2 -2
  48. package/dist/plays/bundle-play-file.mjs +78 -18
  49. package/package.json +1 -1
@@ -609,6 +609,12 @@ export interface ContextOptions {
609
609
  fixtureBehavior?: FixtureBehavior | null;
610
610
  /** Preview/dev test seam that applies provider pacing to fixture responses. */
611
611
  enforceFixtureProviderPacing?: boolean;
612
+ /**
613
+ * Server-validated per-run ceiling for concurrently resident provider-tool
614
+ * executions and direct ctx.fetch calls. Omitted uses the platform default.
615
+ */
616
+ maxConcurrentExternalCalls?: number | null;
617
+ maxConcurrentRows?: number | null;
612
618
  orgId?: string;
613
619
  userEmail?: string;
614
620
  playName?: string;
@@ -646,6 +652,18 @@ export interface ContextOptions {
646
652
  * a transport that validates resolved IP addresses at connect time.
647
653
  */
648
654
  fetchImpl?: (input: string | URL, init?: RequestInit) => Promise<Response>;
655
+ /**
656
+ * Internal runtime-policy override used by focused tests and controlled
657
+ * harnesses. Play authors cannot set these values; ctx.fetch always has a
658
+ * bounded platform deadline.
659
+ */
660
+ ctxFetchTimeouts?: {
661
+ headersMs?: number;
662
+ bodyMs?: number;
663
+ totalMs?: number;
664
+ };
665
+ /** Internal low-cardinality map-stall diagnostic interval override. */
666
+ runtimeMapStallLogIntervalMs?: number;
649
667
  /** Called when a row gains new partial data or stage info. */
650
668
  onRowUpdate?: (update: PlayRowUpdate) => void | Promise<void>;
651
669
  /** Structured execution events emitted from explicit dataset scopes. */
@@ -836,6 +854,8 @@ export interface ContextOptions {
836
854
  ) =>
837
855
  | Promise<Array<RuntimeStepReceipt | null>>
838
856
  | Array<RuntimeStepReceipt | null>;
857
+ /** Internal backend capability: a claimed running receipt is the execution fence. */
858
+ runtimeReceiptClaimsEstablishExecutionFence?: boolean;
839
859
  markRuntimeStepReceiptRunning?: (
840
860
  input: MarkRuntimeStepReceiptRunningInput,
841
861
  ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
@@ -28,8 +28,8 @@ import type {
28
28
  ToolExecutionFailureV1,
29
29
  } from '../tool-execution-error';
30
30
 
31
- const DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS = 240;
32
- const DURABLE_RECEIPT_WAIT_DELAY_MS = 250;
31
+ export const DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS = 240;
32
+ export const DURABLE_RECEIPT_WAIT_DELAY_MS = 250;
33
33
  const TOOL_RECEIPT_DEFAULT_WAIT_MS = 300_000;
34
34
  const TOOL_RECEIPT_COMPLETION_BUFFER_MS = 30_000;
35
35
  const TOOL_RECEIPT_MAX_WAIT_MS = 30 * 60_000;
@@ -358,15 +358,7 @@ export async function waitForCompletedRuntimeReceipts(input: {
358
358
  if (attempt > 0) {
359
359
  await sleepReceiptWait(delayMs, input.abortSignal);
360
360
  }
361
- let receipts: Map<string, RuntimeStepReceipt>;
362
- try {
363
- receipts = await input.store.getMany([...pending]);
364
- } catch (error) {
365
- if (error instanceof RuntimeReceiptWaitTimeoutError) {
366
- break;
367
- }
368
- throw error;
369
- }
361
+ const receipts = await input.store.getMany([...pending]);
370
362
  const statuses = [...receipts.values()].reduce<Record<string, number>>(
371
363
  (counts, receipt) => {
372
364
  counts[receipt.status] = (counts[receipt.status] ?? 0) + 1;
@@ -433,6 +425,9 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
433
425
  /** New receipt-cache path: read completed facts, execute without creating an
434
426
  * in-flight receipt, then atomically insert the completed fact. */
435
427
  completedCacheOnly?: boolean;
428
+ withCompletedReceiptHydration?: <Result>(
429
+ hydrate: () => Promise<Result>,
430
+ ) => Promise<Result>;
436
431
  requiresExecutionLock?: boolean;
437
432
  executionLockTtlMs?: number;
438
433
  toolErrorSchemaVersion: ToolExecutionErrorSchemaVersion;
@@ -480,11 +475,33 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
480
475
  };
481
476
 
482
477
  if (input.completedCacheOnly === true) {
478
+ const withCompletedReceiptHydration = async <Result>(
479
+ hydrate: () => Promise<Result>,
480
+ ): Promise<Result> =>
481
+ input.withCompletedReceiptHydration
482
+ ? await input.withCompletedReceiptHydration(hydrate)
483
+ : await hydrate();
484
+ const inspectCompletedReceipt = async (
485
+ read: () => Promise<RuntimeStepReceipt | null>,
486
+ source: DurableReceiptRecoverySource = 'cache',
487
+ ): Promise<{ kind: 'recovered'; output: T } | { kind: 'unresolved' }> => {
488
+ const inspect = async () => {
489
+ const receipt = await read();
490
+ if (receipt?.status === 'completed' || receipt?.status === 'skipped') {
491
+ return {
492
+ kind: 'recovered' as const,
493
+ output: await recoverCompletedReceipt(receipt, source),
494
+ };
495
+ }
496
+ return { kind: 'unresolved' as const };
497
+ };
498
+ return await withCompletedReceiptHydration(inspect);
499
+ };
483
500
  if (input.force !== true) {
484
- const cached = await input.store.get(input.receiptKey);
485
- if (cached?.status === 'completed' || cached?.status === 'skipped') {
486
- return await recoverCompletedReceipt(cached);
487
- }
501
+ const cached = await inspectCompletedReceipt(() =>
502
+ input.store.get(input.receiptKey),
503
+ );
504
+ if (cached.kind === 'recovered') return cached.output;
488
505
  }
489
506
  const needsLock = input.requiresExecutionLock === true;
490
507
  const ownerExecutionId = `${input.runId}:${crypto.randomUUID()}`;
@@ -511,28 +528,30 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
511
528
  });
512
529
  ownsLock = lock?.ownerExecutionId === ownerExecutionId;
513
530
  if (ownsLock) break;
514
- const winner = await input.store.get(input.receiptKey);
515
- if (
516
- input.force !== true &&
517
- (winner?.status === 'completed' || winner?.status === 'skipped')
518
- ) {
519
- return await recoverCompletedReceipt(winner, 'in_flight');
531
+ if (input.force !== true) {
532
+ const winner = await inspectCompletedReceipt(
533
+ () => input.store.get(input.receiptKey),
534
+ 'in_flight',
535
+ );
536
+ if (winner.kind === 'recovered') return winner.output;
520
537
  }
521
538
  if (Date.now() >= deadline) {
522
539
  throw new RuntimeReceiptWaitTimeoutError(input.receiptKey);
523
540
  }
524
541
  await sleepReceiptWait(DURABLE_RECEIPT_WAIT_DELAY_MS);
525
542
  }
526
- const afterLock = await input.store.get(input.receiptKey);
527
- if (
528
- input.force !== true &&
529
- (afterLock?.status === 'completed' || afterLock?.status === 'skipped')
530
- ) {
531
- await input.store.releaseExecutionLock({
532
- receiptKey: input.receiptKey,
533
- ownerExecutionId,
534
- });
535
- return await recoverCompletedReceipt(afterLock, 'in_flight');
543
+ if (input.force !== true) {
544
+ const afterLock = await inspectCompletedReceipt(
545
+ () => input.store.get(input.receiptKey),
546
+ 'in_flight',
547
+ );
548
+ if (afterLock.kind === 'recovered') {
549
+ await input.store.releaseExecutionLock({
550
+ receiptKey: input.receiptKey,
551
+ ownerExecutionId,
552
+ });
553
+ return afterLock.output;
554
+ }
536
555
  }
537
556
  const lockTtlMs =
538
557
  input.executionLockTtlMs ?? TOOL_RECEIPT_DEFAULT_WAIT_MS;
@@ -597,28 +616,38 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
597
616
  if (input.force === true) {
598
617
  return result;
599
618
  }
600
- const completed = await input.store.complete(
601
- input.receiptKey,
602
- input.runId,
603
- isToolExecuteResult(result)
604
- ? serializeToolExecuteResult(result)
605
- : result,
606
- COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
619
+ // Completion normally returns a compact acknowledgement. Keep the
620
+ // mutation and response decode inside the hydration turn anyway because
621
+ // a concurrent winner may return a full persisted payload.
622
+ const published = await withCompletedReceiptHydration(async () => {
623
+ const completed = await input.store.complete(
624
+ input.receiptKey,
625
+ input.runId,
626
+ isToolExecuteResult(result)
627
+ ? serializeToolExecuteResult(result)
628
+ : result,
629
+ COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
630
+ );
631
+ if (
632
+ completed?.status === 'completed' ||
633
+ completed?.status === 'skipped'
634
+ ) {
635
+ return {
636
+ kind: 'completed' as const,
637
+ output:
638
+ completed.output === undefined
639
+ ? result
640
+ : await recoverCompletedReceipt(completed, 'owner'),
641
+ };
642
+ }
643
+ return { kind: 'unresolved' as const };
644
+ });
645
+ if (published.kind === 'completed') return published.output;
646
+ const winner = await inspectCompletedReceipt(
647
+ () => input.store.get(input.receiptKey),
648
+ 'owner',
607
649
  );
608
- if (
609
- completed?.status === 'completed' ||
610
- completed?.status === 'skipped'
611
- ) {
612
- // Completion is a compact acknowledgement. This execution already
613
- // owns the just-produced value; only a later receipt read needs the
614
- // persisted payload for replay.
615
- if (completed.output === undefined) return result;
616
- return await recoverCompletedReceipt(completed, 'owner');
617
- }
618
- const winner = await input.store.get(input.receiptKey);
619
- if (winner?.status === 'completed' || winner?.status === 'skipped') {
620
- return await recoverCompletedReceipt(winner, 'owner');
621
- }
650
+ if (winner.kind === 'recovered') return winner.output;
622
651
  throw new Error(
623
652
  `ctx.${input.operation}(${input.id}): completed receipt cache insert failed.`,
624
653
  );
@@ -1,5 +1,6 @@
1
1
  export const FIXTURE_BEHAVIOR_VERSION = 1 as const;
2
2
  export const FIXTURE_BEHAVIOR_RESPONSE_VERSION = 2 as const;
3
+ export const FIXTURE_BEHAVIOR_REPLAY_VERSION = 3 as const;
3
4
  export const MAX_FIXTURE_RESPONSE_DELAY_SAMPLES = 256;
4
5
  export const MAX_FIXTURE_RESPONSE_DELAY_MS = 8 * 60_000;
5
6
  export const MAX_FIXTURE_RESPONSE_ERROR_MESSAGE_LENGTH = 500;
@@ -26,7 +27,22 @@ export type FixtureBehaviorV2 = {
26
27
  responseSamples: FixtureResponseSample[];
27
28
  };
28
29
 
29
- export type FixtureBehavior = FixtureBehaviorV1 | FixtureBehaviorV2;
30
+ export type FixtureReplayBundle = {
31
+ bundleId: string;
32
+ manifestSha256: string;
33
+ syntheticFallbackToolIds?: string[];
34
+ };
35
+
36
+ export type FixtureBehaviorV3 = {
37
+ version: typeof FIXTURE_BEHAVIOR_REPLAY_VERSION;
38
+ responseSamples: FixtureResponseSample[];
39
+ replayBundle: FixtureReplayBundle;
40
+ };
41
+
42
+ export type FixtureBehavior =
43
+ | FixtureBehaviorV1
44
+ | FixtureBehaviorV2
45
+ | FixtureBehaviorV3;
30
46
 
31
47
  /** Providers whose fixture-mode path intentionally remains a real local/test adapter. */
32
48
  const FIXTURE_MODE_BYPASS_PROVIDERS = new Set([
@@ -53,8 +69,15 @@ export function validateFixtureBehavior(
53
69
  }
54
70
  const record = value as Record<string, unknown>;
55
71
  const supportedKeys =
56
- record.version === FIXTURE_BEHAVIOR_RESPONSE_VERSION
57
- ? new Set(['version', 'responseSamples'])
72
+ record.version === FIXTURE_BEHAVIOR_RESPONSE_VERSION ||
73
+ record.version === FIXTURE_BEHAVIOR_REPLAY_VERSION
74
+ ? new Set([
75
+ 'version',
76
+ 'responseSamples',
77
+ ...(record.version === FIXTURE_BEHAVIOR_REPLAY_VERSION
78
+ ? ['replayBundle']
79
+ : []),
80
+ ])
58
81
  : new Set(['version', 'responseDelaySamplesMs']);
59
82
  const unknownKeys = Object.keys(record).filter(
60
83
  (key) => !supportedKeys.has(key),
@@ -67,16 +90,21 @@ export function validateFixtureBehavior(
67
90
  }
68
91
  if (
69
92
  record.version !== FIXTURE_BEHAVIOR_VERSION &&
70
- record.version !== FIXTURE_BEHAVIOR_RESPONSE_VERSION
93
+ record.version !== FIXTURE_BEHAVIOR_RESPONSE_VERSION &&
94
+ record.version !== FIXTURE_BEHAVIOR_REPLAY_VERSION
71
95
  ) {
72
96
  return {
73
97
  ok: false,
74
98
  error:
75
99
  `fixtureBehavior.version must be ${FIXTURE_BEHAVIOR_VERSION} or ` +
76
- `${FIXTURE_BEHAVIOR_RESPONSE_VERSION}.`,
100
+ `${FIXTURE_BEHAVIOR_RESPONSE_VERSION}, or ` +
101
+ `${FIXTURE_BEHAVIOR_REPLAY_VERSION}.`,
77
102
  };
78
103
  }
79
- if (record.version === FIXTURE_BEHAVIOR_RESPONSE_VERSION) {
104
+ if (
105
+ record.version === FIXTURE_BEHAVIOR_RESPONSE_VERSION ||
106
+ record.version === FIXTURE_BEHAVIOR_REPLAY_VERSION
107
+ ) {
80
108
  if (
81
109
  !Array.isArray(record.responseSamples) ||
82
110
  record.responseSamples.length === 0 ||
@@ -231,6 +259,101 @@ export function validateFixtureBehavior(
231
259
  ...(httpError ? { httpError } : {}),
232
260
  });
233
261
  }
262
+ let replayBundle: FixtureReplayBundle | undefined;
263
+ if (record.version === FIXTURE_BEHAVIOR_REPLAY_VERSION) {
264
+ if (
265
+ !record.replayBundle ||
266
+ typeof record.replayBundle !== 'object' ||
267
+ Array.isArray(record.replayBundle)
268
+ ) {
269
+ return {
270
+ ok: false,
271
+ error: 'fixtureBehavior.replayBundle must be an object.',
272
+ };
273
+ }
274
+ const replay = record.replayBundle as Record<string, unknown>;
275
+ const replayUnknownKeys = Object.keys(replay).filter(
276
+ (key) =>
277
+ key !== 'bundleId' &&
278
+ key !== 'manifestSha256' &&
279
+ key !== 'syntheticFallbackToolIds',
280
+ );
281
+ if (replayUnknownKeys.length > 0) {
282
+ return {
283
+ ok: false,
284
+ error:
285
+ 'Unsupported fixtureBehavior.replayBundle field ' +
286
+ `"${replayUnknownKeys[0]}".`,
287
+ };
288
+ }
289
+ const digestPattern = /^[a-f0-9]{64}$/;
290
+ if (
291
+ typeof replay.bundleId !== 'string' ||
292
+ !digestPattern.test(replay.bundleId)
293
+ ) {
294
+ return {
295
+ ok: false,
296
+ error:
297
+ 'fixtureBehavior.replayBundle.bundleId must be a lowercase SHA-256 digest.',
298
+ };
299
+ }
300
+ if (
301
+ typeof replay.manifestSha256 !== 'string' ||
302
+ !digestPattern.test(replay.manifestSha256)
303
+ ) {
304
+ return {
305
+ ok: false,
306
+ error:
307
+ 'fixtureBehavior.replayBundle.manifestSha256 must be a lowercase SHA-256 digest.',
308
+ };
309
+ }
310
+ let syntheticFallbackToolIds: string[] | undefined;
311
+ if (replay.syntheticFallbackToolIds !== undefined) {
312
+ if (
313
+ !Array.isArray(replay.syntheticFallbackToolIds) ||
314
+ replay.syntheticFallbackToolIds.length === 0 ||
315
+ replay.syntheticFallbackToolIds.length > 32
316
+ ) {
317
+ return {
318
+ ok: false,
319
+ error:
320
+ 'fixtureBehavior.replayBundle.syntheticFallbackToolIds must contain 1-32 tool ids.',
321
+ };
322
+ }
323
+ syntheticFallbackToolIds = [];
324
+ for (const rawToolId of replay.syntheticFallbackToolIds) {
325
+ if (
326
+ typeof rawToolId !== 'string' ||
327
+ rawToolId !== rawToolId.trim().toLowerCase() ||
328
+ !/^[a-z0-9][a-z0-9_.-]*$/.test(rawToolId) ||
329
+ rawToolId.length > 250 ||
330
+ syntheticFallbackToolIds.includes(rawToolId)
331
+ ) {
332
+ return {
333
+ ok: false,
334
+ error:
335
+ 'fixtureBehavior.replayBundle.syntheticFallbackToolIds contains an invalid tool id.',
336
+ };
337
+ }
338
+ syntheticFallbackToolIds.push(rawToolId);
339
+ }
340
+ }
341
+ replayBundle = {
342
+ bundleId: replay.bundleId,
343
+ manifestSha256: replay.manifestSha256,
344
+ ...(syntheticFallbackToolIds ? { syntheticFallbackToolIds } : {}),
345
+ };
346
+ }
347
+ if (record.version === FIXTURE_BEHAVIOR_REPLAY_VERSION) {
348
+ return {
349
+ ok: true,
350
+ behavior: {
351
+ version: FIXTURE_BEHAVIOR_REPLAY_VERSION,
352
+ responseSamples: samples,
353
+ replayBundle: replayBundle!,
354
+ },
355
+ };
356
+ }
234
357
  return {
235
358
  ok: true,
236
359
  behavior: {
@@ -329,6 +452,23 @@ function stableFixtureRequestHash(value: string): number {
329
452
  return hash >>> 0;
330
453
  }
331
454
 
455
+ export function selectFixtureReplaySampleIndex(input: {
456
+ stableRequestKey: string;
457
+ sampleCount: number;
458
+ }): number {
459
+ const stableRequestKey = input.stableRequestKey.trim();
460
+ if (!stableRequestKey) {
461
+ throw new Error('Fixture replay requires a stable request key.');
462
+ }
463
+ if (!Number.isSafeInteger(input.sampleCount) || input.sampleCount <= 0) {
464
+ throw new Error('Fixture replay requires a positive sample count.');
465
+ }
466
+ return (
467
+ stableFixtureRequestHash(`fixture-replay-v1:${stableRequestKey}`) %
468
+ input.sampleCount
469
+ );
470
+ }
471
+
332
472
  export function selectFixtureResponseDelay(input: {
333
473
  behavior: FixtureBehavior;
334
474
  stableRequestKey: string;
@@ -22,6 +22,34 @@ type StoredSession<TAuth, TLaunch> = {
22
22
  runAttempt: number | null;
23
23
  };
24
24
 
25
+ function abortReason(signal: AbortSignal): Error {
26
+ return signal.reason instanceof Error
27
+ ? signal.reason
28
+ : new Error('Gateway session wait was cancelled.');
29
+ }
30
+
31
+ async function awaitWithAbort<T>(
32
+ pending: Promise<T>,
33
+ signal: AbortSignal | undefined,
34
+ ): Promise<T> {
35
+ if (!signal) return await pending;
36
+ if (signal.aborted) throw abortReason(signal);
37
+ return await new Promise<T>((resolve, reject) => {
38
+ const onAbort = () => reject(abortReason(signal));
39
+ signal.addEventListener('abort', onAbort, { once: true });
40
+ void pending.then(
41
+ (value) => {
42
+ signal.removeEventListener('abort', onAbort);
43
+ resolve(value);
44
+ },
45
+ (error) => {
46
+ signal.removeEventListener('abort', onAbort);
47
+ reject(error);
48
+ },
49
+ );
50
+ });
51
+ }
52
+
25
53
  /** Briefly reuse an authenticated immutable launch, bounded by token identity. */
26
54
  export function createGatewayAuthSessionCache<TAuth, TLaunch>(options: {
27
55
  ttlMs: number;
@@ -54,7 +82,10 @@ export function createGatewayAuthSessionCache<TAuth, TLaunch>(options: {
54
82
  token: string;
55
83
  tokenExpiresAt: number;
56
84
  establish: () => Promise<{ auth: TAuth; launch: TLaunch }>;
85
+ /** Detach this waiter without cancelling the shared establishment. */
86
+ signal?: AbortSignal;
57
87
  }): Promise<GatewayAuthSession<TAuth, TLaunch>> {
88
+ if (input.signal?.aborted) throw abortReason(input.signal);
58
89
  const key = fingerprint(input.token);
59
90
  const at = now();
60
91
  const cached = sessions.get(key);
@@ -70,28 +101,31 @@ export function createGatewayAuthSessionCache<TAuth, TLaunch>(options: {
70
101
  if (cached) sessions.delete(key);
71
102
 
72
103
  const coalesced = flights.has(key);
73
- const session = await flights.run(key, async () => {
74
- const established = await input.establish();
75
- const identity = options.identity(established.auth);
76
- if (!identity.runId) {
77
- throw new Error('Gateway authentication produced no run identity.');
78
- }
79
- const establishedAt = now();
80
- const stored: StoredSession<TAuth, TLaunch> = {
81
- ...established,
82
- expiresAt: Math.min(
83
- establishedAt + options.ttlMs,
84
- input.tokenExpiresAt,
85
- ),
86
- runId: identity.runId,
87
- runAttempt: identity.runAttempt,
88
- };
89
- if (stored.expiresAt > establishedAt) {
90
- prune(establishedAt);
91
- sessions.set(key, stored);
92
- }
93
- return stored;
94
- });
104
+ const session = await awaitWithAbort(
105
+ flights.run(key, async () => {
106
+ const established = await input.establish();
107
+ const identity = options.identity(established.auth);
108
+ if (!identity.runId) {
109
+ throw new Error('Gateway authentication produced no run identity.');
110
+ }
111
+ const establishedAt = now();
112
+ const stored: StoredSession<TAuth, TLaunch> = {
113
+ ...established,
114
+ expiresAt: Math.min(
115
+ establishedAt + options.ttlMs,
116
+ input.tokenExpiresAt,
117
+ ),
118
+ runId: identity.runId,
119
+ runAttempt: identity.runAttempt,
120
+ };
121
+ if (stored.expiresAt > establishedAt) {
122
+ prune(establishedAt);
123
+ sessions.set(key, stored);
124
+ }
125
+ return stored;
126
+ }),
127
+ input.signal,
128
+ );
95
129
  return { ...session, source: coalesced ? 'coalesced' : 'miss' };
96
130
  },
97
131
  size: () => sessions.size,
@@ -814,11 +814,11 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
814
814
  }
815
815
 
816
816
  /**
817
- * Multiplier applied to an adaptive pacing rule's `requestsPerWindow` to derive
818
- * its `adaptiveMaxRps` ceiling. The configured rate is a safe seed, while 429
819
- * feedback drives AIMD toward the provider's actual limit.
817
+ * Multiplier applied to an undeclared default-pacing rule's
818
+ * `requestsPerWindow` to derive its `adaptiveMaxRps` ceiling. Declared rules
819
+ * carry either their explicit advisory ceiling or their hard base rate.
820
820
  */
821
- const ADAPTIVE_MAX_RPS_MULTIPLIER = 25;
821
+ const DEFAULT_PACING_ADAPTIVE_MAX_RPS_MULTIPLIER = 25;
822
822
 
823
823
  /**
824
824
  * A rule id shaped `default:<toolId>` is the undeclared-tool default-pacing rule
@@ -829,23 +829,26 @@ function isDefaultPacingRuleId(ruleId: string): boolean {
829
829
  }
830
830
 
831
831
  /**
832
- * Preserve explicit advisory ceilings. Otherwise stamp the base * 25 ceiling
833
- * onto legacy default-pacing rules and declared per-second rules. This lets a
834
- * declared 10 QPS provider discover a higher entitlement, while longer-window
835
- * declared budgets remain fixed because multiplying RPM into RPS is unsafe.
832
+ * Preserve explicit advisory ceilings, stamp the legacy base * 25 ceiling onto
833
+ * undeclared default-pacing rules, and stamp declared rules with their base
834
+ * rate. Sending the hard base explicitly also repairs bucket rows widened by a
835
+ * previous runtime version on their next acquire.
836
836
  */
837
837
  function withAdaptiveMaxRps(rules: readonly PacingRule[]): PacingRule[] {
838
838
  return rules.map((rule) =>
839
839
  rule.adaptiveMaxRps != null
840
840
  ? rule
841
- : isDefaultPacingRuleId(rule.ruleId) || rule.windowMs === 1_000
841
+ : isDefaultPacingRuleId(rule.ruleId)
842
842
  ? ({
843
843
  ...rule,
844
844
  adaptiveMaxRps:
845
845
  rule.requestsPerWindow *
846
- ADAPTIVE_MAX_RPS_MULTIPLIER,
846
+ DEFAULT_PACING_ADAPTIVE_MAX_RPS_MULTIPLIER,
847
847
  } as PacingRule)
848
- : rule,
848
+ : ({
849
+ ...rule,
850
+ adaptiveMaxRps: (rule.requestsPerWindow / rule.windowMs) * 1_000,
851
+ } as PacingRule),
849
852
  );
850
853
  }
851
854
 
@@ -17,6 +17,7 @@ import {
17
17
  type AdapterId,
18
18
  type ResolvedExecutionPolicy,
19
19
  resolveExecutionPolicy,
20
+ resolveMaxConcurrentExternalCalls,
20
21
  resolveRowConcurrency,
21
22
  } from './policy';
22
23
  import { type PacingRule, type RateStateBackend } from './rate-state-backend';
@@ -187,6 +188,8 @@ interface GovernorInput {
187
188
  resolveRateScope?: RateScopeResolver;
188
189
  adaptiveAdmission?: RuntimeAdaptiveAdmission;
189
190
  resume?: GovernanceSnapshot;
191
+ maxConcurrentExternalCalls?: number | null;
192
+ maxConcurrentRows?: number | null;
190
193
  }
191
194
 
192
195
  class Semaphore {
@@ -262,7 +265,23 @@ export function createDefaultGovernanceSnapshot(scope: {
262
265
  export function createPlayExecutionGovernor(
263
266
  input: GovernorInput,
264
267
  ): PlayExecutionGovernor {
265
- const policy = resolveExecutionPolicy(input.adapter);
268
+ const basePolicy = resolveExecutionPolicy(input.adapter);
269
+ const maxConcurrentExternalCalls = resolveMaxConcurrentExternalCalls(
270
+ input.maxConcurrentExternalCalls,
271
+ );
272
+ const maxInFlightDispatchGroups = Math.min(
273
+ maxConcurrentExternalCalls,
274
+ basePolicy.concurrency.toolDispatchGroups,
275
+ );
276
+ const policy: ResolvedExecutionPolicy = {
277
+ ...basePolicy,
278
+ concurrency: {
279
+ ...basePolicy.concurrency,
280
+ toolCalls: maxConcurrentExternalCalls,
281
+ toolDispatchGroups: maxInFlightDispatchGroups,
282
+ toolDispatchGroupsPerLane: maxInFlightDispatchGroups,
283
+ },
284
+ };
266
285
  const state: GovernanceSnapshot =
267
286
  input.resume ?? createDefaultGovernanceSnapshot(input.scope);
268
287
 
@@ -481,7 +500,7 @@ export function createPlayExecutionGovernor(
481
500
  },
482
501
 
483
502
  resolveRowConcurrency: (requested) =>
484
- resolveRowConcurrency(policy, requested),
503
+ resolveRowConcurrency(policy, requested, input.maxConcurrentRows),
485
504
 
486
505
  async reportProviderBackpressure(bp) {
487
506
  // DB-authoritative pacer: route backpressure ONLY to the row via