deepline 0.3.128 → 0.3.129

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +34 -9
  2. package/dist/bundling-sources/sdk/src/http.ts +22 -0
  3. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  4. package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +22 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +265 -2
  6. package/dist/bundling-sources/shared_libs/play-runtime/bounded-batch-function.ts +682 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +21 -5
  8. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +5390 -1732
  9. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +107 -1
  10. package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts +7 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +229 -7
  12. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +25 -4
  13. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-capacity.ts +71 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-drain.ts +1 -1
  15. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-topology.ts +4 -1
  16. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +694 -28
  17. package/dist/bundling-sources/shared_libs/play-runtime/egress-bandwidth-probe.ts +167 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/execute-server-timing.ts +103 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-receipt-output-reference.ts +34 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-response-size.ts +27 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +66 -1
  22. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +425 -71
  23. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +210 -32
  24. package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts +27 -13
  25. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +19 -6
  26. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +61 -2
  27. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +33 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +42 -1
  29. package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +11 -11
  30. package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +49 -16
  31. package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +1653 -98
  32. package/dist/bundling-sources/shared_libs/play-runtime/memory-ledger.ts +807 -0
  33. package/dist/bundling-sources/shared_libs/play-runtime/memory-pressure-signal.ts +138 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +85 -3
  35. package/dist/bundling-sources/shared_libs/play-runtime/persistence-latch.ts +60 -2
  36. package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +46 -1
  37. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +2 -0
  38. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +1 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +1 -0
  40. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +1 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +70 -27
  42. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +56 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/adaptive-provider-capacity-controller.ts +192 -0
  44. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/epoch-aimd-rate-controller.ts +194 -0
  45. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/fixture-capacity-simulator.ts +220 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/gradient2-concurrency-controller.ts +214 -0
  47. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-capacity-policy.ts +43 -0
  48. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-concurrency-policy.ts +30 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-rate-policy.ts +95 -0
  50. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-throttle-classifier.ts +40 -0
  51. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +1535 -11
  52. package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +49 -2
  53. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +6 -1
  54. package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +55 -0
  55. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/egress-probe.ts +221 -0
  56. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +339 -29
  57. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts +28 -1
  58. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +192 -165
  59. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +6 -0
  60. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +90 -5
  61. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +281 -6
  62. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +27 -1
  63. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +19 -0
  64. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +108 -0
  65. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +40 -0
  66. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +11 -5
  67. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +30 -0
  68. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-compression.ts +276 -0
  69. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +723 -115
  70. package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +98 -2
  71. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +3 -2
  72. package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts +41 -4
  73. package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +448 -0
  74. package/dist/bundling-sources/shared_libs/play-runtime/runtime-map-row-feed.ts +215 -0
  75. package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +11 -0
  76. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +161 -14
  77. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-value.ts +184 -0
  78. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +607 -121
  79. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts +4 -0
  80. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +21 -0
  81. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +15 -2
  82. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +71 -14
  83. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +2 -2
  84. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +288 -119
  85. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -6
  86. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +28 -3
  87. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/canonical-integration-operation-store.ts +179 -0
  88. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +80 -2
  89. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +31 -2
  90. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-store-contract.ts +104 -0
  91. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-reference.ts +47 -0
  92. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-retirement.ts +193 -0
  93. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-storage.ts +420 -0
  94. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +41 -0
  95. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +134 -3
  96. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +1657 -95
  97. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +885 -73
  98. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-completion-coalescer.ts +452 -0
  99. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +601 -42
  100. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +1 -0
  101. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +371 -0
  102. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +86 -7
  103. package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +153 -22
  104. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +1 -0
  105. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +188 -38
  106. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +28 -13
  107. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +44 -1
  108. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +56 -4
  109. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +2 -0
  110. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +16 -10
  111. package/dist/bundling-sources/shared_libs/plays/dataset.ts +4 -1
  112. package/dist/cli/index.js +250 -38
  113. package/dist/cli/index.mjs +250 -38
  114. package/dist/{compiler-manifest-DfSnbCTW.d.mts → compiler-manifest-BIqRyj5m.d.mts} +1 -1
  115. package/dist/{compiler-manifest-DfSnbCTW.d.ts → compiler-manifest-BIqRyj5m.d.ts} +1 -1
  116. package/dist/index.d.mts +4 -2
  117. package/dist/index.d.ts +4 -2
  118. package/dist/index.js +231 -35
  119. package/dist/index.mjs +231 -35
  120. package/dist/install-integrity.json +31 -2
  121. package/dist/plays/bundle-play-file.d.mts +2 -2
  122. package/dist/plays/bundle-play-file.d.ts +2 -2
  123. package/dist/plays/bundle-play-file.mjs +65 -1
  124. package/dist/release.d.mts +1 -1
  125. package/dist/release.d.ts +1 -1
  126. package/dist/release.js +1 -1
  127. package/dist/release.mjs +1 -1
  128. package/package.json +1 -1
@@ -20,7 +20,13 @@ import {
20
20
  resolveMaxConcurrentExternalCalls,
21
21
  resolveRowConcurrency,
22
22
  } from './policy';
23
- import { type PacingRule, type RateStateBackend } from './rate-state-backend';
23
+ import {
24
+ type PacingPermit,
25
+ type PacingRule,
26
+ type ProviderConcurrencyGrant,
27
+ type ProviderPacingGrant,
28
+ type RateStateBackend,
29
+ } from './rate-state-backend';
24
30
  import {
25
31
  createInMemoryAdaptiveAdmission,
26
32
  type RuntimeAdaptiveAdmission,
@@ -36,6 +42,30 @@ export interface WorkLease {
36
42
  release(): void;
37
43
  }
38
44
 
45
+ /** A provider admission that may be returned only before its first fetch. */
46
+ export interface ProviderPermit extends WorkLease {
47
+ /** Canonical provider bucket that admitted this physical attempt. */
48
+ readonly provider: string;
49
+ /** Server-authored attribution context for a later provider 429. */
50
+ readonly concurrencyGrant?: ProviderConcurrencyGrant | null;
51
+ /** Effective rate-controller reading for this physical provider permit. */
52
+ readonly pacingGrant?: ProviderPacingGrant | null;
53
+ /**
54
+ * Return an acquired provider permit before an outbound request started.
55
+ * Unlike `release`, this returns the rate reservation as well as any
56
+ * concurrency lease. It is deliberately required at this boundary so a
57
+ * caller cannot silently burn a ticket while re-admitting after local work.
58
+ */
59
+ cancel(): Promise<void>;
60
+ /**
61
+ * Return a due permit that lost a local capacity race so the next
62
+ * `acquireProviderPermit` for the same lane draws it locally. Frees the
63
+ * local concurrency slot like `cancel`, but keeps the rate ticket in the
64
+ * backend's lease block when that block is still live; otherwise refunds.
65
+ */
66
+ park(): Promise<void>;
67
+ }
68
+
39
69
  export type BudgetKind = 'toolCall' | 'retry' | 'waterfallStep';
40
70
 
41
71
  /** Runtime identity and counters shared by inline composition. */
@@ -113,6 +143,16 @@ export interface PlayExecutionGovernor {
113
143
  opts?: { signal?: AbortSignal },
114
144
  ): Promise<WorkLease>;
115
145
 
146
+ /** Acquire only local external-execution capacity; this never charges a budget. */
147
+ acquireToolExecutionSlot(opts?: { signal?: AbortSignal }): Promise<WorkLease>;
148
+
149
+ /**
150
+ * Take immediately available local external-execution capacity, if any.
151
+ * This is deliberately non-blocking: a provider ticket that is already due
152
+ * must be re-admitted instead of aging behind a local queue.
153
+ */
154
+ tryAcquireToolExecutionSlot(): WorkLease | null;
155
+
116
156
  /**
117
157
  * Admit one brief runtime-to-app HTTP fetch/body exchange. Long provider or
118
158
  * fixture residence happens before this edge and must not occupy this slot.
@@ -121,6 +161,12 @@ export interface PlayExecutionGovernor {
121
161
  signal?: AbortSignal;
122
162
  }): Promise<WorkLease>;
123
163
 
164
+ /**
165
+ * Take a runtime-to-app transport slot only if one is immediately ready.
166
+ * A due provider permit must re-admit rather than age behind this queue.
167
+ */
168
+ tryAcquireIntegrationRequestSlot(): WorkLease | null;
169
+
124
170
  /**
125
171
  * Block until one physical provider HTTP attempt is admitted under the
126
172
  * provider's shared rate/concurrency rules. Call this immediately before the
@@ -129,7 +175,16 @@ export interface PlayExecutionGovernor {
129
175
  acquireProviderPermit(
130
176
  toolId: string,
131
177
  opts?: { signal?: AbortSignal },
132
- ): Promise<WorkLease>;
178
+ ): Promise<ProviderPermit>;
179
+
180
+ /**
181
+ * Admit one physical provider attempt only from capacity this process
182
+ * already holds: the tool's local concurrency slot and a due permit in the
183
+ * rate backend's local state. Synchronous and never remote, so a caller may
184
+ * probe it while holding an execution slot. Null until a blocking
185
+ * `acquireProviderPermit` has resolved the tool's pacing at least once.
186
+ */
187
+ tryAcquireProviderPermit(toolId: string): ProviderPermit | null;
133
188
 
134
189
  /**
135
190
  * Suggested batch parallelism for a tool: the provider's own rate hints
@@ -171,6 +226,10 @@ export interface PlayExecutionGovernor {
171
226
  */
172
227
  toolId?: string;
173
228
  retryAfterMs: number;
229
+ /** Provider residence for the rejected physical request, when observed. */
230
+ latencyMs?: number | null;
231
+ /** Server-authored context for attributing this physical provider 429. */
232
+ concurrencyGrant?: ProviderConcurrencyGrant | null;
174
233
  }): Promise<void>;
175
234
 
176
235
  /** Feed clean provider calls back into adaptive admission. */
@@ -203,9 +262,37 @@ interface GovernorInput {
203
262
 
204
263
  class Semaphore {
205
264
  private inFlight = 0;
206
- private readonly waiters: Array<() => void> = [];
265
+ private readonly waiters: Array<(reserved: boolean) => void> = [];
207
266
  constructor(private readonly limit: number) {}
208
267
 
268
+ private lease(): WorkLease {
269
+ this.inFlight += 1;
270
+ return this.leased();
271
+ }
272
+
273
+ /** Return a lease for capacity already reserved by a waiter handoff. */
274
+ private leased(): WorkLease {
275
+ let released = false;
276
+ return {
277
+ release: () => {
278
+ if (released) return;
279
+ released = true;
280
+ this.inFlight = Math.max(0, this.inFlight - 1);
281
+ const next = this.waiters.shift();
282
+ if (next) {
283
+ // Reserve before waking. A synchronous tryAcquire between this
284
+ // release and the promise continuation must not overtake FIFO work.
285
+ this.inFlight += 1;
286
+ next(true);
287
+ }
288
+ },
289
+ };
290
+ }
291
+
292
+ tryAcquire(): WorkLease | null {
293
+ return this.inFlight >= this.limit ? null : this.lease();
294
+ }
295
+
209
296
  async acquire(signal?: AbortSignal): Promise<WorkLease> {
210
297
  // Fail fast on an already-aborted signal: the parked-promise abort listener
211
298
  // below registers with { once: true } and never fires for a signal that was
@@ -217,10 +304,10 @@ class Semaphore {
217
304
  : new Error('Slot acquire aborted.');
218
305
  }
219
306
  while (this.inFlight >= this.limit) {
220
- await new Promise<void>((resolve, reject) => {
221
- const onResolve = () => {
307
+ const reserved = await new Promise<boolean>((resolve, reject) => {
308
+ const onResolve = (wasReserved: boolean) => {
222
309
  signal?.removeEventListener('abort', onAbort);
223
- resolve();
310
+ resolve(wasReserved);
224
311
  };
225
312
  const onAbort = () => {
226
313
  const idx = this.waiters.indexOf(onResolve);
@@ -235,21 +322,17 @@ class Semaphore {
235
322
  signal?.addEventListener('abort', onAbort, { once: true });
236
323
  });
237
324
  if (signal?.aborted) {
325
+ // A release may have handed this waiter capacity immediately before
326
+ // cancellation became observable. Return that handoff so it cannot
327
+ // leak or strand the next queued call.
328
+ if (reserved) this.leased().release();
238
329
  throw signal.reason instanceof Error
239
330
  ? signal.reason
240
331
  : new Error('Slot acquire aborted.');
241
332
  }
333
+ if (reserved) return this.leased();
242
334
  }
243
- this.inFlight += 1;
244
- let released = false;
245
- return {
246
- release: () => {
247
- if (released) return;
248
- released = true;
249
- this.inFlight = Math.max(0, this.inFlight - 1);
250
- this.waiters.shift()?.();
251
- },
252
- };
335
+ return this.lease();
253
336
  }
254
337
  }
255
338
 
@@ -322,6 +405,63 @@ export function createPlayExecutionGovernor(
322
405
 
323
406
  const orgBucket = (provider: string) => `${input.scope.orgId}:${provider}`;
324
407
 
408
+ /** The tool's local semaphore for a declared `maxConcurrency`, else null. */
409
+ const providerConcurrencySlotsFor = (
410
+ toolId: string,
411
+ rules: readonly PacingRule[],
412
+ ): Semaphore | null => {
413
+ const declaredMaxConcurrency = Math.min(
414
+ ...rules.flatMap((rule) =>
415
+ rule.maxConcurrency != null ? [rule.maxConcurrency] : [],
416
+ ),
417
+ );
418
+ if (!Number.isFinite(declaredMaxConcurrency)) return null;
419
+ let slots = providerConcurrencySlotsByTool.get(toolId);
420
+ if (!slots) {
421
+ slots = new Semaphore(Math.max(1, declaredMaxConcurrency));
422
+ providerConcurrencySlotsByTool.set(toolId, slots);
423
+ }
424
+ return slots;
425
+ };
426
+
427
+ const providerPermitFrom = (
428
+ provider: string,
429
+ rateLease: PacingPermit,
430
+ localConcurrencySlot: WorkLease | null,
431
+ ): ProviderPermit => {
432
+ let settled = false;
433
+ return {
434
+ provider,
435
+ concurrencyGrant: rateLease.concurrencyGrant ?? null,
436
+ pacingGrant: rateLease.pacingGrant ?? null,
437
+ release() {
438
+ if (settled) return;
439
+ settled = true;
440
+ rateLease.release();
441
+ localConcurrencySlot?.release();
442
+ },
443
+ async cancel() {
444
+ if (settled) return;
445
+ settled = true;
446
+ try {
447
+ await rateLease.cancel();
448
+ } finally {
449
+ localConcurrencySlot?.release();
450
+ }
451
+ },
452
+ async park() {
453
+ if (settled) return;
454
+ settled = true;
455
+ try {
456
+ if (rateLease.park) await rateLease.park();
457
+ else await rateLease.cancel();
458
+ } finally {
459
+ localConcurrencySlot?.release();
460
+ }
461
+ },
462
+ };
463
+ };
464
+
325
465
  async function resolveScope(toolId: string, provider: string) {
326
466
  const resolved = await input.resolveRateScope?.(toolId, provider);
327
467
  const scope = [
@@ -427,8 +567,17 @@ export function createPlayExecutionGovernor(
427
567
  policy,
428
568
 
429
569
  acquireRowSlot: (opts) => rowSlots.acquire(opts?.signal),
570
+ async acquireToolExecutionSlot(opts) {
571
+ return await toolSlots.acquire(opts?.signal);
572
+ },
573
+
574
+ tryAcquireToolExecutionSlot: () => toolSlots.tryAcquire(),
575
+
576
+ tryAcquireIntegrationRequestSlot: () =>
577
+ integrationRequestSlots.tryAcquire(),
578
+
430
579
  async acquireToolSlot(_toolId, opts) {
431
- const slot = await toolSlots.acquire(opts?.signal);
580
+ const slot = await governor.acquireToolExecutionSlot(opts);
432
581
  // Charge the logical call only after its local execution slot is held, so
433
582
  // a failed/aborted slot acquisition never consumes the budget. The
434
583
  // physical provider admission is intentionally deferred to
@@ -461,15 +610,12 @@ export function createPlayExecutionGovernor(
461
610
  rule.maxConcurrency != null ? [rule.maxConcurrency] : [],
462
611
  ),
463
612
  );
464
- const localConcurrencySlot = Number.isFinite(declaredMaxConcurrency)
465
- ? await (() => {
466
- let slots = providerConcurrencySlotsByTool.get(toolId);
467
- if (!slots) {
468
- slots = new Semaphore(Math.max(1, declaredMaxConcurrency));
469
- providerConcurrencySlotsByTool.set(toolId, slots);
470
- }
471
- return slots.acquire(opts?.signal);
472
- })()
613
+ const localConcurrencySlots = providerConcurrencySlotsFor(
614
+ toolId,
615
+ pacing.rules,
616
+ );
617
+ const localConcurrencySlot = localConcurrencySlots
618
+ ? await localConcurrencySlots.acquire(opts?.signal)
473
619
  : null;
474
620
  try {
475
621
  const rateLease = await input.rateState.acquire({
@@ -478,18 +624,38 @@ export function createPlayExecutionGovernor(
478
624
  rules: pacing.rules,
479
625
  signal: opts?.signal,
480
626
  });
481
- return {
482
- release() {
483
- rateLease.release();
484
- localConcurrencySlot?.release();
485
- },
486
- };
627
+ return providerPermitFrom(pacing.provider, rateLease, localConcurrencySlot);
487
628
  } catch (error) {
488
629
  localConcurrencySlot?.release();
489
630
  throw error;
490
631
  }
491
632
  },
492
633
 
634
+ tryAcquireProviderPermit(toolId) {
635
+ const tryAcquire = input.rateState.tryAcquire;
636
+ const provider = providerByTool.get(toolId);
637
+ const rules = pacingRulesByTool.get(toolId);
638
+ const rateScope = scopeByTool.get(toolId);
639
+ if (!tryAcquire || !provider || !rules || !rateScope) return null;
640
+ const localConcurrencySlots = providerConcurrencySlotsFor(toolId, rules);
641
+ const localConcurrencySlot = localConcurrencySlots
642
+ ? localConcurrencySlots.tryAcquire()
643
+ : null;
644
+ if (localConcurrencySlots && !localConcurrencySlot) return null;
645
+ let rateLease: PacingPermit | null = null;
646
+ try {
647
+ rateLease = tryAcquire.call(input.rateState, {
648
+ bucketId: rateScope[0],
649
+ rateScopeToken: rateScope[1],
650
+ rules,
651
+ });
652
+ } finally {
653
+ if (!rateLease) localConcurrencySlot?.release();
654
+ }
655
+ if (!rateLease) return null;
656
+ return providerPermitFrom(provider, rateLease, localConcurrencySlot);
657
+ },
658
+
493
659
  async suggestedParallelism(toolId, fallback) {
494
660
  const pacing = await resolveAdaptivePacing(toolId);
495
661
  const concurrencyLimits = pacing.rules.flatMap((rule) =>
@@ -556,10 +722,16 @@ export function createPlayExecutionGovernor(
556
722
  });
557
723
  }
558
724
  const rateScope = scopeFor(bp.toolId, bp.provider);
725
+ const rules = bp.toolId ? pacingRulesByTool.get(bp.toolId) : undefined;
559
726
  await input.rateState.penalize({
560
727
  bucketId: rateScope[0],
561
728
  ...(rateScope[1] ? { rateScopeToken: rateScope[1] } : {}),
562
729
  cooldownMs: bp.retryAfterMs,
730
+ ...(rules ? { rules } : {}),
731
+ ...(bp.latencyMs != null ? { latencyMs: bp.latencyMs } : {}),
732
+ ...(bp.concurrencyGrant
733
+ ? { concurrencyGrant: bp.concurrencyGrant }
734
+ : {}),
563
735
  });
564
736
  },
565
737
 
@@ -572,6 +744,7 @@ export function createPlayExecutionGovernor(
572
744
  input.rateState.observeSuccess?.({
573
745
  bucketId: rateScope[0],
574
746
  rules,
747
+ latencyMs: success.latencyMs,
575
748
  });
576
749
  return;
577
750
  }
@@ -651,10 +824,15 @@ function createInlineChildGovernor(
651
824
  policy: root.policy,
652
825
  acquireRowSlot: (opts) => root.acquireRowSlot(opts),
653
826
  acquireToolSlot: (toolId, opts) => root.acquireToolSlot(toolId, opts),
827
+ acquireToolExecutionSlot: (opts) => root.acquireToolExecutionSlot(opts),
828
+ tryAcquireToolExecutionSlot: () => root.tryAcquireToolExecutionSlot(),
829
+ tryAcquireIntegrationRequestSlot: () =>
830
+ root.tryAcquireIntegrationRequestSlot(),
654
831
  acquireIntegrationRequestSlot: (opts) =>
655
832
  root.acquireIntegrationRequestSlot(opts),
656
833
  acquireProviderPermit: (toolId, opts) =>
657
834
  root.acquireProviderPermit(toolId, opts),
835
+ tryAcquireProviderPermit: (toolId) => root.tryAcquireProviderPermit(toolId),
658
836
  suggestedParallelism: (toolId, fallback) =>
659
837
  root.suggestedParallelism(toolId, fallback),
660
838
  suggestedDirectToolParallelism: (toolId, fallback) =>
@@ -3,6 +3,7 @@ import type {
3
3
  PacingRule,
4
4
  RateStateBackend,
5
5
  } from './rate-state-backend';
6
+ import { noopPacingPermit } from './rate-state-backend';
6
7
 
7
8
  /**
8
9
  * In-memory Rate State Backend for the single-process `cjs_node20` runner.
@@ -48,7 +49,7 @@ export class InMemoryRateStateBackend implements RateStateBackend {
48
49
  }): Promise<PacingPermit> {
49
50
  const { bucketId, rules, signal } = input;
50
51
  if (rules.length === 0) {
51
- return { release() {} };
52
+ return noopPacingPermit();
52
53
  }
53
54
  // Stable order to avoid deadlock between concurrent acquirers on shared rules.
54
55
  const ordered = [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
@@ -95,23 +96,36 @@ export class InMemoryRateStateBackend implements RateStateBackend {
95
96
  });
96
97
 
97
98
  if (decision.acquired) {
98
- let released = false;
99
+ let settled = false;
100
+ const releaseConcurrency = async (refundRateReservation: boolean) => {
101
+ await this.withLock(() => {
102
+ for (const rule of ordered) {
103
+ const state = this.ruleStates.get(
104
+ this.key(bucketId, rule.ruleId),
105
+ );
106
+ if (!state) continue;
107
+ if (refundRateReservation && state.startedInWindow > 0) {
108
+ state.startedInWindow -= 1;
109
+ }
110
+ if (rule.maxConcurrency != null && state.activeCount > 0) {
111
+ state.activeCount -= 1;
112
+ }
113
+ }
114
+ });
115
+ };
99
116
  return {
100
117
  // The concurrency decrement is intentionally not awaitable (the permit
101
118
  // signature is sync `release(): void`). It still runs serialized under
102
119
  // the lock; do not "fix" this by trying to await a void.
103
120
  release: () => {
104
- if (released) return;
105
- released = true;
106
- void this.withLock(() => {
107
- for (const rule of ordered) {
108
- if (rule.maxConcurrency == null) continue;
109
- const state = this.ruleStates.get(
110
- this.key(bucketId, rule.ruleId),
111
- );
112
- if (state && state.activeCount > 0) state.activeCount -= 1;
113
- }
114
- });
121
+ if (settled) return;
122
+ settled = true;
123
+ void releaseConcurrency(false);
124
+ },
125
+ cancel: async () => {
126
+ if (settled) return;
127
+ settled = true;
128
+ await releaseConcurrency(true);
115
129
  },
116
130
  };
117
131
  }
@@ -92,6 +92,14 @@ export const MAX_CONFIGURABLE_CONCURRENT_ROWS = 1_000;
92
92
  // available to long-residence providers while still bounding live calls.
93
93
  export const DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS =
94
94
  MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS;
95
+ // One lane is one tool (or one batch bucket). Its in-flight groups are not
96
+ // capped below the run-wide envelope: a group counts as in flight until its
97
+ // slowest call returns, so with provider tails of 8-40 s a small per-lane
98
+ // cap pinned every slot on nearly finished groups and left ~20 calls on the
99
+ // wire per shape. Coalescing under saturation is handled by the round-trip
100
+ // adaptive window in the dispatcher instead.
101
+ export const DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS_PER_LANE =
102
+ DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS;
95
103
  // Launches persisted before this field existed used the original fixed
96
104
  // eight-slot policy. Preserve that value on their retries and resumes.
97
105
  export const LEGACY_MAX_CONCURRENT_EXTERNAL_CALLS = 8;
@@ -153,18 +161,23 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
153
161
  // Runs may request a value through the validated 1..1000 launch contract;
154
162
  // provider pacing and the runner-owned hard ceiling still apply.
155
163
  toolCalls: DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS,
156
- // Physical runtime-to-app fetch/body admission. Keep short socket bursts
157
- // below the sandbox/Vercel transport cliff while allowing many more logical
158
- // calls to remain resident. This is deliberately independent of provider
159
- // rate limits: it protects Deepline's own transport edge.
160
- integrationRequests: 64,
164
+ // Physical runtime-to-app fetch/body admission. This is deliberately
165
+ // independent of provider rate limits: it protects Deepline's own
166
+ // transport edge. Measured at 1k rows on a warm production-shape preview
167
+ // estate: 256 here with a 1,024-request proxy limit took kareem from
168
+ // 8.7 to 11.2 rows/s and modal from 6.9 to 9.4; at 64 sockets 700-1,000
169
+ // resident calls waited seconds for a socket while the sandbox idled.
170
+ // Only a cold estate (replay samples fetched per call until cached)
171
+ // queues badly at this width, and that is a warm-up concern, not a cap.
172
+ integrationRequests: 256,
161
173
  // Scheduling cohorts overlap provider-response tails, but each cohort may
162
174
  // itself contain many logical calls. Keep this materially above the legacy
163
175
  // eight-group ceiling without creating one independently flushed cohort
164
176
  // per configured tool slot. The latter fragments batchable providers and
165
177
  // adds receipt/gateway round trips under wide (128+) runs.
166
178
  toolDispatchGroups: DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS,
167
- toolDispatchGroupsPerLane: DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS,
179
+ toolDispatchGroupsPerLane:
180
+ DEFAULT_MAX_IN_FLIGHT_TOOL_DISPATCH_GROUPS_PER_LANE,
168
181
  },
169
182
  budgets: {
170
183
  // Runaway guards, not workload limits. A 5,000-row map calling several tools
@@ -30,6 +30,24 @@ export interface PacingRule {
30
30
  readonly maxConcurrency: number | null;
31
31
  }
32
32
 
33
+ /**
34
+ * Effective shared rate captured when a physical request was admitted. It is
35
+ * read-only diagnostic context: callers cannot widen a provider limit.
36
+ */
37
+ export type ProviderPacingGrant = {
38
+ readonly targetRequestsPerSecond: number | null;
39
+ };
40
+
41
+ /**
42
+ * Global occupancy and RTT baseline captured with one concurrency permit.
43
+ * Pure-rate permits publish only {@link ProviderPacingGrant}.
44
+ */
45
+ export type ProviderConcurrencyGrant = ProviderPacingGrant & {
46
+ readonly admissionId: string;
47
+ readonly inFlightAtStart: number;
48
+ readonly minimumRttMs: number | null;
49
+ };
50
+
33
51
  /**
34
52
  * Per-tool queue-hint metadata produced by the runtime tool catalog
35
53
  * (`packages/integrations/runtime-tool-metadata.ts`) and surfaced to the runtime via
@@ -49,9 +67,26 @@ export interface PlayQueueHint {
49
67
  maxConcurrency: number | null;
50
68
  }
51
69
 
52
- /** Handle returned by acquire(); release frees any concurrency held by the rules. */
70
+ /** Handle returned by acquire(). */
53
71
  export interface PacingPermit {
72
+ /** Complete a dispatched request; frees only any concurrency held by the rules. */
54
73
  release(): void;
74
+ /**
75
+ * Cancel a ticket that never reached the provider. Unlike `release`, this
76
+ * returns its rate reservation before freeing any held concurrency.
77
+ */
78
+ cancel(): Promise<void>;
79
+ /**
80
+ * Return a due ticket that never reached the provider to the backend's local
81
+ * lease block, so the next acquire on the same lane draws it with no store
82
+ * round trip. Falls back to `cancel` when that block is gone or expired.
83
+ * Absent on backends that keep no local block; callers use `cancel` then.
84
+ */
85
+ park?(): Promise<void>;
86
+ /** Server-authored context for attributing this request's provider 429. */
87
+ readonly concurrencyGrant?: ProviderConcurrencyGrant | null;
88
+ /** Effective shared rate for every app-runtime provider permit. */
89
+ readonly pacingGrant?: ProviderPacingGrant | null;
55
90
  }
56
91
 
57
92
  /**
@@ -85,6 +120,18 @@ export interface RateStateBackend {
85
120
  signal?: AbortSignal;
86
121
  }): Promise<PacingPermit>;
87
122
 
123
+ /**
124
+ * Take a permit for `bucketId` only if one is already due in local state.
125
+ * Synchronous and non-blocking: never refills, sleeps, or reaches the store.
126
+ * Returns null when no due permit is held locally, when the pacer is
127
+ * holding the provider off, or when the backend keeps no local state.
128
+ */
129
+ tryAcquire?(input: {
130
+ bucketId: string;
131
+ rateScopeToken?: string | null;
132
+ rules: readonly PacingRule[];
133
+ }): PacingPermit | null;
134
+
88
135
  /**
89
136
  * Feed a server-observed Retry-After back so future acquires for this bucket
90
137
  * back off. Advisory and idempotent; never un-charges an in-flight call.
@@ -93,6 +140,12 @@ export interface RateStateBackend {
93
140
  bucketId: string;
94
141
  rateScopeToken?: string | null;
95
142
  cooldownMs: number;
143
+ /** Resolved provider rules for the rejected physical attempt. */
144
+ rules?: readonly PacingRule[];
145
+ /** Provider residence measured at the HTTP boundary. */
146
+ latencyMs?: number | null;
147
+ /** Server-authored context for attributing this physical provider 429. */
148
+ concurrencyGrant?: ProviderConcurrencyGrant | null;
96
149
  }): void | Promise<void>;
97
150
 
98
151
  /** Record confirmed provider success for adaptive recovery. */
@@ -100,10 +153,16 @@ export interface RateStateBackend {
100
153
  bucketId: string;
101
154
  rateScopeToken?: string | null;
102
155
  rules: readonly PacingRule[];
156
+ /** Provider residence measured at the HTTP boundary. */
157
+ latencyMs?: number | null;
103
158
  }): void;
104
159
  }
105
160
 
106
- const NOOP_PERMIT: PacingPermit = { release() {} };
161
+ const NOOP_PERMIT: PacingPermit = {
162
+ release() {},
163
+ async cancel() {},
164
+ async park() {},
165
+ };
107
166
 
108
167
  /** Permit used when a bucket has no rules — nothing to debit or release. */
109
168
  export function noopPacingPermit(): PacingPermit {
@@ -12,6 +12,22 @@ export type IntegrationOperationStorageMode =
12
12
  export const INTEGRATION_OPERATION_CANONICAL_READER_CAPABILITY =
13
13
  'integration-operation-canonical-v1' as const;
14
14
 
15
+ /**
16
+ * A strictly stronger reader proof. A gateway that reports this capability can
17
+ * materialize the existing canonical Integration Operation outcome and the
18
+ * immutable R2 receipt-value references that may be published beside it.
19
+ *
20
+ * Keep this distinct from the original canonical capability: an already-active
21
+ * canonical fence may have certified an older gateway image that cannot read a
22
+ * receipt reference yet.
23
+ */
24
+ export const RUNTIME_RECEIPT_VALUE_REFERENCE_READER_CAPABILITY =
25
+ 'runtime-receipt-value-reference-v1' as const;
26
+
27
+ export type RuntimeGatewayReaderCapability =
28
+ | typeof INTEGRATION_OPERATION_CANONICAL_READER_CAPABILITY
29
+ | typeof RUNTIME_RECEIPT_VALUE_REFERENCE_READER_CAPABILITY;
30
+
15
31
  export type RuntimeGatewayRole = 'receipt' | 'execution';
16
32
 
17
33
  /**
@@ -20,7 +36,7 @@ export type RuntimeGatewayRole = 'receipt' | 'execution';
20
36
  * collects this evidence; this module records and validates its shape.
21
37
  */
22
38
  export type CanonicalReaderProof = {
23
- capability: typeof INTEGRATION_OPERATION_CANONICAL_READER_CAPABILITY;
39
+ capability: RuntimeGatewayReaderCapability;
24
40
  sourceSha: string;
25
41
  machineIds: Readonly<Record<RuntimeGatewayRole, readonly string[]>>;
26
42
  };
@@ -47,6 +63,18 @@ export type ReaderFenceWriterMode = {
47
63
  failure?: 'store_unavailable' | 'invalid_record';
48
64
  };
49
65
 
66
+ /**
67
+ * The additional format is write-enabled only after the stronger, exact-fleet
68
+ * proof reaches the durable mirror. A missing or unreadable mirror stays
69
+ * inline, which is compatible with every prior gateway image.
70
+ */
71
+ export type RuntimeReceiptValueReferenceWriterMode = {
72
+ enabled: boolean;
73
+ source: 'fence' | 'absent' | 'fail_closed';
74
+ epoch: number | null;
75
+ failure?: 'store_unavailable' | 'invalid_record';
76
+ };
77
+
50
78
  export type ReaderFenceApplyOutcome =
51
79
  | { kind: 'advanced'; fence: RuntimeGatewayReaderFence }
52
80
  | { kind: 'already_applied'; fence: RuntimeGatewayReaderFence }
@@ -59,6 +87,10 @@ export type ReaderFenceApplyOutcome =
59
87
  export type RuntimeGatewayReaderFenceJob = {
60
88
  /** Resolve the only format this gateway may write. Never returns canonical on uncertainty. */
61
89
  writerMode(input: { schema: string }): Promise<ReaderFenceWriterMode>;
90
+ /** Resolve whether this gateway may publish receipt value references. */
91
+ runtimeReceiptValueReferenceWriterMode(input: {
92
+ schema: string;
93
+ }): Promise<RuntimeReceiptValueReferenceWriterMode>;
62
94
  /** Advance the mirror only with a strictly newer epoch, or retry the exact same fence. */
63
95
  apply(
64
96
  candidate: RuntimeGatewayReaderFenceCandidate,