deepline 0.1.206 → 0.1.208

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +25 -16
  2. package/dist/bundling-sources/sdk/src/http.ts +10 -1
  3. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  4. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +96 -12
  5. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +51 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +16 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/child-run-lifecycle.ts +70 -0
  8. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +321 -175
  9. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +13 -1
  10. package/dist/bundling-sources/shared_libs/play-runtime/governor/adaptive-admission.ts +16 -4
  11. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +161 -81
  12. package/dist/bundling-sources/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts +26 -9
  13. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +109 -1
  14. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +1 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +7 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/run-execution-scope.ts +256 -0
  17. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +17 -5
  18. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +103 -18
  19. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +8 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +1 -1
  21. package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +5 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts +74 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +5 -12
  24. package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +2 -1
  25. package/dist/bundling-sources/shared_libs/runtime-env.ts +55 -0
  26. package/dist/cli/index.js +33 -7
  27. package/dist/cli/index.mjs +33 -7
  28. package/dist/index.js +8 -3
  29. package/dist/index.mjs +8 -3
  30. package/package.json +1 -1
@@ -12,7 +12,10 @@ import type {
12
12
  PlayQueueHint,
13
13
  RateStateBackend,
14
14
  } from './governor/rate-state-backend';
15
- import type { GovernanceSnapshot } from './governor/governor';
15
+ import type {
16
+ GovernanceSnapshot,
17
+ PlayExecutionGovernor,
18
+ } from './governor/governor';
16
19
  import type { BudgetStateBackend } from './governor/budget-state-backend';
17
20
  import type { RuntimeMapMemoryLimits } from './map-memory-limits';
18
21
  import type { AnyBatchOperationStrategy } from './batching-types';
@@ -20,6 +23,7 @@ import type { ToolResultMetadataInput } from './tool-result';
20
23
  import type { PreviousCell } from './cell-staleness';
21
24
  import type { MapRowOutcome } from './durability-store';
22
25
  import type { WorkReceiptFailureKind } from './work-receipts';
26
+ import type { RunExecutionScope } from './run-execution-scope';
23
27
 
24
28
  export interface RowState {
25
29
  results: Map<string, unknown>;
@@ -470,6 +474,8 @@ export type IntegrationEventWaitHandler = {
470
474
  };
471
475
 
472
476
  export interface ContextOptions {
477
+ /** Immutable logical and physical execution identity for this context. */
478
+ executionScope?: RunExecutionScope;
473
479
  /** Short-lived HMAC-signed internal token for tool callbacks. Required for cloud execution. */
474
480
  executorToken?: string;
475
481
  baseUrl?: string;
@@ -568,6 +574,10 @@ export interface ContextOptions {
568
574
  }) => Promise<void>;
569
575
  playId?: string;
570
576
  runId?: string;
577
+ /** Physical executor run that owns leases for an inline child context. */
578
+ runtimeReceiptOwnerRunId?: string;
579
+ /** Logical invocation scope for nested step/fetch/runPlay receipt identity. */
580
+ runtimeReceiptScope?: string;
571
581
  runAttempt?: number | null;
572
582
  /**
573
583
  * Optional shared provider-rate state for fleet substrates. Local/test
@@ -695,6 +705,8 @@ export interface ContextOptions {
695
705
  * guards accumulate across the nested-play tree. Only set on child contexts.
696
706
  */
697
707
  governance?: GovernanceSnapshot;
708
+ /** Internal in-process child view sharing the root run's resource governor. */
709
+ executionGovernor?: PlayExecutionGovernor;
698
710
  }
699
711
 
700
712
  export interface PlayCheckpoint {
@@ -95,8 +95,8 @@ export function createInMemoryAdaptiveAdmission(
95
95
  finitePositive(options.maxRequestsPerSecond, initialRequestsPerSecond),
96
96
  );
97
97
  const additiveIncreasePerWindow = finitePositive(
98
- options.additiveIncreasePerWindow ?? 5,
99
- 5,
98
+ options.additiveIncreasePerWindow ?? 1,
99
+ 1,
100
100
  );
101
101
  const cleanWindowMs = finitePositive(options.cleanWindowMs ?? 1_000, 1_000);
102
102
  const states = new Map<string, ProviderState>();
@@ -192,14 +192,26 @@ export function createInMemoryAdaptiveAdmission(
192
192
  observeProviderBackpressure(input) {
193
193
  const state = getExistingState(input.orgId, input.provider);
194
194
  if (!state) return;
195
+ const currentTime = now();
195
196
  state.rateLimitCount += 1;
197
+ const nextRetryAfterUntilMs =
198
+ currentTime + Math.max(0, input.retryAfterMs);
199
+ if (
200
+ state.retryAfterUntilMs != null &&
201
+ state.retryAfterUntilMs > currentTime
202
+ ) {
203
+ state.retryAfterUntilMs = Math.max(
204
+ state.retryAfterUntilMs,
205
+ nextRetryAfterUntilMs,
206
+ );
207
+ return;
208
+ }
196
209
  state.hasSeenBackpressure = true;
197
210
  state.currentRequestsPerSecond = Math.max(
198
211
  minRequestsPerSecond,
199
212
  Math.floor(state.currentRequestsPerSecond / 2),
200
213
  );
201
- const currentTime = now();
202
- state.retryAfterUntilMs = currentTime + Math.max(0, input.retryAfterMs);
214
+ state.retryAfterUntilMs = nextRetryAfterUntilMs;
203
215
  state.lastIncreaseAtMs = currentTime;
204
216
  state.successCountAtLastIncrease = state.successCount;
205
217
  },
@@ -18,7 +18,8 @@ import {
18
18
  * stale-block over-issue to 8 x live runner processes per bucket/window.
19
19
  */
20
20
  export const APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE = 8;
21
- const LEASE_BLOCK_TTL_MS = 250;
21
+ const LEASE_BLOCK_MIN_TTL_MS = 250;
22
+ const LEASE_BLOCK_MAX_TTL_MS = 5_000;
22
23
  const MAX_ACQUIRE_SLEEP_MS = 5_000;
23
24
  const STORE_FAILURE_RETRY_DELAYS_MS = [100, 250] as const;
24
25
 
@@ -42,6 +43,13 @@ interface LeasedBlock {
42
43
  rules: PacingRule[];
43
44
  }
44
45
 
46
+ interface PendingRelease {
47
+ bucketId: string;
48
+ rules: PacingRule[];
49
+ leaseIds: string[];
50
+ flushing: boolean;
51
+ }
52
+
45
53
  interface Options {
46
54
  schedulerSchema?: string | null;
47
55
  now?: () => number;
@@ -59,6 +67,8 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
59
67
  error: string;
60
68
  }) => void;
61
69
  private readonly blocks = new Map<string, LeasedBlock>();
70
+ private readonly acquisitionTails = new Map<string, Promise<void>>();
71
+ private readonly pendingReleases = new Map<string, PendingRelease>();
62
72
  private pendingReleaseFailure: Error | null = null;
63
73
 
64
74
  constructor(context: WorkerRuntimeApiContext, options: Options = {}) {
@@ -86,80 +96,101 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
86
96
 
87
97
  const ordered = [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
88
98
  const rulesKey = rulesSignature(ordered);
89
- const localPermit = this.drawFromBlock(bucketId, rulesKey);
90
- if (localPermit) return localPermit;
99
+ return await this.withBucketAcquisitionLock(bucketId, async () => {
100
+ const localPermit = this.drawFromBlock(bucketId, rulesKey);
101
+ if (localPermit) return localPermit;
91
102
 
92
- let consecutiveFailures = 0;
93
- while (true) {
94
- if (signal?.aborted) {
95
- throw signal.reason instanceof Error
96
- ? signal.reason
97
- : new Error('Rate-state acquire aborted.');
98
- }
99
- try {
100
- const response = await acquireRateStateViaAppRuntime(this.context, {
101
- bucketId,
102
- rules: ordered,
103
- requested: APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE,
104
- schedulerSchema: this.schedulerSchema,
105
- });
106
- consecutiveFailures = 0;
107
- if (response.granted > 0) {
108
- const hasConcurrency = ordered.some(
109
- (rule) => rule.maxConcurrency != null,
110
- );
111
- const leaseIds = Array.isArray(response.leaseIds)
112
- ? response.leaseIds.filter(
113
- (leaseId): leaseId is string =>
114
- typeof leaseId === 'string' && leaseId.trim().length > 0,
115
- )
116
- : [];
117
- if (hasConcurrency && leaseIds.length !== response.granted) {
118
- throw new Error(
119
- `Rate-state store granted ${response.granted} concurrency permits for ${bucketId} with ${leaseIds.length} lease tokens.`,
103
+ let consecutiveFailures = 0;
104
+ while (true) {
105
+ if (signal?.aborted) {
106
+ throw signal.reason instanceof Error
107
+ ? signal.reason
108
+ : new Error('Rate-state acquire aborted.');
109
+ }
110
+ try {
111
+ const response = await acquireRateStateViaAppRuntime(this.context, {
112
+ bucketId,
113
+ rules: ordered,
114
+ requested: APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE,
115
+ schedulerSchema: this.schedulerSchema,
116
+ });
117
+ consecutiveFailures = 0;
118
+ if (response.granted > 0) {
119
+ const hasConcurrency = ordered.some(
120
+ (rule) => rule.maxConcurrency != null,
120
121
  );
122
+ const leaseIds = Array.isArray(response.leaseIds)
123
+ ? response.leaseIds.filter(
124
+ (leaseId): leaseId is string =>
125
+ typeof leaseId === 'string' && leaseId.trim().length > 0,
126
+ )
127
+ : [];
128
+ if (hasConcurrency && leaseIds.length !== response.granted) {
129
+ throw new Error(
130
+ `Rate-state store granted ${response.granted} concurrency permits for ${bucketId} with ${leaseIds.length} lease tokens.`,
131
+ );
132
+ }
133
+ if (!hasConcurrency && leaseIds.length > 0) {
134
+ throw new Error(
135
+ `Rate-state store returned ${leaseIds.length} lease tokens for pure-rate bucket ${bucketId}.`,
136
+ );
137
+ }
138
+ const currentLeaseId = hasConcurrency ? leaseIds[0]! : null;
139
+ const remainingLeaseIds = hasConcurrency ? leaseIds.slice(1) : [];
140
+ const remainingWindowPermits = hasConcurrency
141
+ ? 0
142
+ : response.granted - 1;
143
+ if (remainingLeaseIds.length > 0 || remainingWindowPermits > 0) {
144
+ this.mergeBlock(
145
+ bucketId,
146
+ remainingLeaseIds,
147
+ remainingWindowPermits,
148
+ rulesKey,
149
+ ordered,
150
+ );
151
+ }
152
+ return this.permitFor(bucketId, ordered, currentLeaseId);
121
153
  }
122
- if (!hasConcurrency && leaseIds.length > 0) {
154
+ await this.sleep(
155
+ Math.max(1, Math.min(response.waitMs, MAX_ACQUIRE_SLEEP_MS)),
156
+ );
157
+ } catch (error) {
158
+ // Fail closed. A degraded rate-state store must not grant unlimited
159
+ // provider calls; after this bounded retry ladder the run fails loudly.
160
+ consecutiveFailures += 1;
161
+ const normalized = normalizeError(error);
162
+ this.onStoreFailure({ bucketId, error: normalized.message });
163
+ if (consecutiveFailures > STORE_FAILURE_RETRY_DELAYS_MS.length) {
123
164
  throw new Error(
124
- `Rate-state store returned ${leaseIds.length} lease tokens for pure-rate bucket ${bucketId}.`,
165
+ `Rate-state store unavailable for ${bucketId}; failing closed after ${consecutiveFailures} acquire attempts: ${normalized.message}`,
125
166
  );
126
167
  }
127
- // Serve this acquire from the grant, then cache the rest so the whole
128
- // block amortizes into a single round trip. Concurrency rules draw a
129
- // lease token (needs release); pure-rate rules draw a window permit
130
- // (already decremented server-side, no release).
131
- const currentLeaseId = hasConcurrency ? leaseIds[0]! : null;
132
- const remainingLeaseIds = hasConcurrency ? leaseIds.slice(1) : [];
133
- const remainingWindowPermits = hasConcurrency
134
- ? 0
135
- : response.granted - 1;
136
- if (remainingLeaseIds.length > 0 || remainingWindowPermits > 0)
137
- this.mergeBlock(
138
- bucketId,
139
- remainingLeaseIds,
140
- remainingWindowPermits,
141
- rulesKey,
142
- ordered,
143
- );
144
- return this.permitFor(bucketId, ordered, currentLeaseId);
145
- }
146
- await this.sleep(
147
- Math.max(1, Math.min(response.waitMs, MAX_ACQUIRE_SLEEP_MS)),
148
- );
149
- } catch (error) {
150
- // Fail closed. A degraded rate-state store must not grant unlimited
151
- // provider calls; after this bounded retry ladder the run fails loudly.
152
- consecutiveFailures += 1;
153
- const normalized = normalizeError(error);
154
- this.onStoreFailure({ bucketId, error: normalized.message });
155
- if (consecutiveFailures > STORE_FAILURE_RETRY_DELAYS_MS.length) {
156
- throw new Error(
157
- `Rate-state store unavailable for ${bucketId}; failing closed after ${consecutiveFailures} acquire attempts: ${normalized.message}`,
168
+ await this.sleep(
169
+ STORE_FAILURE_RETRY_DELAYS_MS[consecutiveFailures - 1] ?? 250,
158
170
  );
159
171
  }
160
- await this.sleep(
161
- STORE_FAILURE_RETRY_DELAYS_MS[consecutiveFailures - 1] ?? 250,
162
- );
172
+ }
173
+ });
174
+ }
175
+
176
+ private async withBucketAcquisitionLock<T>(
177
+ bucketId: string,
178
+ operation: () => Promise<T>,
179
+ ): Promise<T> {
180
+ const previous = this.acquisitionTails.get(bucketId) ?? Promise.resolve();
181
+ let unlock!: () => void;
182
+ const gate = new Promise<void>((resolve) => {
183
+ unlock = resolve;
184
+ });
185
+ const tail = previous.then(() => gate);
186
+ this.acquisitionTails.set(bucketId, tail);
187
+ await previous;
188
+ try {
189
+ return await operation();
190
+ } finally {
191
+ unlock();
192
+ if (this.acquisitionTails.get(bucketId) === tail) {
193
+ this.acquisitionTails.delete(bucketId);
163
194
  }
164
195
  }
165
196
  }
@@ -208,7 +239,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
208
239
  rulesKey: string,
209
240
  rules: PacingRule[],
210
241
  ): void {
211
- const freshExpiresAt = this.now() + LEASE_BLOCK_TTL_MS;
242
+ const freshExpiresAt = this.now() + leasedBlockTtlMs(rules);
212
243
  const existing = this.blocks.get(bucketId);
213
244
  if (
214
245
  existing &&
@@ -240,8 +271,8 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
240
271
  if (block.rulesKey !== rulesKey || block.expiresAt <= this.now()) {
241
272
  this.blocks.delete(bucketId);
242
273
  // Unused window permits are forfeited on TTL expiry (the server already
243
- // decremented the provider window for them, and the LEASE_BLOCK_TTL_MS
244
- // window is short so at most one block's worth is stranded per bucket).
274
+ // decremented the provider window for them, and the rule-scoped TTL is
275
+ // capped so at most one block's worth is stranded per bucket/window).
245
276
  // Concurrency leases must be released so the shared slot count drains.
246
277
  this.releaseReserved(bucketId, block.rules, block.leaseIds);
247
278
  return null;
@@ -275,18 +306,54 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
275
306
  !rules.some((rule) => rule.maxConcurrency != null)
276
307
  )
277
308
  return;
278
- void releaseRateStateViaAppRuntime(this.context, {
309
+ const ordered = [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
310
+ const key = `${bucketId}\0${rulesSignature(ordered)}`;
311
+ const pending = this.pendingReleases.get(key) ?? {
279
312
  bucketId,
280
- rules: [...rules],
281
- leaseIds: [...leaseIds],
282
- schedulerSchema: this.schedulerSchema,
283
- }).catch((error) => {
284
- const normalized = normalizeError(error);
285
- this.pendingReleaseFailure = new Error(
286
- `Rate-state release failed for ${bucketId}: ${normalized.message}`,
287
- );
288
- this.onStoreFailure({ bucketId, error: normalized.message });
289
- });
313
+ rules: ordered,
314
+ leaseIds: [],
315
+ flushing: false,
316
+ };
317
+ pending.leaseIds.push(...leaseIds);
318
+ this.pendingReleases.set(key, pending);
319
+ if (pending.flushing) return;
320
+ pending.flushing = true;
321
+ queueMicrotask(() => void this.flushReleases(key, pending));
322
+ }
323
+
324
+ private async flushReleases(
325
+ key: string,
326
+ pending: PendingRelease,
327
+ ): Promise<void> {
328
+ while (pending.leaseIds.length > 0) {
329
+ const leaseIds = pending.leaseIds.splice(0);
330
+ try {
331
+ await releaseRateStateViaAppRuntime(this.context, {
332
+ bucketId: pending.bucketId,
333
+ rules: pending.rules,
334
+ leaseIds,
335
+ schedulerSchema: this.schedulerSchema,
336
+ });
337
+ } catch (error) {
338
+ const normalized = normalizeError(error);
339
+ this.pendingReleaseFailure = new Error(
340
+ `Rate-state release failed for ${pending.bucketId}: ${normalized.message}`,
341
+ );
342
+ this.onStoreFailure({
343
+ bucketId: pending.bucketId,
344
+ error: normalized.message,
345
+ });
346
+ }
347
+ }
348
+ pending.flushing = false;
349
+ if (pending.leaseIds.length > 0) {
350
+ pending.flushing = true;
351
+ queueMicrotask(() => void this.flushReleases(key, pending));
352
+ return;
353
+ }
354
+ if (this.pendingReleases.get(key) === pending) {
355
+ this.pendingReleases.delete(key);
356
+ }
290
357
  }
291
358
 
292
359
  private throwPendingReleaseFailure(): void {
@@ -307,6 +374,19 @@ function rulesSignature(rules: readonly PacingRule[]): string {
307
374
  .join('|');
308
375
  }
309
376
 
377
+ function leasedBlockTtlMs(rules: readonly PacingRule[]): number {
378
+ const shortestWindowMs = Math.min(
379
+ ...rules
380
+ .map((rule) => rule.windowMs)
381
+ .filter((windowMs) => Number.isFinite(windowMs) && windowMs > 0),
382
+ );
383
+ if (!Number.isFinite(shortestWindowMs)) return LEASE_BLOCK_MIN_TTL_MS;
384
+ return Math.max(
385
+ LEASE_BLOCK_MIN_TTL_MS,
386
+ Math.min(LEASE_BLOCK_MAX_TTL_MS, Math.floor(shortestWindowMs)),
387
+ );
388
+ }
389
+
310
390
  function normalizeError(error: unknown): Error {
311
391
  return error instanceof Error ? error : new Error(String(error));
312
392
  }
@@ -21,9 +21,9 @@ import {
21
21
  * hello-world latency baseline. Instead the backend LEASES SMALL PERMIT BLOCKS:
22
22
  * one `/rate-acquire` round-trip debits up to {@link LEASE_BLOCK_SIZE} permits
23
23
  * from the global window, and subsequent acquires draw from the local block
24
- * until it is exhausted or its short TTL expires. This bounds round-trips to
25
- * roughly `calls / LEASE_BLOCK_SIZE` while keeping over-issuance bounded by one
26
- * block per isolate per window.
24
+ * until it is exhausted or its rule-scoped TTL expires. This bounds round-trips
25
+ * to roughly `calls / LEASE_BLOCK_SIZE` while keeping over-issuance bounded by
26
+ * one block per isolate per window.
27
27
  *
28
28
  * Fail-open: if the coordinator is unreachable the backend logs once and
29
29
  * PROCEEDS (grants the permit) rather than stalling the run, matching the
@@ -35,11 +35,14 @@ import {
35
35
  /** Permits leased per round-trip. Tuned to amortize the DO hop, not to batch. */
36
36
  const LEASE_BLOCK_SIZE = 16;
37
37
  /**
38
- * Max age of a leased block. A leased permit debited the global window already,
39
- * but if it is held past roughly one window it could let an isolate run ahead
40
- * of a rolled-over window. Discarding stale blocks bounds that to sub-window.
38
+ * Min/max age of a leased block. A leased permit debited the global window
39
+ * already, but if it is held well past the provider window it could let an
40
+ * isolate run ahead of a rolled-over window. The effective TTL is derived from
41
+ * the shortest rule window so low-RPS pure-rate providers can draw already-
42
+ * debited permits throughout a normal window instead of burning most blocks.
41
43
  */
42
- const LEASE_BLOCK_TTL_MS = 250;
44
+ const LEASE_BLOCK_MIN_TTL_MS = 250;
45
+ const LEASE_BLOCK_MAX_TTL_MS = 5_000;
43
46
  /** Cap one coordinator-provided sleep hint; total wait is bounded by run timeout. */
44
47
  const MAX_ACQUIRE_SLEEP_MS = 5_000;
45
48
 
@@ -148,7 +151,7 @@ export class CoordinatorRateStateBackend implements RateStateBackend {
148
151
  // Consume one for this call; cache the rest as a short-lived block.
149
152
  const remaining = response.granted - 1;
150
153
  if (remaining > 0) {
151
- this.mergeBlock(bucketId, remaining, rulesKey);
154
+ this.mergeBlock(bucketId, remaining, rulesKey, rules);
152
155
  }
153
156
  return noopPacingPermit();
154
157
  }
@@ -196,8 +199,9 @@ export class CoordinatorRateStateBackend implements RateStateBackend {
196
199
  bucketId: string,
197
200
  remaining: number,
198
201
  rulesKey: string,
202
+ rules: readonly PacingRule[],
199
203
  ): void {
200
- const freshExpiresAt = this.now() + LEASE_BLOCK_TTL_MS;
204
+ const freshExpiresAt = this.now() + leasedBlockTtlMs(rules);
201
205
  const existing = this.blocks.get(bucketId);
202
206
  if (
203
207
  existing &&
@@ -229,3 +233,16 @@ export class CoordinatorRateStateBackend implements RateStateBackend {
229
233
  return true;
230
234
  }
231
235
  }
236
+
237
+ function leasedBlockTtlMs(rules: readonly PacingRule[]): number {
238
+ const shortestWindowMs = Math.min(
239
+ ...rules
240
+ .map((rule) => rule.windowMs)
241
+ .filter((windowMs) => Number.isFinite(windowMs) && windowMs > 0),
242
+ );
243
+ if (!Number.isFinite(shortestWindowMs)) return LEASE_BLOCK_MIN_TTL_MS;
244
+ return Math.max(
245
+ LEASE_BLOCK_MIN_TTL_MS,
246
+ Math.min(LEASE_BLOCK_MAX_TTL_MS, Math.floor(shortestWindowMs)),
247
+ );
248
+ }
@@ -147,6 +147,16 @@ export interface PlayExecutionGovernor {
147
147
  childRunId: string;
148
148
  }): Promise<GovernanceSnapshot>;
149
149
 
150
+ /**
151
+ * Fork an in-process child view. The child gets its own lineage while sharing
152
+ * the parent's row/tool/child semaphores, adaptive admission, and global
153
+ * budget counters.
154
+ */
155
+ forkInlineChild(input: {
156
+ childPlayName: string;
157
+ childRunId: string;
158
+ }): Promise<PlayExecutionGovernor>;
159
+
150
160
  /** Effective row concurrency: explicit request clamped to [1, rowMax], else default. */
151
161
  resolveRowConcurrency(requested?: number): number;
152
162
 
@@ -364,7 +374,7 @@ export function createPlayExecutionGovernor(
364
374
  }
365
375
  }
366
376
 
367
- return {
377
+ const governor: PlayExecutionGovernor = {
368
378
  adapter: input.adapter,
369
379
  policy,
370
380
 
@@ -510,6 +520,13 @@ export function createPlayExecutionGovernor(
510
520
  };
511
521
  },
512
522
 
523
+ async forkInlineChild(childInput) {
524
+ return createInlineChildGovernor(
525
+ governor,
526
+ await governor.forkChild(childInput),
527
+ );
528
+ },
529
+
513
530
  resolveRowConcurrency: (requested) =>
514
531
  resolveRowConcurrency(policy, requested),
515
532
 
@@ -551,4 +568,95 @@ export function createPlayExecutionGovernor(
551
568
  parentChildCalls: { ...state.parentChildCalls },
552
569
  }),
553
570
  };
571
+ return governor;
572
+ }
573
+
574
+ function createInlineChildGovernor(
575
+ root: PlayExecutionGovernor,
576
+ initial: GovernanceSnapshot,
577
+ ): PlayExecutionGovernor {
578
+ const lineage = {
579
+ rootRunId: initial.rootRunId,
580
+ currentRunId: initial.currentRunId,
581
+ currentPlayId: initial.currentPlayId,
582
+ ancestryPlayIds: [...initial.ancestryPlayIds],
583
+ ancestryRunIds: [...initial.ancestryRunIds],
584
+ callDepth: initial.callDepth,
585
+ parentChildCalls: { ...initial.parentChildCalls },
586
+ };
587
+
588
+ const child: PlayExecutionGovernor = {
589
+ adapter: root.adapter,
590
+ policy: root.policy,
591
+ acquireRowSlot: (opts) => root.acquireRowSlot(opts),
592
+ acquireChildSubmitSlot: (opts) => root.acquireChildSubmitSlot(opts),
593
+ acquireToolSlot: (toolId, opts) => root.acquireToolSlot(toolId, opts),
594
+ suggestedParallelism: (toolId, fallback) =>
595
+ root.suggestedParallelism(toolId, fallback),
596
+ chargeBudget: (kind, amount) => root.chargeBudget(kind, amount),
597
+ resolveRowConcurrency: (requested) => root.resolveRowConcurrency(requested),
598
+ reportProviderBackpressure: (input) =>
599
+ root.reportProviderBackpressure(input),
600
+ observeProviderSuccess: (input) => root.observeProviderSuccess(input),
601
+ observeToolSuccess: (input) => root.observeToolSuccess(input),
602
+ async forkChild(input) {
603
+ if (lineage.ancestryPlayIds.includes(input.childPlayName)) {
604
+ throw new Error(
605
+ `Recursive play graph detected: ${[
606
+ ...lineage.ancestryPlayIds,
607
+ input.childPlayName,
608
+ ].join(' -> ')}`,
609
+ );
610
+ }
611
+ const nextDepth = lineage.callDepth + 1;
612
+ if (nextDepth > child.policy.budgets.maxPlayCallDepth) {
613
+ throw new GovernorBudgetError(
614
+ 'playDepth',
615
+ nextDepth,
616
+ child.policy.budgets.maxPlayCallDepth,
617
+ );
618
+ }
619
+ const nextParent =
620
+ (lineage.parentChildCalls[lineage.currentPlayId] ?? 0) + 1;
621
+ if (nextParent > child.policy.budgets.maxChildPlayCallsPerParent) {
622
+ throw new GovernorBudgetError(
623
+ 'childPerParent',
624
+ nextParent,
625
+ child.policy.budgets.maxChildPlayCallsPerParent,
626
+ );
627
+ }
628
+ await root.chargeBudget('playCall');
629
+ await root.chargeBudget('descendant');
630
+ lineage.parentChildCalls[lineage.currentPlayId] = nextParent;
631
+ const counters = root.snapshot();
632
+ return {
633
+ rootRunId: lineage.rootRunId,
634
+ currentRunId: input.childRunId,
635
+ currentPlayId: input.childPlayName,
636
+ ancestryPlayIds: [...lineage.ancestryPlayIds, input.childPlayName],
637
+ ancestryRunIds: [...lineage.ancestryRunIds, input.childRunId],
638
+ callDepth: nextDepth,
639
+ playCallCount: counters.playCallCount,
640
+ toolCallCount: counters.toolCallCount,
641
+ retryCount: counters.retryCount,
642
+ descendantCount: counters.descendantCount,
643
+ waterfallStepExecutions: counters.waterfallStepExecutions,
644
+ parentChildCalls: {},
645
+ };
646
+ },
647
+ async forkInlineChild(input) {
648
+ return createInlineChildGovernor(child, await child.forkChild(input));
649
+ },
650
+ snapshot() {
651
+ const counters = root.snapshot();
652
+ return {
653
+ ...counters,
654
+ ...lineage,
655
+ ancestryPlayIds: [...lineage.ancestryPlayIds],
656
+ ancestryRunIds: [...lineage.ancestryRunIds],
657
+ parentChildCalls: { ...lineage.parentChildCalls },
658
+ };
659
+ },
660
+ };
661
+ return child;
554
662
  }
@@ -70,6 +70,7 @@ export interface PlayRunnerContextConfig {
70
70
  executorToken?: string;
71
71
  baseUrl?: string;
72
72
  receiptGatewayBaseUrl?: string | null;
73
+ runtimeTestFaultHeader?: string | null;
73
74
  vercelProtectionBypassToken?: string | null;
74
75
  integrationMode?: 'live' | 'eval_stub' | 'fixture';
75
76
  orgId?: string;
@@ -141,6 +141,13 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
141
141
  return await Promise.all(outputPromises);
142
142
  }
143
143
 
144
+ async addManyAndFlush(inputs: readonly TInput[]): Promise<TOutput[]> {
145
+ const outputsPromise = this.addMany(inputs);
146
+ const flushPromise = this.flush();
147
+ const [outputs] = await Promise.all([outputsPromise, flushPromise]);
148
+ return outputs;
149
+ }
150
+
144
151
  async flush(): Promise<void> {
145
152
  if (this.#failure) throw this.#failure;
146
153
  this.#clearFlushTimer();