deepline 0.1.206 → 0.1.207
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +25 -16
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +96 -12
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +51 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-lifecycle.ts +70 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +321 -175
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +13 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/adaptive-admission.ts +16 -4
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +161 -81
- package/dist/bundling-sources/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts +26 -9
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +109 -1
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-execution-scope.ts +256 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +17 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +103 -18
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-pg-driver-pg.ts +5 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-compute-usage.ts +74 -0
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +5 -12
- package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +2 -1
- package/dist/bundling-sources/shared_libs/runtime-env.ts +55 -0
- package/dist/cli/index.js +27 -6
- package/dist/cli/index.mjs +27 -6
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/package.json +1 -1
|
@@ -12,7 +12,10 @@ import type {
|
|
|
12
12
|
PlayQueueHint,
|
|
13
13
|
RateStateBackend,
|
|
14
14
|
} from './governor/rate-state-backend';
|
|
15
|
-
import type {
|
|
15
|
+
import type {
|
|
16
|
+
GovernanceSnapshot,
|
|
17
|
+
PlayExecutionGovernor,
|
|
18
|
+
} from './governor/governor';
|
|
16
19
|
import type { BudgetStateBackend } from './governor/budget-state-backend';
|
|
17
20
|
import type { RuntimeMapMemoryLimits } from './map-memory-limits';
|
|
18
21
|
import type { AnyBatchOperationStrategy } from './batching-types';
|
|
@@ -20,6 +23,7 @@ import type { ToolResultMetadataInput } from './tool-result';
|
|
|
20
23
|
import type { PreviousCell } from './cell-staleness';
|
|
21
24
|
import type { MapRowOutcome } from './durability-store';
|
|
22
25
|
import type { WorkReceiptFailureKind } from './work-receipts';
|
|
26
|
+
import type { RunExecutionScope } from './run-execution-scope';
|
|
23
27
|
|
|
24
28
|
export interface RowState {
|
|
25
29
|
results: Map<string, unknown>;
|
|
@@ -470,6 +474,8 @@ export type IntegrationEventWaitHandler = {
|
|
|
470
474
|
};
|
|
471
475
|
|
|
472
476
|
export interface ContextOptions {
|
|
477
|
+
/** Immutable logical and physical execution identity for this context. */
|
|
478
|
+
executionScope?: RunExecutionScope;
|
|
473
479
|
/** Short-lived HMAC-signed internal token for tool callbacks. Required for cloud execution. */
|
|
474
480
|
executorToken?: string;
|
|
475
481
|
baseUrl?: string;
|
|
@@ -568,6 +574,10 @@ export interface ContextOptions {
|
|
|
568
574
|
}) => Promise<void>;
|
|
569
575
|
playId?: string;
|
|
570
576
|
runId?: string;
|
|
577
|
+
/** Physical executor run that owns leases for an inline child context. */
|
|
578
|
+
runtimeReceiptOwnerRunId?: string;
|
|
579
|
+
/** Logical invocation scope for nested step/fetch/runPlay receipt identity. */
|
|
580
|
+
runtimeReceiptScope?: string;
|
|
571
581
|
runAttempt?: number | null;
|
|
572
582
|
/**
|
|
573
583
|
* Optional shared provider-rate state for fleet substrates. Local/test
|
|
@@ -695,6 +705,8 @@ export interface ContextOptions {
|
|
|
695
705
|
* guards accumulate across the nested-play tree. Only set on child contexts.
|
|
696
706
|
*/
|
|
697
707
|
governance?: GovernanceSnapshot;
|
|
708
|
+
/** Internal in-process child view sharing the root run's resource governor. */
|
|
709
|
+
executionGovernor?: PlayExecutionGovernor;
|
|
698
710
|
}
|
|
699
711
|
|
|
700
712
|
export interface PlayCheckpoint {
|
|
@@ -95,8 +95,8 @@ export function createInMemoryAdaptiveAdmission(
|
|
|
95
95
|
finitePositive(options.maxRequestsPerSecond, initialRequestsPerSecond),
|
|
96
96
|
);
|
|
97
97
|
const additiveIncreasePerWindow = finitePositive(
|
|
98
|
-
options.additiveIncreasePerWindow ??
|
|
99
|
-
|
|
98
|
+
options.additiveIncreasePerWindow ?? 1,
|
|
99
|
+
1,
|
|
100
100
|
);
|
|
101
101
|
const cleanWindowMs = finitePositive(options.cleanWindowMs ?? 1_000, 1_000);
|
|
102
102
|
const states = new Map<string, ProviderState>();
|
|
@@ -192,14 +192,26 @@ export function createInMemoryAdaptiveAdmission(
|
|
|
192
192
|
observeProviderBackpressure(input) {
|
|
193
193
|
const state = getExistingState(input.orgId, input.provider);
|
|
194
194
|
if (!state) return;
|
|
195
|
+
const currentTime = now();
|
|
195
196
|
state.rateLimitCount += 1;
|
|
197
|
+
const nextRetryAfterUntilMs =
|
|
198
|
+
currentTime + Math.max(0, input.retryAfterMs);
|
|
199
|
+
if (
|
|
200
|
+
state.retryAfterUntilMs != null &&
|
|
201
|
+
state.retryAfterUntilMs > currentTime
|
|
202
|
+
) {
|
|
203
|
+
state.retryAfterUntilMs = Math.max(
|
|
204
|
+
state.retryAfterUntilMs,
|
|
205
|
+
nextRetryAfterUntilMs,
|
|
206
|
+
);
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
196
209
|
state.hasSeenBackpressure = true;
|
|
197
210
|
state.currentRequestsPerSecond = Math.max(
|
|
198
211
|
minRequestsPerSecond,
|
|
199
212
|
Math.floor(state.currentRequestsPerSecond / 2),
|
|
200
213
|
);
|
|
201
|
-
|
|
202
|
-
state.retryAfterUntilMs = currentTime + Math.max(0, input.retryAfterMs);
|
|
214
|
+
state.retryAfterUntilMs = nextRetryAfterUntilMs;
|
|
203
215
|
state.lastIncreaseAtMs = currentTime;
|
|
204
216
|
state.successCountAtLastIncrease = state.successCount;
|
|
205
217
|
},
|
package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts
CHANGED
|
@@ -18,7 +18,8 @@ import {
|
|
|
18
18
|
* stale-block over-issue to 8 x live runner processes per bucket/window.
|
|
19
19
|
*/
|
|
20
20
|
export const APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE = 8;
|
|
21
|
-
const
|
|
21
|
+
const LEASE_BLOCK_MIN_TTL_MS = 250;
|
|
22
|
+
const LEASE_BLOCK_MAX_TTL_MS = 5_000;
|
|
22
23
|
const MAX_ACQUIRE_SLEEP_MS = 5_000;
|
|
23
24
|
const STORE_FAILURE_RETRY_DELAYS_MS = [100, 250] as const;
|
|
24
25
|
|
|
@@ -42,6 +43,13 @@ interface LeasedBlock {
|
|
|
42
43
|
rules: PacingRule[];
|
|
43
44
|
}
|
|
44
45
|
|
|
46
|
+
interface PendingRelease {
|
|
47
|
+
bucketId: string;
|
|
48
|
+
rules: PacingRule[];
|
|
49
|
+
leaseIds: string[];
|
|
50
|
+
flushing: boolean;
|
|
51
|
+
}
|
|
52
|
+
|
|
45
53
|
interface Options {
|
|
46
54
|
schedulerSchema?: string | null;
|
|
47
55
|
now?: () => number;
|
|
@@ -59,6 +67,8 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
59
67
|
error: string;
|
|
60
68
|
}) => void;
|
|
61
69
|
private readonly blocks = new Map<string, LeasedBlock>();
|
|
70
|
+
private readonly acquisitionTails = new Map<string, Promise<void>>();
|
|
71
|
+
private readonly pendingReleases = new Map<string, PendingRelease>();
|
|
62
72
|
private pendingReleaseFailure: Error | null = null;
|
|
63
73
|
|
|
64
74
|
constructor(context: WorkerRuntimeApiContext, options: Options = {}) {
|
|
@@ -86,80 +96,101 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
86
96
|
|
|
87
97
|
const ordered = [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
|
|
88
98
|
const rulesKey = rulesSignature(ordered);
|
|
89
|
-
|
|
90
|
-
|
|
99
|
+
return await this.withBucketAcquisitionLock(bucketId, async () => {
|
|
100
|
+
const localPermit = this.drawFromBlock(bucketId, rulesKey);
|
|
101
|
+
if (localPermit) return localPermit;
|
|
91
102
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
);
|
|
111
|
-
const leaseIds = Array.isArray(response.leaseIds)
|
|
112
|
-
? response.leaseIds.filter(
|
|
113
|
-
(leaseId): leaseId is string =>
|
|
114
|
-
typeof leaseId === 'string' && leaseId.trim().length > 0,
|
|
115
|
-
)
|
|
116
|
-
: [];
|
|
117
|
-
if (hasConcurrency && leaseIds.length !== response.granted) {
|
|
118
|
-
throw new Error(
|
|
119
|
-
`Rate-state store granted ${response.granted} concurrency permits for ${bucketId} with ${leaseIds.length} lease tokens.`,
|
|
103
|
+
let consecutiveFailures = 0;
|
|
104
|
+
while (true) {
|
|
105
|
+
if (signal?.aborted) {
|
|
106
|
+
throw signal.reason instanceof Error
|
|
107
|
+
? signal.reason
|
|
108
|
+
: new Error('Rate-state acquire aborted.');
|
|
109
|
+
}
|
|
110
|
+
try {
|
|
111
|
+
const response = await acquireRateStateViaAppRuntime(this.context, {
|
|
112
|
+
bucketId,
|
|
113
|
+
rules: ordered,
|
|
114
|
+
requested: APP_RUNTIME_RATE_STATE_LEASE_BLOCK_SIZE,
|
|
115
|
+
schedulerSchema: this.schedulerSchema,
|
|
116
|
+
});
|
|
117
|
+
consecutiveFailures = 0;
|
|
118
|
+
if (response.granted > 0) {
|
|
119
|
+
const hasConcurrency = ordered.some(
|
|
120
|
+
(rule) => rule.maxConcurrency != null,
|
|
120
121
|
);
|
|
122
|
+
const leaseIds = Array.isArray(response.leaseIds)
|
|
123
|
+
? response.leaseIds.filter(
|
|
124
|
+
(leaseId): leaseId is string =>
|
|
125
|
+
typeof leaseId === 'string' && leaseId.trim().length > 0,
|
|
126
|
+
)
|
|
127
|
+
: [];
|
|
128
|
+
if (hasConcurrency && leaseIds.length !== response.granted) {
|
|
129
|
+
throw new Error(
|
|
130
|
+
`Rate-state store granted ${response.granted} concurrency permits for ${bucketId} with ${leaseIds.length} lease tokens.`,
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
if (!hasConcurrency && leaseIds.length > 0) {
|
|
134
|
+
throw new Error(
|
|
135
|
+
`Rate-state store returned ${leaseIds.length} lease tokens for pure-rate bucket ${bucketId}.`,
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
const currentLeaseId = hasConcurrency ? leaseIds[0]! : null;
|
|
139
|
+
const remainingLeaseIds = hasConcurrency ? leaseIds.slice(1) : [];
|
|
140
|
+
const remainingWindowPermits = hasConcurrency
|
|
141
|
+
? 0
|
|
142
|
+
: response.granted - 1;
|
|
143
|
+
if (remainingLeaseIds.length > 0 || remainingWindowPermits > 0) {
|
|
144
|
+
this.mergeBlock(
|
|
145
|
+
bucketId,
|
|
146
|
+
remainingLeaseIds,
|
|
147
|
+
remainingWindowPermits,
|
|
148
|
+
rulesKey,
|
|
149
|
+
ordered,
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
return this.permitFor(bucketId, ordered, currentLeaseId);
|
|
121
153
|
}
|
|
122
|
-
|
|
154
|
+
await this.sleep(
|
|
155
|
+
Math.max(1, Math.min(response.waitMs, MAX_ACQUIRE_SLEEP_MS)),
|
|
156
|
+
);
|
|
157
|
+
} catch (error) {
|
|
158
|
+
// Fail closed. A degraded rate-state store must not grant unlimited
|
|
159
|
+
// provider calls; after this bounded retry ladder the run fails loudly.
|
|
160
|
+
consecutiveFailures += 1;
|
|
161
|
+
const normalized = normalizeError(error);
|
|
162
|
+
this.onStoreFailure({ bucketId, error: normalized.message });
|
|
163
|
+
if (consecutiveFailures > STORE_FAILURE_RETRY_DELAYS_MS.length) {
|
|
123
164
|
throw new Error(
|
|
124
|
-
`Rate-state store
|
|
165
|
+
`Rate-state store unavailable for ${bucketId}; failing closed after ${consecutiveFailures} acquire attempts: ${normalized.message}`,
|
|
125
166
|
);
|
|
126
167
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
// lease token (needs release); pure-rate rules draw a window permit
|
|
130
|
-
// (already decremented server-side, no release).
|
|
131
|
-
const currentLeaseId = hasConcurrency ? leaseIds[0]! : null;
|
|
132
|
-
const remainingLeaseIds = hasConcurrency ? leaseIds.slice(1) : [];
|
|
133
|
-
const remainingWindowPermits = hasConcurrency
|
|
134
|
-
? 0
|
|
135
|
-
: response.granted - 1;
|
|
136
|
-
if (remainingLeaseIds.length > 0 || remainingWindowPermits > 0)
|
|
137
|
-
this.mergeBlock(
|
|
138
|
-
bucketId,
|
|
139
|
-
remainingLeaseIds,
|
|
140
|
-
remainingWindowPermits,
|
|
141
|
-
rulesKey,
|
|
142
|
-
ordered,
|
|
143
|
-
);
|
|
144
|
-
return this.permitFor(bucketId, ordered, currentLeaseId);
|
|
145
|
-
}
|
|
146
|
-
await this.sleep(
|
|
147
|
-
Math.max(1, Math.min(response.waitMs, MAX_ACQUIRE_SLEEP_MS)),
|
|
148
|
-
);
|
|
149
|
-
} catch (error) {
|
|
150
|
-
// Fail closed. A degraded rate-state store must not grant unlimited
|
|
151
|
-
// provider calls; after this bounded retry ladder the run fails loudly.
|
|
152
|
-
consecutiveFailures += 1;
|
|
153
|
-
const normalized = normalizeError(error);
|
|
154
|
-
this.onStoreFailure({ bucketId, error: normalized.message });
|
|
155
|
-
if (consecutiveFailures > STORE_FAILURE_RETRY_DELAYS_MS.length) {
|
|
156
|
-
throw new Error(
|
|
157
|
-
`Rate-state store unavailable for ${bucketId}; failing closed after ${consecutiveFailures} acquire attempts: ${normalized.message}`,
|
|
168
|
+
await this.sleep(
|
|
169
|
+
STORE_FAILURE_RETRY_DELAYS_MS[consecutiveFailures - 1] ?? 250,
|
|
158
170
|
);
|
|
159
171
|
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
172
|
+
}
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
private async withBucketAcquisitionLock<T>(
|
|
177
|
+
bucketId: string,
|
|
178
|
+
operation: () => Promise<T>,
|
|
179
|
+
): Promise<T> {
|
|
180
|
+
const previous = this.acquisitionTails.get(bucketId) ?? Promise.resolve();
|
|
181
|
+
let unlock!: () => void;
|
|
182
|
+
const gate = new Promise<void>((resolve) => {
|
|
183
|
+
unlock = resolve;
|
|
184
|
+
});
|
|
185
|
+
const tail = previous.then(() => gate);
|
|
186
|
+
this.acquisitionTails.set(bucketId, tail);
|
|
187
|
+
await previous;
|
|
188
|
+
try {
|
|
189
|
+
return await operation();
|
|
190
|
+
} finally {
|
|
191
|
+
unlock();
|
|
192
|
+
if (this.acquisitionTails.get(bucketId) === tail) {
|
|
193
|
+
this.acquisitionTails.delete(bucketId);
|
|
163
194
|
}
|
|
164
195
|
}
|
|
165
196
|
}
|
|
@@ -208,7 +239,7 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
208
239
|
rulesKey: string,
|
|
209
240
|
rules: PacingRule[],
|
|
210
241
|
): void {
|
|
211
|
-
const freshExpiresAt = this.now() +
|
|
242
|
+
const freshExpiresAt = this.now() + leasedBlockTtlMs(rules);
|
|
212
243
|
const existing = this.blocks.get(bucketId);
|
|
213
244
|
if (
|
|
214
245
|
existing &&
|
|
@@ -240,8 +271,8 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
240
271
|
if (block.rulesKey !== rulesKey || block.expiresAt <= this.now()) {
|
|
241
272
|
this.blocks.delete(bucketId);
|
|
242
273
|
// Unused window permits are forfeited on TTL expiry (the server already
|
|
243
|
-
// decremented the provider window for them, and the
|
|
244
|
-
//
|
|
274
|
+
// decremented the provider window for them, and the rule-scoped TTL is
|
|
275
|
+
// capped so at most one block's worth is stranded per bucket/window).
|
|
245
276
|
// Concurrency leases must be released so the shared slot count drains.
|
|
246
277
|
this.releaseReserved(bucketId, block.rules, block.leaseIds);
|
|
247
278
|
return null;
|
|
@@ -275,18 +306,54 @@ export class AppRuntimeRateStateBackend implements RateStateBackend {
|
|
|
275
306
|
!rules.some((rule) => rule.maxConcurrency != null)
|
|
276
307
|
)
|
|
277
308
|
return;
|
|
278
|
-
|
|
309
|
+
const ordered = [...rules].sort((a, b) => a.ruleId.localeCompare(b.ruleId));
|
|
310
|
+
const key = `${bucketId}\0${rulesSignature(ordered)}`;
|
|
311
|
+
const pending = this.pendingReleases.get(key) ?? {
|
|
279
312
|
bucketId,
|
|
280
|
-
rules:
|
|
281
|
-
leaseIds: [
|
|
282
|
-
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
313
|
+
rules: ordered,
|
|
314
|
+
leaseIds: [],
|
|
315
|
+
flushing: false,
|
|
316
|
+
};
|
|
317
|
+
pending.leaseIds.push(...leaseIds);
|
|
318
|
+
this.pendingReleases.set(key, pending);
|
|
319
|
+
if (pending.flushing) return;
|
|
320
|
+
pending.flushing = true;
|
|
321
|
+
queueMicrotask(() => void this.flushReleases(key, pending));
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
private async flushReleases(
|
|
325
|
+
key: string,
|
|
326
|
+
pending: PendingRelease,
|
|
327
|
+
): Promise<void> {
|
|
328
|
+
while (pending.leaseIds.length > 0) {
|
|
329
|
+
const leaseIds = pending.leaseIds.splice(0);
|
|
330
|
+
try {
|
|
331
|
+
await releaseRateStateViaAppRuntime(this.context, {
|
|
332
|
+
bucketId: pending.bucketId,
|
|
333
|
+
rules: pending.rules,
|
|
334
|
+
leaseIds,
|
|
335
|
+
schedulerSchema: this.schedulerSchema,
|
|
336
|
+
});
|
|
337
|
+
} catch (error) {
|
|
338
|
+
const normalized = normalizeError(error);
|
|
339
|
+
this.pendingReleaseFailure = new Error(
|
|
340
|
+
`Rate-state release failed for ${pending.bucketId}: ${normalized.message}`,
|
|
341
|
+
);
|
|
342
|
+
this.onStoreFailure({
|
|
343
|
+
bucketId: pending.bucketId,
|
|
344
|
+
error: normalized.message,
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
pending.flushing = false;
|
|
349
|
+
if (pending.leaseIds.length > 0) {
|
|
350
|
+
pending.flushing = true;
|
|
351
|
+
queueMicrotask(() => void this.flushReleases(key, pending));
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
if (this.pendingReleases.get(key) === pending) {
|
|
355
|
+
this.pendingReleases.delete(key);
|
|
356
|
+
}
|
|
290
357
|
}
|
|
291
358
|
|
|
292
359
|
private throwPendingReleaseFailure(): void {
|
|
@@ -307,6 +374,19 @@ function rulesSignature(rules: readonly PacingRule[]): string {
|
|
|
307
374
|
.join('|');
|
|
308
375
|
}
|
|
309
376
|
|
|
377
|
+
function leasedBlockTtlMs(rules: readonly PacingRule[]): number {
|
|
378
|
+
const shortestWindowMs = Math.min(
|
|
379
|
+
...rules
|
|
380
|
+
.map((rule) => rule.windowMs)
|
|
381
|
+
.filter((windowMs) => Number.isFinite(windowMs) && windowMs > 0),
|
|
382
|
+
);
|
|
383
|
+
if (!Number.isFinite(shortestWindowMs)) return LEASE_BLOCK_MIN_TTL_MS;
|
|
384
|
+
return Math.max(
|
|
385
|
+
LEASE_BLOCK_MIN_TTL_MS,
|
|
386
|
+
Math.min(LEASE_BLOCK_MAX_TTL_MS, Math.floor(shortestWindowMs)),
|
|
387
|
+
);
|
|
388
|
+
}
|
|
389
|
+
|
|
310
390
|
function normalizeError(error: unknown): Error {
|
|
311
391
|
return error instanceof Error ? error : new Error(String(error));
|
|
312
392
|
}
|
package/dist/bundling-sources/shared_libs/play-runtime/governor/coordinator-rate-state-backend.ts
CHANGED
|
@@ -21,9 +21,9 @@ import {
|
|
|
21
21
|
* hello-world latency baseline. Instead the backend LEASES SMALL PERMIT BLOCKS:
|
|
22
22
|
* one `/rate-acquire` round-trip debits up to {@link LEASE_BLOCK_SIZE} permits
|
|
23
23
|
* from the global window, and subsequent acquires draw from the local block
|
|
24
|
-
* until it is exhausted or its
|
|
25
|
-
* roughly `calls / LEASE_BLOCK_SIZE` while keeping over-issuance bounded by
|
|
26
|
-
* block per isolate per window.
|
|
24
|
+
* until it is exhausted or its rule-scoped TTL expires. This bounds round-trips
|
|
25
|
+
* to roughly `calls / LEASE_BLOCK_SIZE` while keeping over-issuance bounded by
|
|
26
|
+
* one block per isolate per window.
|
|
27
27
|
*
|
|
28
28
|
* Fail-open: if the coordinator is unreachable the backend logs once and
|
|
29
29
|
* PROCEEDS (grants the permit) rather than stalling the run, matching the
|
|
@@ -35,11 +35,14 @@ import {
|
|
|
35
35
|
/** Permits leased per round-trip. Tuned to amortize the DO hop, not to batch. */
|
|
36
36
|
const LEASE_BLOCK_SIZE = 16;
|
|
37
37
|
/**
|
|
38
|
-
*
|
|
39
|
-
* but if it is held past
|
|
40
|
-
* of a rolled-over window.
|
|
38
|
+
* Min/max age of a leased block. A leased permit debited the global window
|
|
39
|
+
* already, but if it is held well past the provider window it could let an
|
|
40
|
+
* isolate run ahead of a rolled-over window. The effective TTL is derived from
|
|
41
|
+
* the shortest rule window so low-RPS pure-rate providers can draw already-
|
|
42
|
+
* debited permits throughout a normal window instead of burning most blocks.
|
|
41
43
|
*/
|
|
42
|
-
const
|
|
44
|
+
const LEASE_BLOCK_MIN_TTL_MS = 250;
|
|
45
|
+
const LEASE_BLOCK_MAX_TTL_MS = 5_000;
|
|
43
46
|
/** Cap one coordinator-provided sleep hint; total wait is bounded by run timeout. */
|
|
44
47
|
const MAX_ACQUIRE_SLEEP_MS = 5_000;
|
|
45
48
|
|
|
@@ -148,7 +151,7 @@ export class CoordinatorRateStateBackend implements RateStateBackend {
|
|
|
148
151
|
// Consume one for this call; cache the rest as a short-lived block.
|
|
149
152
|
const remaining = response.granted - 1;
|
|
150
153
|
if (remaining > 0) {
|
|
151
|
-
this.mergeBlock(bucketId, remaining, rulesKey);
|
|
154
|
+
this.mergeBlock(bucketId, remaining, rulesKey, rules);
|
|
152
155
|
}
|
|
153
156
|
return noopPacingPermit();
|
|
154
157
|
}
|
|
@@ -196,8 +199,9 @@ export class CoordinatorRateStateBackend implements RateStateBackend {
|
|
|
196
199
|
bucketId: string,
|
|
197
200
|
remaining: number,
|
|
198
201
|
rulesKey: string,
|
|
202
|
+
rules: readonly PacingRule[],
|
|
199
203
|
): void {
|
|
200
|
-
const freshExpiresAt = this.now() +
|
|
204
|
+
const freshExpiresAt = this.now() + leasedBlockTtlMs(rules);
|
|
201
205
|
const existing = this.blocks.get(bucketId);
|
|
202
206
|
if (
|
|
203
207
|
existing &&
|
|
@@ -229,3 +233,16 @@ export class CoordinatorRateStateBackend implements RateStateBackend {
|
|
|
229
233
|
return true;
|
|
230
234
|
}
|
|
231
235
|
}
|
|
236
|
+
|
|
237
|
+
function leasedBlockTtlMs(rules: readonly PacingRule[]): number {
|
|
238
|
+
const shortestWindowMs = Math.min(
|
|
239
|
+
...rules
|
|
240
|
+
.map((rule) => rule.windowMs)
|
|
241
|
+
.filter((windowMs) => Number.isFinite(windowMs) && windowMs > 0),
|
|
242
|
+
);
|
|
243
|
+
if (!Number.isFinite(shortestWindowMs)) return LEASE_BLOCK_MIN_TTL_MS;
|
|
244
|
+
return Math.max(
|
|
245
|
+
LEASE_BLOCK_MIN_TTL_MS,
|
|
246
|
+
Math.min(LEASE_BLOCK_MAX_TTL_MS, Math.floor(shortestWindowMs)),
|
|
247
|
+
);
|
|
248
|
+
}
|
|
@@ -147,6 +147,16 @@ export interface PlayExecutionGovernor {
|
|
|
147
147
|
childRunId: string;
|
|
148
148
|
}): Promise<GovernanceSnapshot>;
|
|
149
149
|
|
|
150
|
+
/**
|
|
151
|
+
* Fork an in-process child view. The child gets its own lineage while sharing
|
|
152
|
+
* the parent's row/tool/child semaphores, adaptive admission, and global
|
|
153
|
+
* budget counters.
|
|
154
|
+
*/
|
|
155
|
+
forkInlineChild(input: {
|
|
156
|
+
childPlayName: string;
|
|
157
|
+
childRunId: string;
|
|
158
|
+
}): Promise<PlayExecutionGovernor>;
|
|
159
|
+
|
|
150
160
|
/** Effective row concurrency: explicit request clamped to [1, rowMax], else default. */
|
|
151
161
|
resolveRowConcurrency(requested?: number): number;
|
|
152
162
|
|
|
@@ -364,7 +374,7 @@ export function createPlayExecutionGovernor(
|
|
|
364
374
|
}
|
|
365
375
|
}
|
|
366
376
|
|
|
367
|
-
|
|
377
|
+
const governor: PlayExecutionGovernor = {
|
|
368
378
|
adapter: input.adapter,
|
|
369
379
|
policy,
|
|
370
380
|
|
|
@@ -510,6 +520,13 @@ export function createPlayExecutionGovernor(
|
|
|
510
520
|
};
|
|
511
521
|
},
|
|
512
522
|
|
|
523
|
+
async forkInlineChild(childInput) {
|
|
524
|
+
return createInlineChildGovernor(
|
|
525
|
+
governor,
|
|
526
|
+
await governor.forkChild(childInput),
|
|
527
|
+
);
|
|
528
|
+
},
|
|
529
|
+
|
|
513
530
|
resolveRowConcurrency: (requested) =>
|
|
514
531
|
resolveRowConcurrency(policy, requested),
|
|
515
532
|
|
|
@@ -551,4 +568,95 @@ export function createPlayExecutionGovernor(
|
|
|
551
568
|
parentChildCalls: { ...state.parentChildCalls },
|
|
552
569
|
}),
|
|
553
570
|
};
|
|
571
|
+
return governor;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
function createInlineChildGovernor(
|
|
575
|
+
root: PlayExecutionGovernor,
|
|
576
|
+
initial: GovernanceSnapshot,
|
|
577
|
+
): PlayExecutionGovernor {
|
|
578
|
+
const lineage = {
|
|
579
|
+
rootRunId: initial.rootRunId,
|
|
580
|
+
currentRunId: initial.currentRunId,
|
|
581
|
+
currentPlayId: initial.currentPlayId,
|
|
582
|
+
ancestryPlayIds: [...initial.ancestryPlayIds],
|
|
583
|
+
ancestryRunIds: [...initial.ancestryRunIds],
|
|
584
|
+
callDepth: initial.callDepth,
|
|
585
|
+
parentChildCalls: { ...initial.parentChildCalls },
|
|
586
|
+
};
|
|
587
|
+
|
|
588
|
+
const child: PlayExecutionGovernor = {
|
|
589
|
+
adapter: root.adapter,
|
|
590
|
+
policy: root.policy,
|
|
591
|
+
acquireRowSlot: (opts) => root.acquireRowSlot(opts),
|
|
592
|
+
acquireChildSubmitSlot: (opts) => root.acquireChildSubmitSlot(opts),
|
|
593
|
+
acquireToolSlot: (toolId, opts) => root.acquireToolSlot(toolId, opts),
|
|
594
|
+
suggestedParallelism: (toolId, fallback) =>
|
|
595
|
+
root.suggestedParallelism(toolId, fallback),
|
|
596
|
+
chargeBudget: (kind, amount) => root.chargeBudget(kind, amount),
|
|
597
|
+
resolveRowConcurrency: (requested) => root.resolveRowConcurrency(requested),
|
|
598
|
+
reportProviderBackpressure: (input) =>
|
|
599
|
+
root.reportProviderBackpressure(input),
|
|
600
|
+
observeProviderSuccess: (input) => root.observeProviderSuccess(input),
|
|
601
|
+
observeToolSuccess: (input) => root.observeToolSuccess(input),
|
|
602
|
+
async forkChild(input) {
|
|
603
|
+
if (lineage.ancestryPlayIds.includes(input.childPlayName)) {
|
|
604
|
+
throw new Error(
|
|
605
|
+
`Recursive play graph detected: ${[
|
|
606
|
+
...lineage.ancestryPlayIds,
|
|
607
|
+
input.childPlayName,
|
|
608
|
+
].join(' -> ')}`,
|
|
609
|
+
);
|
|
610
|
+
}
|
|
611
|
+
const nextDepth = lineage.callDepth + 1;
|
|
612
|
+
if (nextDepth > child.policy.budgets.maxPlayCallDepth) {
|
|
613
|
+
throw new GovernorBudgetError(
|
|
614
|
+
'playDepth',
|
|
615
|
+
nextDepth,
|
|
616
|
+
child.policy.budgets.maxPlayCallDepth,
|
|
617
|
+
);
|
|
618
|
+
}
|
|
619
|
+
const nextParent =
|
|
620
|
+
(lineage.parentChildCalls[lineage.currentPlayId] ?? 0) + 1;
|
|
621
|
+
if (nextParent > child.policy.budgets.maxChildPlayCallsPerParent) {
|
|
622
|
+
throw new GovernorBudgetError(
|
|
623
|
+
'childPerParent',
|
|
624
|
+
nextParent,
|
|
625
|
+
child.policy.budgets.maxChildPlayCallsPerParent,
|
|
626
|
+
);
|
|
627
|
+
}
|
|
628
|
+
await root.chargeBudget('playCall');
|
|
629
|
+
await root.chargeBudget('descendant');
|
|
630
|
+
lineage.parentChildCalls[lineage.currentPlayId] = nextParent;
|
|
631
|
+
const counters = root.snapshot();
|
|
632
|
+
return {
|
|
633
|
+
rootRunId: lineage.rootRunId,
|
|
634
|
+
currentRunId: input.childRunId,
|
|
635
|
+
currentPlayId: input.childPlayName,
|
|
636
|
+
ancestryPlayIds: [...lineage.ancestryPlayIds, input.childPlayName],
|
|
637
|
+
ancestryRunIds: [...lineage.ancestryRunIds, input.childRunId],
|
|
638
|
+
callDepth: nextDepth,
|
|
639
|
+
playCallCount: counters.playCallCount,
|
|
640
|
+
toolCallCount: counters.toolCallCount,
|
|
641
|
+
retryCount: counters.retryCount,
|
|
642
|
+
descendantCount: counters.descendantCount,
|
|
643
|
+
waterfallStepExecutions: counters.waterfallStepExecutions,
|
|
644
|
+
parentChildCalls: {},
|
|
645
|
+
};
|
|
646
|
+
},
|
|
647
|
+
async forkInlineChild(input) {
|
|
648
|
+
return createInlineChildGovernor(child, await child.forkChild(input));
|
|
649
|
+
},
|
|
650
|
+
snapshot() {
|
|
651
|
+
const counters = root.snapshot();
|
|
652
|
+
return {
|
|
653
|
+
...counters,
|
|
654
|
+
...lineage,
|
|
655
|
+
ancestryPlayIds: [...lineage.ancestryPlayIds],
|
|
656
|
+
ancestryRunIds: [...lineage.ancestryRunIds],
|
|
657
|
+
parentChildCalls: { ...lineage.parentChildCalls },
|
|
658
|
+
};
|
|
659
|
+
},
|
|
660
|
+
};
|
|
661
|
+
return child;
|
|
554
662
|
}
|
|
@@ -70,6 +70,7 @@ export interface PlayRunnerContextConfig {
|
|
|
70
70
|
executorToken?: string;
|
|
71
71
|
baseUrl?: string;
|
|
72
72
|
receiptGatewayBaseUrl?: string | null;
|
|
73
|
+
runtimeTestFaultHeader?: string | null;
|
|
73
74
|
vercelProtectionBypassToken?: string | null;
|
|
74
75
|
integrationMode?: 'live' | 'eval_stub' | 'fixture';
|
|
75
76
|
orgId?: string;
|
|
@@ -141,6 +141,13 @@ export class RuntimeReceiptCompletionSink<TInput, TOutput> {
|
|
|
141
141
|
return await Promise.all(outputPromises);
|
|
142
142
|
}
|
|
143
143
|
|
|
144
|
+
async addManyAndFlush(inputs: readonly TInput[]): Promise<TOutput[]> {
|
|
145
|
+
const outputsPromise = this.addMany(inputs);
|
|
146
|
+
const flushPromise = this.flush();
|
|
147
|
+
const [outputs] = await Promise.all([outputsPromise, flushPromise]);
|
|
148
|
+
return outputs;
|
|
149
|
+
}
|
|
150
|
+
|
|
144
151
|
async flush(): Promise<void> {
|
|
145
152
|
if (this.#failure) throw this.#failure;
|
|
146
153
|
this.#clearFlushTimer();
|