deepline 0.3.49 → 0.3.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/index.ts +1 -0
- package/dist/bundling-sources/sdk/src/play.ts +1 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +106 -89
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +0 -2
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +52 -21
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +45 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +11 -3
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +0 -2
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/secret-capability.ts +5 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +1 -14
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/contract.ts +17 -7
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +0 -1
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +112 -5
- package/dist/bundling-sources/shared_libs/security/safe-fetch.ts +6 -1
- package/dist/cli/index.js +837 -3147
- package/dist/cli/index.mjs +704 -3010
- package/dist/{compiler-manifest-CbzdZrJj.d.mts → compiler-manifest-B47AA7As.d.mts} +46 -4
- package/dist/{compiler-manifest-CbzdZrJj.d.ts → compiler-manifest-B47AA7As.d.ts} +46 -4
- package/dist/index.d.mts +151 -149
- package/dist/index.d.ts +151 -149
- package/dist/index.js +1515 -117
- package/dist/index.mjs +1514 -117
- package/dist/install-integrity.json +2 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +5 -2614
- package/package.json +2 -1
|
@@ -80,6 +80,7 @@ import {
|
|
|
80
80
|
} from '../../shared_libs/play-runtime/tool-result.js';
|
|
81
81
|
export { readValue, readList } from '../../shared_libs/play-runtime/tool-result.js';
|
|
82
82
|
import { createDeferredPlayDataset } from '../../shared_libs/plays/dataset.js';
|
|
83
|
+
export { CtxFetchHttpError } from '../../shared_libs/plays/authoring-contract.js';
|
|
83
84
|
import {
|
|
84
85
|
QUERY_RESULT_DATASET_PAGE_SIZE,
|
|
85
86
|
isCustomerDbDatasetTool,
|
|
@@ -199,7 +199,7 @@ export const SDK_RELEASE = {
|
|
|
199
199
|
// available at toolResponse.rawV2 while toolResponse.raw and all declared
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
|
-
version: '0.3.
|
|
202
|
+
version: '0.3.51',
|
|
203
203
|
updateSummary:
|
|
204
204
|
'Automatic CLI updates are now enabled by default. To opt out, run `deepline settings autoupdate off`; use `deepline settings autoupdate on` to re-enable updates or `deepline settings autoupdate pin <version>` to hold an exact release. This release also adds raw-v2 tool responses at toolResponse.rawV2 while preserving existing toolResponse.raw and declared getters.',
|
|
205
205
|
packageCapabilities: {
|
|
@@ -657,6 +657,8 @@ export interface PlayRunPackage {
|
|
|
657
657
|
playName: string;
|
|
658
658
|
status: string;
|
|
659
659
|
dashboardUrl?: string;
|
|
660
|
+
/** Durable acceptance time: the run record was created. */
|
|
661
|
+
acceptedAt?: number | null;
|
|
660
662
|
updatedAt?: number | null;
|
|
661
663
|
startedAt?: number | null;
|
|
662
664
|
finishedAt?: number | null;
|
|
@@ -652,6 +652,7 @@ function isRetryableAppRuntimeAction(
|
|
|
652
652
|
action === 'create_signed_staged_file_url' ||
|
|
653
653
|
action === 'get_runtime_step_receipt' ||
|
|
654
654
|
action === 'get_runtime_step_receipts' ||
|
|
655
|
+
action === 'governor_budget_charge' ||
|
|
655
656
|
action === 'heartbeat_runtime_step_receipts' ||
|
|
656
657
|
action === 'claim_runtime_step_receipt' ||
|
|
657
658
|
action === 'claim_runtime_step_receipts' ||
|
|
@@ -178,6 +178,7 @@ import {
|
|
|
178
178
|
resolveDurableCallCachePolicy,
|
|
179
179
|
} from './durable-call-cache';
|
|
180
180
|
import {
|
|
181
|
+
CtxFetchHttpError,
|
|
181
182
|
PLAY_AUTHORING_CONTRACT_EDITION,
|
|
182
183
|
normalizePlayAuthoringCustomerDbStatement,
|
|
183
184
|
validateOptionalPlayAuthoringField,
|
|
@@ -287,6 +288,7 @@ import {
|
|
|
287
288
|
isSecretAuthInput,
|
|
288
289
|
isPlaintextSecretPromise,
|
|
289
290
|
isSecretHandle,
|
|
291
|
+
isSecretValue,
|
|
290
292
|
secretAuthEntries,
|
|
291
293
|
secretAuthHeaderMarkers,
|
|
292
294
|
valueContainsSecret,
|
|
@@ -2040,8 +2042,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2040
2042
|
#options: ContextOptions;
|
|
2041
2043
|
private readonly executionScope: RunExecutionScope;
|
|
2042
2044
|
private logBuffer: string[] = [];
|
|
2043
|
-
private fixtureProviderPacingBypassLogged = false;
|
|
2044
|
-
private fixtureProviderPacingEnforcementLogged = false;
|
|
2045
2045
|
private checkpoint: PlayCheckpoint;
|
|
2046
2046
|
private readonly durableCallCacheEpochMs: number;
|
|
2047
2047
|
/**
|
|
@@ -2866,7 +2866,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2866
2866
|
this.secretRedactor.register(value);
|
|
2867
2867
|
return value;
|
|
2868
2868
|
}
|
|
2869
|
-
|
|
2869
|
+
if (isSecretHandle(secret) || isSecretValue(secret)) {
|
|
2870
|
+
return this.resolveSecretValue(secret);
|
|
2871
|
+
}
|
|
2872
|
+
throw new Error(
|
|
2873
|
+
'ctx.secrets auth requires a resolved string or an approved secret value.',
|
|
2874
|
+
);
|
|
2870
2875
|
}
|
|
2871
2876
|
|
|
2872
2877
|
private async resolveSecretValue(secret: SecretValue): Promise<string> {
|
|
@@ -4253,6 +4258,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4253
4258
|
) => T;
|
|
4254
4259
|
onClaimedResult?: (output: T, receiptKey: string) => T;
|
|
4255
4260
|
shouldPersistFailure?: (error: unknown) => boolean;
|
|
4261
|
+
transient?: boolean;
|
|
4256
4262
|
markRunningBeforeExecute?: boolean;
|
|
4257
4263
|
requiresExecutionLock?: boolean;
|
|
4258
4264
|
executionLockTtlMs?: number;
|
|
@@ -4262,6 +4268,28 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4262
4268
|
}) => Promise<T>;
|
|
4263
4269
|
},
|
|
4264
4270
|
): Promise<T> {
|
|
4271
|
+
let releaseExternalCallSlot: (() => void) | null = null;
|
|
4272
|
+
const execute = async (leaseId: string | null): Promise<T> =>
|
|
4273
|
+
await opts.execute({
|
|
4274
|
+
leaseId,
|
|
4275
|
+
retainExternalCallSlot: (release) => {
|
|
4276
|
+
if (releaseExternalCallSlot) {
|
|
4277
|
+
release();
|
|
4278
|
+
throw new Error(
|
|
4279
|
+
`ctx.${operation}(${id}) attempted to retain more than one external-call slot.`,
|
|
4280
|
+
);
|
|
4281
|
+
}
|
|
4282
|
+
releaseExternalCallSlot = release;
|
|
4283
|
+
},
|
|
4284
|
+
});
|
|
4285
|
+
if (opts.transient === true) {
|
|
4286
|
+
try {
|
|
4287
|
+
return await execute(null);
|
|
4288
|
+
} finally {
|
|
4289
|
+
const release = releaseExternalCallSlot as (() => void) | null;
|
|
4290
|
+
release?.();
|
|
4291
|
+
}
|
|
4292
|
+
}
|
|
4265
4293
|
const stalePolicy = resolveDurableCallCachePolicy(
|
|
4266
4294
|
opts.staleAfterSeconds,
|
|
4267
4295
|
operation === 'step'
|
|
@@ -4281,7 +4309,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4281
4309
|
staleAfterSeconds: stalePolicy.staleAfterSeconds,
|
|
4282
4310
|
cacheEpochMs: this.durableCallCacheEpochMs,
|
|
4283
4311
|
});
|
|
4284
|
-
let releaseExternalCallSlot: (() => void) | null = null;
|
|
4285
4312
|
try {
|
|
4286
4313
|
return await executeWithDurableRuntimeReceipt<T>({
|
|
4287
4314
|
operation,
|
|
@@ -4309,19 +4336,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4309
4336
|
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
4310
4337
|
formatError: (error) => this.formatRuntimeError(error),
|
|
4311
4338
|
log: (message) => this.log(message),
|
|
4312
|
-
execute: ({ leaseId }) =>
|
|
4313
|
-
opts.execute({
|
|
4314
|
-
leaseId,
|
|
4315
|
-
retainExternalCallSlot: (release) => {
|
|
4316
|
-
if (releaseExternalCallSlot) {
|
|
4317
|
-
release();
|
|
4318
|
-
throw new Error(
|
|
4319
|
-
`ctx.${operation}(${id}) attempted to retain more than one external-call slot.`,
|
|
4320
|
-
);
|
|
4321
|
-
}
|
|
4322
|
-
releaseExternalCallSlot = release;
|
|
4323
|
-
},
|
|
4324
|
-
}),
|
|
4339
|
+
execute: ({ leaseId }) => execute(leaseId),
|
|
4325
4340
|
});
|
|
4326
4341
|
} finally {
|
|
4327
4342
|
const release = releaseExternalCallSlot as (() => void) | null;
|
|
@@ -8698,13 +8713,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8698
8713
|
}
|
|
8699
8714
|
}
|
|
8700
8715
|
|
|
8701
|
-
private fixtureProviderPacingDisabled(): boolean {
|
|
8702
|
-
return (
|
|
8703
|
-
this.#options.integrationMode === 'fixture' &&
|
|
8704
|
-
this.#options.enforceFixtureProviderPacing !== true
|
|
8705
|
-
);
|
|
8706
|
-
}
|
|
8707
|
-
|
|
8708
8716
|
private toolDispatchLane(request: ToolCallRequest): {
|
|
8709
8717
|
key: string;
|
|
8710
8718
|
readyCount: number;
|
|
@@ -10215,9 +10223,31 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10215
10223
|
...secretHeaderMarkers,
|
|
10216
10224
|
},
|
|
10217
10225
|
row: rowFetchScope,
|
|
10226
|
+
// Edition 5 changes non-2xx responses from cached values to
|
|
10227
|
+
// non-cacheable errors. Partition those receipts from editions
|
|
10228
|
+
// 1–4 so a republished artifact makes a fresh request instead of
|
|
10229
|
+
// inheriting a legacy error record.
|
|
10230
|
+
...(this.currentAuthoringContractEdition >= 5
|
|
10231
|
+
? {
|
|
10232
|
+
authoringContractEdition:
|
|
10233
|
+
this.currentAuthoringContractEdition,
|
|
10234
|
+
}
|
|
10235
|
+
: {}),
|
|
10218
10236
|
}),
|
|
10219
10237
|
),
|
|
10220
10238
|
staleAfterSeconds: options?.staleAfterSeconds,
|
|
10239
|
+
transient: options?.transient === true,
|
|
10240
|
+
onRecovered: (output) => {
|
|
10241
|
+
if (!output.ok && this.currentAuthoringContractEdition >= 5) {
|
|
10242
|
+
throw new CtxFetchHttpError(output);
|
|
10243
|
+
}
|
|
10244
|
+
return output;
|
|
10245
|
+
},
|
|
10246
|
+
shouldPersistFailure: (error) =>
|
|
10247
|
+
!(
|
|
10248
|
+
error instanceof CtxFetchHttpError &&
|
|
10249
|
+
this.currentAuthoringContractEdition >= 5
|
|
10250
|
+
),
|
|
10221
10251
|
execute: async ({ retainExternalCallSlot }) => {
|
|
10222
10252
|
const method = (init.method ?? 'GET').toUpperCase();
|
|
10223
10253
|
const secretHeaders = await this.resolveSecretAuth(secretAuth);
|
|
@@ -10225,6 +10255,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10225
10255
|
...normalizeFetchHeaders(init.headers),
|
|
10226
10256
|
...secretHeaders,
|
|
10227
10257
|
};
|
|
10258
|
+
if (headers['user-agent'] === undefined) {
|
|
10259
|
+
headers['user-agent'] = 'Deepline Play Runtime';
|
|
10260
|
+
}
|
|
10228
10261
|
const fetchInit = { ...init, headers };
|
|
10229
10262
|
delete fetchInit.auth;
|
|
10230
10263
|
const boundaryId = this.durableBoundaryId(
|
|
@@ -10243,7 +10276,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10243
10276
|
);
|
|
10244
10277
|
|
|
10245
10278
|
const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
|
|
10246
|
-
|
|
10279
|
+
// A transient request deliberately has no durable recovery path, so
|
|
10280
|
+
// its response stays in the running Play rather than a checkpoint.
|
|
10281
|
+
if (
|
|
10282
|
+
options?.transient !== true &&
|
|
10283
|
+
existing?.kind === 'fetch' &&
|
|
10284
|
+
'output' in existing
|
|
10285
|
+
) {
|
|
10247
10286
|
this.log(`ctx.fetch(${url}): recovered response from checkpoint`);
|
|
10248
10287
|
if (this.durableDirectToolResultsBackedByReceipts) {
|
|
10249
10288
|
// The outer durable receipt is the replay authority in hosted
|
|
@@ -10324,6 +10363,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10324
10363
|
fetchImpl: this.#options.fetchImpl,
|
|
10325
10364
|
sensitiveHeaders: Object.keys(secretHeaderMarkers),
|
|
10326
10365
|
stripHeadersOnCrossOriginRedirect: true,
|
|
10366
|
+
headersOnCrossOriginRedirect: {
|
|
10367
|
+
'user-agent': 'Deepline Play Runtime',
|
|
10368
|
+
},
|
|
10327
10369
|
}),
|
|
10328
10370
|
});
|
|
10329
10371
|
bodyText = await readCtxFetchBody({
|
|
@@ -10375,6 +10417,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10375
10417
|
`ctx.fetch(${method} ${url}) failed while reading the response body.`,
|
|
10376
10418
|
);
|
|
10377
10419
|
}
|
|
10420
|
+
const rawJson = parseJsonOrNull(bodyText);
|
|
10378
10421
|
const redactedBodyText = this.secretRedactor.redactString(bodyText);
|
|
10379
10422
|
const output: PlayFetchResponse = {
|
|
10380
10423
|
ok: response.ok,
|
|
@@ -10385,12 +10428,21 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10385
10428
|
Object.fromEntries(response.headers.entries()),
|
|
10386
10429
|
) as Record<string, string>,
|
|
10387
10430
|
bodyText: redactedBodyText,
|
|
10388
|
-
json: this.secretRedactor.redactKnownSecrets(
|
|
10389
|
-
parseJsonOrNull(bodyText),
|
|
10390
|
-
),
|
|
10431
|
+
json: this.secretRedactor.redactKnownSecrets(rawJson),
|
|
10391
10432
|
};
|
|
10392
10433
|
|
|
10393
|
-
|
|
10434
|
+
// Edition 5 adopts normal fetch semantics: a non-2xx response is
|
|
10435
|
+
// a failed durable operation. Throw before checkpoint/receipt
|
|
10436
|
+
// completion so the failure is never cached. Editions 1–4 retain
|
|
10437
|
+
// their response-record behavior for the same upstream response.
|
|
10438
|
+
if (!output.ok && this.currentAuthoringContractEdition >= 5) {
|
|
10439
|
+
throw new CtxFetchHttpError(output);
|
|
10440
|
+
}
|
|
10441
|
+
|
|
10442
|
+
if (
|
|
10443
|
+
options?.transient !== true &&
|
|
10444
|
+
!this.durableDirectToolResultsBackedByReceipts
|
|
10445
|
+
) {
|
|
10394
10446
|
this.checkpoint.resolvedBoundaries = {
|
|
10395
10447
|
...(this.checkpoint.resolvedBoundaries ?? {}),
|
|
10396
10448
|
[boundaryId]: {
|
|
@@ -11065,12 +11117,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11065
11117
|
this.governor.policy.concurrency.toolCalls,
|
|
11066
11118
|
Math.max(
|
|
11067
11119
|
1,
|
|
11068
|
-
this.
|
|
11069
|
-
|
|
11070
|
-
|
|
11071
|
-
|
|
11072
|
-
this.governor.policy.concurrency.toolCalls,
|
|
11073
|
-
),
|
|
11120
|
+
await this.resourceGovernor.suggestedToolParallelism(
|
|
11121
|
+
toolId,
|
|
11122
|
+
this.governor.policy.concurrency.toolCalls,
|
|
11123
|
+
),
|
|
11074
11124
|
),
|
|
11075
11125
|
);
|
|
11076
11126
|
const dispatchOwned = async (
|
|
@@ -11226,7 +11276,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11226
11276
|
const batchParallelismCeiling =
|
|
11227
11277
|
this.governor.policy.pacing.workerToolBatchDefaultParallelism;
|
|
11228
11278
|
const batchSize =
|
|
11229
|
-
compiledBatches.length > 0
|
|
11279
|
+
compiledBatches.length > 0
|
|
11230
11280
|
? await this.resourceGovernor.suggestedToolParallelism(
|
|
11231
11281
|
compiledBatches[0]!.batchOperation,
|
|
11232
11282
|
batchParallelismCeiling,
|
|
@@ -12501,8 +12551,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12501
12551
|
const batchParallelismCeiling =
|
|
12502
12552
|
this.governor.policy.pacing.workerToolBatchDefaultParallelism;
|
|
12503
12553
|
const batchSize =
|
|
12504
|
-
compiledBatches.length > 0
|
|
12505
|
-
!this.fixtureProviderPacingDisabled()
|
|
12554
|
+
compiledBatches.length > 0
|
|
12506
12555
|
? await this.resourceGovernor.suggestedToolParallelism(
|
|
12507
12556
|
compiledBatches[0]!.batchOperation,
|
|
12508
12557
|
batchParallelismCeiling,
|
|
@@ -12756,23 +12805,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12756
12805
|
void flushCompletionBuffer();
|
|
12757
12806
|
}, 0);
|
|
12758
12807
|
});
|
|
12759
|
-
//
|
|
12760
|
-
//
|
|
12761
|
-
//
|
|
12762
|
-
// the
|
|
12763
|
-
//
|
|
12764
|
-
// provider's RPS/maxConcurrency pacing rules (see
|
|
12765
|
-
// suggestedParallelism in governor.ts), floored at 1 and capped by
|
|
12766
|
-
// the global tool-call concurrency ceiling. The pacer still gates
|
|
12767
|
-
// each in-flight call, so this only trims the launch burst.
|
|
12808
|
+
// Bound direct map dispatch separately from ordinary provider
|
|
12809
|
+
// residency. This prevents a blocked first chunk from launching
|
|
12810
|
+
// another chunk ahead of a provider's admitted start rate, while
|
|
12811
|
+
// preserving the wider resident-call capacity used by batched and
|
|
12812
|
+
// high-latency providers.
|
|
12768
12813
|
const toolCallConcurrencyCeiling =
|
|
12769
12814
|
this.governor.policy.concurrency.toolCalls;
|
|
12770
|
-
const shapedToolParallelism =
|
|
12771
|
-
|
|
12772
|
-
|
|
12773
|
-
|
|
12774
|
-
|
|
12775
|
-
);
|
|
12815
|
+
const shapedToolParallelism =
|
|
12816
|
+
await this.resourceGovernor.suggestedDirectToolParallelism(
|
|
12817
|
+
toolId,
|
|
12818
|
+
toolCallConcurrencyCeiling,
|
|
12819
|
+
);
|
|
12776
12820
|
const dispatchWidth = Math.min(
|
|
12777
12821
|
toolCallConcurrencyCeiling,
|
|
12778
12822
|
Math.max(1, shapedToolParallelism),
|
|
@@ -13339,44 +13383,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13339
13383
|
// is the fetch itself, so independently delayed runners cannot
|
|
13340
13384
|
// compress real provider arrivals after their tickets.
|
|
13341
13385
|
const protectionHeaders = await this.vercelProtectionHeaders();
|
|
13342
|
-
//
|
|
13343
|
-
//
|
|
13344
|
-
//
|
|
13345
|
-
//
|
|
13346
|
-
// latter as live would make fixture runs both slow and
|
|
13347
|
-
// unauditable even though the app never dispatches a provider
|
|
13348
|
-
// request.
|
|
13386
|
+
// Fixture responses simulate provider residence, but provider
|
|
13387
|
+
// admission remains real. This makes fixture volume results a
|
|
13388
|
+
// faithful measurement of the runtime controller without
|
|
13389
|
+
// spending provider credits.
|
|
13349
13390
|
const fixtureExecution =
|
|
13350
13391
|
this.#options.integrationMode === 'fixture';
|
|
13351
|
-
const
|
|
13352
|
-
|
|
13353
|
-
|
|
13354
|
-
|
|
13355
|
-
|
|
13356
|
-
if (
|
|
13357
|
-
fixtureOnlyExecution &&
|
|
13358
|
-
!this.fixtureProviderPacingBypassLogged
|
|
13359
|
-
) {
|
|
13360
|
-
this.fixtureProviderPacingBypassLogged = true;
|
|
13361
|
-
this.log(
|
|
13362
|
-
'Fixture mode: provider pacing bypassed because no provider request is dispatched.',
|
|
13363
|
-
);
|
|
13364
|
-
}
|
|
13365
|
-
if (
|
|
13366
|
-
enforceFixtureProviderPacing &&
|
|
13367
|
-
!this.fixtureProviderPacingEnforcementLogged
|
|
13368
|
-
) {
|
|
13369
|
-
this.fixtureProviderPacingEnforcementLogged = true;
|
|
13370
|
-
this.log(
|
|
13371
|
-
'Fixture mode: provider pacing explicitly enforced for production-parity testing.',
|
|
13372
|
-
);
|
|
13373
|
-
}
|
|
13374
|
-
const providerPermit = fixtureOnlyExecution
|
|
13375
|
-
? { release() {} }
|
|
13376
|
-
: await this.resourceGovernor.acquireProviderPermit({
|
|
13377
|
-
toolId,
|
|
13378
|
-
signal: abortController?.signal,
|
|
13379
|
-
});
|
|
13392
|
+
const providerPermit =
|
|
13393
|
+
await this.resourceGovernor.acquireProviderPermit({
|
|
13394
|
+
toolId,
|
|
13395
|
+
signal: abortController?.signal,
|
|
13396
|
+
});
|
|
13380
13397
|
try {
|
|
13381
13398
|
// Provider admission is our queue, not provider execution.
|
|
13382
13399
|
// Start the tool deadline only after admission so a busy
|
|
@@ -619,8 +619,6 @@ export interface ContextOptions {
|
|
|
619
619
|
docflowEnabled?: boolean;
|
|
620
620
|
/** Internal fixture-only simulation of provider response residence. */
|
|
621
621
|
fixtureBehavior?: FixtureBehavior | null;
|
|
622
|
-
/** Preview/dev test seam that applies provider pacing to fixture responses. */
|
|
623
|
-
enforceFixtureProviderPacing?: boolean;
|
|
624
622
|
/**
|
|
625
623
|
* Server-validated per-run ceiling for concurrently resident provider-tool
|
|
626
624
|
* executions and direct ctx.fetch calls. Omitted uses the platform default.
|
|
@@ -697,26 +697,39 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
697
697
|
isInFlightRuntimeReceipt(receipt) &&
|
|
698
698
|
typeof receipt.runId === 'string' &&
|
|
699
699
|
receipt.runId.trim() === input.runId;
|
|
700
|
-
const
|
|
701
|
-
kind: 'recovered';
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
700
|
+
const waitForRunningReceiptOrRelease = async (): Promise<
|
|
701
|
+
{ kind: 'recovered'; output: T } | { kind: 'claimed' }
|
|
702
|
+
> => {
|
|
703
|
+
const maxAttempts =
|
|
704
|
+
input.runningReceiptWaitMaxAttempts ?? DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
|
|
705
|
+
const delayMs =
|
|
706
|
+
input.runningReceiptWaitDelayMs ?? DURABLE_RECEIPT_WAIT_DELAY_MS;
|
|
707
|
+
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
708
|
+
if (attempt > 0) await sleepReceiptWait(delayMs);
|
|
709
|
+
const current = await input.store.get(input.receiptKey);
|
|
710
|
+
if (current?.status === 'completed' || current?.status === 'skipped') {
|
|
711
|
+
return {
|
|
712
|
+
kind: 'recovered',
|
|
713
|
+
output: await recoverCompletedReceipt(current, 'in_flight'),
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
if (current?.status === 'failed') {
|
|
717
|
+
throw runtimeReceiptFailureError(
|
|
718
|
+
current,
|
|
719
|
+
`ctx.${input.operation}(${input.id}): previous execution failed and cannot be reused`,
|
|
720
|
+
input.toolErrorSchemaVersion,
|
|
721
|
+
);
|
|
722
|
+
}
|
|
723
|
+
if (
|
|
724
|
+
(current?.status === 'queued' || current?.status === 'pending') &&
|
|
725
|
+
!current.leaseId &&
|
|
726
|
+
!current.leaseExpiresAt
|
|
727
|
+
) {
|
|
728
|
+
return await reclaimReceipt();
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
throw new RuntimeReceiptWaitTimeoutError(input.receiptKey);
|
|
732
|
+
};
|
|
720
733
|
const repairOrWaitForRunningReceipt = async (
|
|
721
734
|
receipt: RuntimeStepReceipt,
|
|
722
735
|
): Promise<{ kind: 'recovered'; output: T } | { kind: 'claimed' }> => {
|
|
@@ -726,7 +739,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
726
739
|
return { kind: 'claimed' };
|
|
727
740
|
}
|
|
728
741
|
try {
|
|
729
|
-
return await
|
|
742
|
+
return await waitForRunningReceiptOrRelease();
|
|
730
743
|
} catch (error) {
|
|
731
744
|
if (error instanceof RuntimeReceiptWaitTimeoutError) {
|
|
732
745
|
const recovered = await reclaimReceipt();
|
|
@@ -840,6 +853,24 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
840
853
|
assertNoSecretTaint(result, `ctx.${input.operation} result`);
|
|
841
854
|
} catch (error) {
|
|
842
855
|
if (input.shouldPersistFailure?.(error) === false) {
|
|
856
|
+
// This error is deliberately not a durable execution result (for
|
|
857
|
+
// example, an HTTP non-2xx response under the current ctx.fetch
|
|
858
|
+
// contract). Release only the lease we still own so a later run can
|
|
859
|
+
// make a fresh, idempotency-protected attempt rather than inheriting a
|
|
860
|
+
// running or failed receipt.
|
|
861
|
+
const released = await input.store.release(
|
|
862
|
+
input.receiptKey,
|
|
863
|
+
input.runId,
|
|
864
|
+
ownedLeaseId,
|
|
865
|
+
);
|
|
866
|
+
if (
|
|
867
|
+
!released ||
|
|
868
|
+
(released.status !== 'queued' && released.status !== 'pending')
|
|
869
|
+
) {
|
|
870
|
+
throw new Error(
|
|
871
|
+
`ctx.${input.operation}(${input.id}): non-cacheable execution failed but receipt ownership could not be released: ${input.formatError(error)}.`,
|
|
872
|
+
);
|
|
873
|
+
}
|
|
843
874
|
throw error;
|
|
844
875
|
}
|
|
845
876
|
// The ownership is uncertain, so neither `fail` nor `release` is safe.
|
|
@@ -137,6 +137,15 @@ export interface PlayExecutionGovernor {
|
|
|
137
137
|
*/
|
|
138
138
|
suggestedParallelism(toolId: string, fallback: number): Promise<number>;
|
|
139
139
|
|
|
140
|
+
/**
|
|
141
|
+
* Suggested resident width for direct, unbatched tool dispatch.
|
|
142
|
+
* Provider permits, not this queue width, govern outbound starts.
|
|
143
|
+
*/
|
|
144
|
+
suggestedDirectToolParallelism(
|
|
145
|
+
toolId: string,
|
|
146
|
+
fallback: number,
|
|
147
|
+
): Promise<number>;
|
|
148
|
+
|
|
140
149
|
/** Increment a monotonic budget counter; throws GovernorBudgetError on breach. */
|
|
141
150
|
chargeBudget(kind: BudgetKind, amount?: number): Promise<void>;
|
|
142
151
|
|
|
@@ -479,14 +488,43 @@ export function createPlayExecutionGovernor(
|
|
|
479
488
|
|
|
480
489
|
async suggestedParallelism(toolId, fallback) {
|
|
481
490
|
const pacing = await resolveAdaptivePacing(toolId);
|
|
482
|
-
const
|
|
483
|
-
rule.maxConcurrency != null
|
|
484
|
-
? [rule.requestsPerWindow, rule.maxConcurrency]
|
|
485
|
-
: [rule.requestsPerWindow],
|
|
491
|
+
const concurrencyLimits = pacing.rules.flatMap((rule) =>
|
|
492
|
+
rule.maxConcurrency != null ? [rule.maxConcurrency] : [],
|
|
486
493
|
);
|
|
494
|
+
// A provider that declares only RPS has no resident-call contract, so
|
|
495
|
+
// retain the caller's bounded envelope; the rate-state backend still
|
|
496
|
+
// governs every outbound start.
|
|
497
|
+
const widthLimits =
|
|
498
|
+
concurrencyLimits.length > 0 ? concurrencyLimits : [fallback];
|
|
487
499
|
return Math.max(
|
|
488
500
|
1,
|
|
489
|
-
Math.min(
|
|
501
|
+
Math.min(
|
|
502
|
+
Math.max(1, fallback),
|
|
503
|
+
policy.pacing.suggestedMaxParallelism,
|
|
504
|
+
...widthLimits,
|
|
505
|
+
),
|
|
506
|
+
);
|
|
507
|
+
},
|
|
508
|
+
|
|
509
|
+
async suggestedDirectToolParallelism(toolId, fallback) {
|
|
510
|
+
const pacing = await resolveAdaptivePacing(toolId);
|
|
511
|
+
const concurrencyLimits = pacing.rules.flatMap((rule) =>
|
|
512
|
+
rule.maxConcurrency != null ? [rule.maxConcurrency] : [],
|
|
513
|
+
);
|
|
514
|
+
// The shared rate backend controls every outbound start. An RPS-only
|
|
515
|
+
// provider needs enough resident work to cover response residence; using
|
|
516
|
+
// requestsPerWindow as this width turns, for example, a 25-RPS provider
|
|
517
|
+
// with 4-second responses into a 6.25-RPS lane. Only an explicit provider
|
|
518
|
+
// concurrency contract may tighten the resident envelope.
|
|
519
|
+
const widthLimits =
|
|
520
|
+
concurrencyLimits.length > 0 ? concurrencyLimits : [fallback];
|
|
521
|
+
return Math.max(
|
|
522
|
+
1,
|
|
523
|
+
Math.min(
|
|
524
|
+
Math.max(1, fallback),
|
|
525
|
+
policy.pacing.suggestedMaxParallelism,
|
|
526
|
+
...widthLimits,
|
|
527
|
+
),
|
|
490
528
|
);
|
|
491
529
|
},
|
|
492
530
|
|
|
@@ -615,6 +653,8 @@ function createInlineChildGovernor(
|
|
|
615
653
|
root.acquireProviderPermit(toolId, opts),
|
|
616
654
|
suggestedParallelism: (toolId, fallback) =>
|
|
617
655
|
root.suggestedParallelism(toolId, fallback),
|
|
656
|
+
suggestedDirectToolParallelism: (toolId, fallback) =>
|
|
657
|
+
root.suggestedDirectToolParallelism(toolId, fallback),
|
|
618
658
|
chargeBudget: (kind, amount) => root.chargeBudget(kind, amount),
|
|
619
659
|
resolveRowConcurrency: (requested) => root.resolveRowConcurrency(requested),
|
|
620
660
|
reportProviderBackpressure: async (input) =>
|
|
@@ -80,7 +80,11 @@ export interface ResolvedExecutionPolicy {
|
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
export const DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS = 64;
|
|
83
|
-
|
|
83
|
+
// A 1,000-row Play may legitimately need 1,000 resident provider calls to
|
|
84
|
+
// sustain a declared start rate while responses are slow. This is a validated
|
|
85
|
+
// opt-in ceiling, not the default transport load: short runtime-to-app fetches
|
|
86
|
+
// remain guarded independently by `integrationRequests` below.
|
|
87
|
+
export const MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS = 1_000;
|
|
84
88
|
export const MAX_CONFIGURABLE_CONCURRENT_ROWS = 1_000;
|
|
85
89
|
// Dispatch groups are only a scheduler envelope. The external-call semaphore
|
|
86
90
|
// remains the resource bound, so do not impose a lower hidden ceiling here.
|
|
@@ -146,7 +150,7 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
|
|
|
146
150
|
rowMax: 1_000,
|
|
147
151
|
// Global logical-call backstop. The default is intentionally close to the
|
|
148
152
|
// standard row cohort so slow small responses do not serialize most rows.
|
|
149
|
-
// Runs may request a value through the validated 1..
|
|
153
|
+
// Runs may request a value through the validated 1..1000 launch contract;
|
|
150
154
|
// provider pacing and the runner-owned hard ceiling still apply.
|
|
151
155
|
toolCalls: DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS,
|
|
152
156
|
// Physical runtime-to-app fetch/body admission. Keep short socket bursts
|
|
@@ -173,7 +177,11 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
|
|
|
173
177
|
pacing: {
|
|
174
178
|
// Undeclared providers; declared providers (rate-limit-definitions.ts) win.
|
|
175
179
|
defaultProviderRequestsPerSecond: 10,
|
|
176
|
-
|
|
180
|
+
// This is the scheduler's resident-call envelope, not an RPS target.
|
|
181
|
+
// Keep it aligned with the explicit run ceiling so an RPS-only provider
|
|
182
|
+
// can retain the calls required by its observed residence time. A declared
|
|
183
|
+
// provider maxConcurrency still tightens this per lane.
|
|
184
|
+
suggestedMaxParallelism: MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS,
|
|
177
185
|
// Worker isolate pacing knobs. The budget module owns platform accounting;
|
|
178
186
|
// the Governor owns the policy values that decide when work is chunked.
|
|
179
187
|
workerYieldElapsedMs: 45_000,
|
|
@@ -138,8 +138,6 @@ export interface PlayRunnerContextConfig {
|
|
|
138
138
|
docflowEnabled?: boolean;
|
|
139
139
|
/** Validated internal fixture-only provider response simulation. */
|
|
140
140
|
fixtureBehavior?: FixtureBehavior | null;
|
|
141
|
-
/** Preview/dev test seam that applies provider pacing to fixture responses. */
|
|
142
|
-
enforceFixtureProviderPacing?: boolean;
|
|
143
141
|
/** Validated per-run ceiling for provider-tool executions and ctx.fetch. */
|
|
144
142
|
maxConcurrentExternalCalls?: number | null;
|
|
145
143
|
maxConcurrentRows?: number | null;
|
|
@@ -60,6 +60,10 @@ export interface RuntimeResourceGovernor {
|
|
|
60
60
|
signal?: AbortSignal;
|
|
61
61
|
}): Promise<RuntimeResourceLease>;
|
|
62
62
|
suggestedToolParallelism(toolId: string, fallback: number): Promise<number>;
|
|
63
|
+
suggestedDirectToolParallelism(
|
|
64
|
+
toolId: string,
|
|
65
|
+
fallback: number,
|
|
66
|
+
): Promise<number>;
|
|
63
67
|
resolveRowConcurrency(requested?: number): number;
|
|
64
68
|
reportProviderBackpressure(input: {
|
|
65
69
|
provider: string;
|
|
@@ -228,6 +232,13 @@ export function createRuntimeResourceGovernor(input: {
|
|
|
228
232
|
return input.executionGovernor.suggestedParallelism(toolId, fallback);
|
|
229
233
|
},
|
|
230
234
|
|
|
235
|
+
suggestedDirectToolParallelism(toolId, fallback) {
|
|
236
|
+
return input.executionGovernor.suggestedDirectToolParallelism(
|
|
237
|
+
toolId,
|
|
238
|
+
fallback,
|
|
239
|
+
);
|
|
240
|
+
},
|
|
241
|
+
|
|
231
242
|
resolveRowConcurrency(requested) {
|
|
232
243
|
return input.executionGovernor.resolveRowConcurrency(requested);
|
|
233
244
|
},
|
|
@@ -129,7 +129,11 @@ export function valueContainsSecret(value: unknown): boolean {
|
|
|
129
129
|
const seen = new WeakSet<object>();
|
|
130
130
|
while (pending.length > 0) {
|
|
131
131
|
const candidate = pending.pop();
|
|
132
|
-
if (
|
|
132
|
+
if (
|
|
133
|
+
isSecretValue(candidate) || isSecretAuth(candidate)
|
|
134
|
+
) {
|
|
135
|
+
return true;
|
|
136
|
+
}
|
|
133
137
|
if (typeof candidate === 'string') {
|
|
134
138
|
if (SECRET_HANDLE_MARKER_RE.test(candidate)) return true;
|
|
135
139
|
continue;
|