deepline 0.3.48 → 0.3.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +6 -36
- package/dist/bundling-sources/sdk/src/http.ts +1 -4
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +2 -61
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +25 -69
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +0 -2
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +45 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +11 -3
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +0 -2
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +0 -26
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +1 -14
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/contract.ts +17 -7
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +0 -1
- package/dist/cli/index.js +739 -457
- package/dist/cli/index.mjs +709 -422
- package/dist/index.d.mts +2 -8
- package/dist/index.d.ts +2 -8
- package/dist/index.js +9 -24
- package/dist/index.mjs +9 -24
- package/package.json +1 -1
|
@@ -3106,14 +3106,6 @@ export class DeeplineClient {
|
|
|
3106
3106
|
signal?: AbortSignal;
|
|
3107
3107
|
lastEventId?: string;
|
|
3108
3108
|
mode?: 'cli' | 'ui';
|
|
3109
|
-
/**
|
|
3110
|
-
* A run just accepted by durable scheduler admission can take a short
|
|
3111
|
-
* time to appear in the asynchronous Convex read model. Callers that
|
|
3112
|
-
* received that admission may opt into a bounded retry of the initial
|
|
3113
|
-
* pending response; a normal arbitrary run id still fails loudly by
|
|
3114
|
-
* default.
|
|
3115
|
-
*/
|
|
3116
|
-
waitForProjection?: boolean;
|
|
3117
3109
|
},
|
|
3118
3110
|
): AsyncGenerator<PlayLiveEvent> {
|
|
3119
3111
|
const headers =
|
|
@@ -3122,33 +3114,12 @@ export class DeeplineClient {
|
|
|
3122
3114
|
: undefined;
|
|
3123
3115
|
const params = new URLSearchParams();
|
|
3124
3116
|
params.set('mode', options?.mode ?? 'cli');
|
|
3125
|
-
const
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
`/api/v2/runs/${encodeURIComponent(workflowId)}/tail?${params.toString()}`,
|
|
3132
|
-
{ signal: options?.signal, headers },
|
|
3133
|
-
)) {
|
|
3134
|
-
sawEvent = true;
|
|
3135
|
-
if (event.scope === 'play') {
|
|
3136
|
-
yield event;
|
|
3137
|
-
}
|
|
3138
|
-
}
|
|
3139
|
-
return;
|
|
3140
|
-
} catch (error) {
|
|
3141
|
-
const projectionPending =
|
|
3142
|
-
options?.waitForProjection === true &&
|
|
3143
|
-
!sawEvent &&
|
|
3144
|
-
error instanceof DeeplineError &&
|
|
3145
|
-
(error.statusCode === 404 ||
|
|
3146
|
-
(error.statusCode === 202 &&
|
|
3147
|
-
error.code === 'RUN_PROJECTION_PENDING')) &&
|
|
3148
|
-
Date.now() < projectionDeadline;
|
|
3149
|
-
if (!projectionPending) throw error;
|
|
3150
|
-
await sleep(streamReconnectDelayMs(projectionAttempt));
|
|
3151
|
-
projectionAttempt += 1;
|
|
3117
|
+
for await (const event of this.http.streamSse<PlayLiveEvent>(
|
|
3118
|
+
`/api/v2/runs/${encodeURIComponent(workflowId)}/tail?${params.toString()}`,
|
|
3119
|
+
{ signal: options?.signal, headers },
|
|
3120
|
+
)) {
|
|
3121
|
+
if (event.scope === 'play') {
|
|
3122
|
+
yield event;
|
|
3152
3123
|
}
|
|
3153
3124
|
}
|
|
3154
3125
|
}
|
|
@@ -4468,7 +4439,6 @@ export class DeeplineClient {
|
|
|
4468
4439
|
for await (const event of this.streamPlayRunEvents(workflowId, {
|
|
4469
4440
|
mode: 'cli',
|
|
4470
4441
|
signal: options?.signal,
|
|
4471
|
-
waitForProjection: true,
|
|
4472
4442
|
})) {
|
|
4473
4443
|
if (options?.signal?.aborted) {
|
|
4474
4444
|
await this.cancelPlay(workflowId);
|
|
@@ -621,10 +621,7 @@ export class HttpClient {
|
|
|
621
621
|
signal: options?.signal,
|
|
622
622
|
});
|
|
623
623
|
|
|
624
|
-
|
|
625
|
-
// request. A 202 JSON response is a deliberately retryable admission
|
|
626
|
-
// state, not an empty, successfully completed stream.
|
|
627
|
-
if (!response.ok || response.status === 202) {
|
|
624
|
+
if (!response.ok) {
|
|
628
625
|
const body = await response.text();
|
|
629
626
|
const parsed = parseResponseBody(body);
|
|
630
627
|
if (
|
|
@@ -199,7 +199,7 @@ export const SDK_RELEASE = {
|
|
|
199
199
|
// available at toolResponse.rawV2 while toolResponse.raw and all declared
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
|
-
version: '0.3.
|
|
202
|
+
version: '0.3.50',
|
|
203
203
|
updateSummary:
|
|
204
204
|
'Automatic CLI updates are now enabled by default. To opt out, run `deepline settings autoupdate off`; use `deepline settings autoupdate on` to re-enable updates or `deepline settings autoupdate pin <version>` to hold an exact release. This release also adds raw-v2 tool responses at toolResponse.rawV2 while preserving existing toolResponse.raw and declared getters.',
|
|
205
205
|
packageCapabilities: {
|
|
@@ -657,6 +657,8 @@ export interface PlayRunPackage {
|
|
|
657
657
|
playName: string;
|
|
658
658
|
status: string;
|
|
659
659
|
dashboardUrl?: string;
|
|
660
|
+
/** Durable acceptance time: the run record was created. */
|
|
661
|
+
acceptedAt?: number | null;
|
|
660
662
|
updatedAt?: number | null;
|
|
661
663
|
startedAt?: number | null;
|
|
662
664
|
finishedAt?: number | null;
|
|
@@ -156,7 +156,6 @@ type RuntimeApiRequest =
|
|
|
156
156
|
action: 'start_run';
|
|
157
157
|
idempotencyKey?: string;
|
|
158
158
|
playName: string;
|
|
159
|
-
playReference?: string | null;
|
|
160
159
|
runId: string;
|
|
161
160
|
artifactStorageKey?: string | null;
|
|
162
161
|
artifactHash?: string | null;
|
|
@@ -173,45 +172,6 @@ type RuntimeApiRequest =
|
|
|
173
172
|
inputSha256?: string;
|
|
174
173
|
replayedFromRunId?: string | null;
|
|
175
174
|
}
|
|
176
|
-
| {
|
|
177
|
-
/** Ordered projection of a Runtime Postgres/Absurd admission. */
|
|
178
|
-
action: 'project_run_created';
|
|
179
|
-
idempotencyKey?: string;
|
|
180
|
-
playName: string;
|
|
181
|
-
playReference?: string | null;
|
|
182
|
-
workflowId: string;
|
|
183
|
-
runId: string;
|
|
184
|
-
workflowFamilyKey: string;
|
|
185
|
-
definitionScope?: 'org' | 'system';
|
|
186
|
-
revisionId?: string | null;
|
|
187
|
-
artifactStorageKey?: string | null;
|
|
188
|
-
artifactHash?: string | null;
|
|
189
|
-
graphHash?: string | null;
|
|
190
|
-
runtimeBackend?: string | null;
|
|
191
|
-
schedulerBackend?: string | null;
|
|
192
|
-
schedulerSchema?: string | null;
|
|
193
|
-
executionProfile?: string | null;
|
|
194
|
-
runtimeReleaseId?: string | null;
|
|
195
|
-
runtimeDeployVersion?: string | null;
|
|
196
|
-
runtimeCallbackBaseUrl?: string | null;
|
|
197
|
-
coordinatorUrl?: string | null;
|
|
198
|
-
coordinatorWorkerName?: string | null;
|
|
199
|
-
runtimeWorkflowName?: string | null;
|
|
200
|
-
runtimeHarnessWorkerName?: string | null;
|
|
201
|
-
maxCreditsPerRun?: number | null;
|
|
202
|
-
staticPipeline?: unknown;
|
|
203
|
-
source?: 'published' | 'ad_hoc' | 'draft';
|
|
204
|
-
triggerSource?: 'webhook' | 'cron' | 'sql_listener' | 'api';
|
|
205
|
-
inputFileId?: string;
|
|
206
|
-
inputBytes?: number;
|
|
207
|
-
inputSha256?: string;
|
|
208
|
-
replayedFromRunId?: string | null;
|
|
209
|
-
secretRefs?: unknown[];
|
|
210
|
-
inputSummary?: Record<string, unknown>;
|
|
211
|
-
/** Terminal bypass used only when the causal admission projection is
|
|
212
|
-
* permanently blocked and no queued row can be projected. */
|
|
213
|
-
admissionFailure?: string;
|
|
214
|
-
}
|
|
215
175
|
| ({
|
|
216
176
|
action: 'save_results';
|
|
217
177
|
} & RuntimeSaveResults)
|
|
@@ -692,6 +652,7 @@ function isRetryableAppRuntimeAction(
|
|
|
692
652
|
action === 'create_signed_staged_file_url' ||
|
|
693
653
|
action === 'get_runtime_step_receipt' ||
|
|
694
654
|
action === 'get_runtime_step_receipts' ||
|
|
655
|
+
action === 'governor_budget_charge' ||
|
|
695
656
|
action === 'heartbeat_runtime_step_receipts' ||
|
|
696
657
|
action === 'claim_runtime_step_receipt' ||
|
|
697
658
|
action === 'claim_runtime_step_receipts' ||
|
|
@@ -708,8 +669,7 @@ function isRetryableAppRuntimeAction(
|
|
|
708
669
|
action === 'release_runtime_step_receipt' ||
|
|
709
670
|
action === 'save_results' ||
|
|
710
671
|
action === 'skip_runtime_step_receipt' ||
|
|
711
|
-
action === 'start_run'
|
|
712
|
-
action === 'project_run_created'
|
|
672
|
+
action === 'start_run'
|
|
713
673
|
);
|
|
714
674
|
}
|
|
715
675
|
|
|
@@ -1966,7 +1926,6 @@ export async function startRunViaAppRuntime(
|
|
|
1966
1926
|
context: WorkerRuntimeApiContext,
|
|
1967
1927
|
input: {
|
|
1968
1928
|
playName: string;
|
|
1969
|
-
playReference?: string | null;
|
|
1970
1929
|
runId: string;
|
|
1971
1930
|
artifactStorageKey?: string | null;
|
|
1972
1931
|
artifactHash?: string | null;
|
|
@@ -1991,24 +1950,6 @@ export async function startRunViaAppRuntime(
|
|
|
1991
1950
|
});
|
|
1992
1951
|
}
|
|
1993
1952
|
|
|
1994
|
-
/**
|
|
1995
|
-
* Deliver the causally-first `run.created` projection after Runtime Postgres
|
|
1996
|
-
* has atomically admitted the run and its Absurd task. It is safe to retry:
|
|
1997
|
-
* the server preserves any existing queued, running, or terminal run.
|
|
1998
|
-
*/
|
|
1999
|
-
export async function projectRunCreatedViaAppRuntime(
|
|
2000
|
-
context: WorkerRuntimeApiContext,
|
|
2001
|
-
input: Omit<
|
|
2002
|
-
Extract<RuntimeApiRequest, { action: 'project_run_created' }>,
|
|
2003
|
-
'action'
|
|
2004
|
-
>,
|
|
2005
|
-
): Promise<void> {
|
|
2006
|
-
await postAppRuntimeApi<{ ok: true; status: 'queued' }>(context, {
|
|
2007
|
-
action: 'project_run_created',
|
|
2008
|
-
...input,
|
|
2009
|
-
});
|
|
2010
|
-
}
|
|
2011
|
-
|
|
2012
1953
|
export async function saveResultsViaAppRuntime(
|
|
2013
1954
|
context: WorkerRuntimeApiContext,
|
|
2014
1955
|
input: RuntimeSaveResults,
|
|
@@ -2040,8 +2040,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2040
2040
|
#options: ContextOptions;
|
|
2041
2041
|
private readonly executionScope: RunExecutionScope;
|
|
2042
2042
|
private logBuffer: string[] = [];
|
|
2043
|
-
private fixtureProviderPacingBypassLogged = false;
|
|
2044
|
-
private fixtureProviderPacingEnforcementLogged = false;
|
|
2045
2043
|
private checkpoint: PlayCheckpoint;
|
|
2046
2044
|
private readonly durableCallCacheEpochMs: number;
|
|
2047
2045
|
/**
|
|
@@ -8698,13 +8696,6 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8698
8696
|
}
|
|
8699
8697
|
}
|
|
8700
8698
|
|
|
8701
|
-
private fixtureProviderPacingDisabled(): boolean {
|
|
8702
|
-
return (
|
|
8703
|
-
this.#options.integrationMode === 'fixture' &&
|
|
8704
|
-
this.#options.enforceFixtureProviderPacing !== true
|
|
8705
|
-
);
|
|
8706
|
-
}
|
|
8707
|
-
|
|
8708
8699
|
private toolDispatchLane(request: ToolCallRequest): {
|
|
8709
8700
|
key: string;
|
|
8710
8701
|
readyCount: number;
|
|
@@ -11065,12 +11056,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11065
11056
|
this.governor.policy.concurrency.toolCalls,
|
|
11066
11057
|
Math.max(
|
|
11067
11058
|
1,
|
|
11068
|
-
this.
|
|
11069
|
-
|
|
11070
|
-
|
|
11071
|
-
|
|
11072
|
-
this.governor.policy.concurrency.toolCalls,
|
|
11073
|
-
),
|
|
11059
|
+
await this.resourceGovernor.suggestedToolParallelism(
|
|
11060
|
+
toolId,
|
|
11061
|
+
this.governor.policy.concurrency.toolCalls,
|
|
11062
|
+
),
|
|
11074
11063
|
),
|
|
11075
11064
|
);
|
|
11076
11065
|
const dispatchOwned = async (
|
|
@@ -11226,7 +11215,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11226
11215
|
const batchParallelismCeiling =
|
|
11227
11216
|
this.governor.policy.pacing.workerToolBatchDefaultParallelism;
|
|
11228
11217
|
const batchSize =
|
|
11229
|
-
compiledBatches.length > 0
|
|
11218
|
+
compiledBatches.length > 0
|
|
11230
11219
|
? await this.resourceGovernor.suggestedToolParallelism(
|
|
11231
11220
|
compiledBatches[0]!.batchOperation,
|
|
11232
11221
|
batchParallelismCeiling,
|
|
@@ -12501,8 +12490,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12501
12490
|
const batchParallelismCeiling =
|
|
12502
12491
|
this.governor.policy.pacing.workerToolBatchDefaultParallelism;
|
|
12503
12492
|
const batchSize =
|
|
12504
|
-
compiledBatches.length > 0
|
|
12505
|
-
!this.fixtureProviderPacingDisabled()
|
|
12493
|
+
compiledBatches.length > 0
|
|
12506
12494
|
? await this.resourceGovernor.suggestedToolParallelism(
|
|
12507
12495
|
compiledBatches[0]!.batchOperation,
|
|
12508
12496
|
batchParallelismCeiling,
|
|
@@ -12756,23 +12744,18 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12756
12744
|
void flushCompletionBuffer();
|
|
12757
12745
|
}, 0);
|
|
12758
12746
|
});
|
|
12759
|
-
//
|
|
12760
|
-
//
|
|
12761
|
-
//
|
|
12762
|
-
// the
|
|
12763
|
-
//
|
|
12764
|
-
// provider's RPS/maxConcurrency pacing rules (see
|
|
12765
|
-
// suggestedParallelism in governor.ts), floored at 1 and capped by
|
|
12766
|
-
// the global tool-call concurrency ceiling. The pacer still gates
|
|
12767
|
-
// each in-flight call, so this only trims the launch burst.
|
|
12747
|
+
// Bound direct map dispatch separately from ordinary provider
|
|
12748
|
+
// residency. This prevents a blocked first chunk from launching
|
|
12749
|
+
// another chunk ahead of a provider's admitted start rate, while
|
|
12750
|
+
// preserving the wider resident-call capacity used by batched and
|
|
12751
|
+
// high-latency providers.
|
|
12768
12752
|
const toolCallConcurrencyCeiling =
|
|
12769
12753
|
this.governor.policy.concurrency.toolCalls;
|
|
12770
|
-
const shapedToolParallelism =
|
|
12771
|
-
|
|
12772
|
-
|
|
12773
|
-
|
|
12774
|
-
|
|
12775
|
-
);
|
|
12754
|
+
const shapedToolParallelism =
|
|
12755
|
+
await this.resourceGovernor.suggestedDirectToolParallelism(
|
|
12756
|
+
toolId,
|
|
12757
|
+
toolCallConcurrencyCeiling,
|
|
12758
|
+
);
|
|
12776
12759
|
const dispatchWidth = Math.min(
|
|
12777
12760
|
toolCallConcurrencyCeiling,
|
|
12778
12761
|
Math.max(1, shapedToolParallelism),
|
|
@@ -13339,44 +13322,17 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13339
13322
|
// is the fetch itself, so independently delayed runners cannot
|
|
13340
13323
|
// compress real provider arrivals after their tickets.
|
|
13341
13324
|
const protectionHeaders = await this.vercelProtectionHeaders();
|
|
13342
|
-
//
|
|
13343
|
-
//
|
|
13344
|
-
//
|
|
13345
|
-
//
|
|
13346
|
-
// latter as live would make fixture runs both slow and
|
|
13347
|
-
// unauditable even though the app never dispatches a provider
|
|
13348
|
-
// request.
|
|
13325
|
+
// Fixture responses simulate provider residence, but provider
|
|
13326
|
+
// admission remains real. This makes fixture volume results a
|
|
13327
|
+
// faithful measurement of the runtime controller without
|
|
13328
|
+
// spending provider credits.
|
|
13349
13329
|
const fixtureExecution =
|
|
13350
13330
|
this.#options.integrationMode === 'fixture';
|
|
13351
|
-
const
|
|
13352
|
-
|
|
13353
|
-
|
|
13354
|
-
|
|
13355
|
-
|
|
13356
|
-
if (
|
|
13357
|
-
fixtureOnlyExecution &&
|
|
13358
|
-
!this.fixtureProviderPacingBypassLogged
|
|
13359
|
-
) {
|
|
13360
|
-
this.fixtureProviderPacingBypassLogged = true;
|
|
13361
|
-
this.log(
|
|
13362
|
-
'Fixture mode: provider pacing bypassed because no provider request is dispatched.',
|
|
13363
|
-
);
|
|
13364
|
-
}
|
|
13365
|
-
if (
|
|
13366
|
-
enforceFixtureProviderPacing &&
|
|
13367
|
-
!this.fixtureProviderPacingEnforcementLogged
|
|
13368
|
-
) {
|
|
13369
|
-
this.fixtureProviderPacingEnforcementLogged = true;
|
|
13370
|
-
this.log(
|
|
13371
|
-
'Fixture mode: provider pacing explicitly enforced for production-parity testing.',
|
|
13372
|
-
);
|
|
13373
|
-
}
|
|
13374
|
-
const providerPermit = fixtureOnlyExecution
|
|
13375
|
-
? { release() {} }
|
|
13376
|
-
: await this.resourceGovernor.acquireProviderPermit({
|
|
13377
|
-
toolId,
|
|
13378
|
-
signal: abortController?.signal,
|
|
13379
|
-
});
|
|
13331
|
+
const providerPermit =
|
|
13332
|
+
await this.resourceGovernor.acquireProviderPermit({
|
|
13333
|
+
toolId,
|
|
13334
|
+
signal: abortController?.signal,
|
|
13335
|
+
});
|
|
13380
13336
|
try {
|
|
13381
13337
|
// Provider admission is our queue, not provider execution.
|
|
13382
13338
|
// Start the tool deadline only after admission so a busy
|
|
@@ -619,8 +619,6 @@ export interface ContextOptions {
|
|
|
619
619
|
docflowEnabled?: boolean;
|
|
620
620
|
/** Internal fixture-only simulation of provider response residence. */
|
|
621
621
|
fixtureBehavior?: FixtureBehavior | null;
|
|
622
|
-
/** Preview/dev test seam that applies provider pacing to fixture responses. */
|
|
623
|
-
enforceFixtureProviderPacing?: boolean;
|
|
624
622
|
/**
|
|
625
623
|
* Server-validated per-run ceiling for concurrently resident provider-tool
|
|
626
624
|
* executions and direct ctx.fetch calls. Omitted uses the platform default.
|
|
@@ -137,6 +137,15 @@ export interface PlayExecutionGovernor {
|
|
|
137
137
|
*/
|
|
138
138
|
suggestedParallelism(toolId: string, fallback: number): Promise<number>;
|
|
139
139
|
|
|
140
|
+
/**
|
|
141
|
+
* Suggested resident width for direct, unbatched tool dispatch.
|
|
142
|
+
* Provider permits, not this queue width, govern outbound starts.
|
|
143
|
+
*/
|
|
144
|
+
suggestedDirectToolParallelism(
|
|
145
|
+
toolId: string,
|
|
146
|
+
fallback: number,
|
|
147
|
+
): Promise<number>;
|
|
148
|
+
|
|
140
149
|
/** Increment a monotonic budget counter; throws GovernorBudgetError on breach. */
|
|
141
150
|
chargeBudget(kind: BudgetKind, amount?: number): Promise<void>;
|
|
142
151
|
|
|
@@ -479,14 +488,43 @@ export function createPlayExecutionGovernor(
|
|
|
479
488
|
|
|
480
489
|
async suggestedParallelism(toolId, fallback) {
|
|
481
490
|
const pacing = await resolveAdaptivePacing(toolId);
|
|
482
|
-
const
|
|
483
|
-
rule.maxConcurrency != null
|
|
484
|
-
? [rule.requestsPerWindow, rule.maxConcurrency]
|
|
485
|
-
: [rule.requestsPerWindow],
|
|
491
|
+
const concurrencyLimits = pacing.rules.flatMap((rule) =>
|
|
492
|
+
rule.maxConcurrency != null ? [rule.maxConcurrency] : [],
|
|
486
493
|
);
|
|
494
|
+
// A provider that declares only RPS has no resident-call contract, so
|
|
495
|
+
// retain the caller's bounded envelope; the rate-state backend still
|
|
496
|
+
// governs every outbound start.
|
|
497
|
+
const widthLimits =
|
|
498
|
+
concurrencyLimits.length > 0 ? concurrencyLimits : [fallback];
|
|
487
499
|
return Math.max(
|
|
488
500
|
1,
|
|
489
|
-
Math.min(
|
|
501
|
+
Math.min(
|
|
502
|
+
Math.max(1, fallback),
|
|
503
|
+
policy.pacing.suggestedMaxParallelism,
|
|
504
|
+
...widthLimits,
|
|
505
|
+
),
|
|
506
|
+
);
|
|
507
|
+
},
|
|
508
|
+
|
|
509
|
+
async suggestedDirectToolParallelism(toolId, fallback) {
|
|
510
|
+
const pacing = await resolveAdaptivePacing(toolId);
|
|
511
|
+
const concurrencyLimits = pacing.rules.flatMap((rule) =>
|
|
512
|
+
rule.maxConcurrency != null ? [rule.maxConcurrency] : [],
|
|
513
|
+
);
|
|
514
|
+
// The shared rate backend controls every outbound start. An RPS-only
|
|
515
|
+
// provider needs enough resident work to cover response residence; using
|
|
516
|
+
// requestsPerWindow as this width turns, for example, a 25-RPS provider
|
|
517
|
+
// with 4-second responses into a 6.25-RPS lane. Only an explicit provider
|
|
518
|
+
// concurrency contract may tighten the resident envelope.
|
|
519
|
+
const widthLimits =
|
|
520
|
+
concurrencyLimits.length > 0 ? concurrencyLimits : [fallback];
|
|
521
|
+
return Math.max(
|
|
522
|
+
1,
|
|
523
|
+
Math.min(
|
|
524
|
+
Math.max(1, fallback),
|
|
525
|
+
policy.pacing.suggestedMaxParallelism,
|
|
526
|
+
...widthLimits,
|
|
527
|
+
),
|
|
490
528
|
);
|
|
491
529
|
},
|
|
492
530
|
|
|
@@ -615,6 +653,8 @@ function createInlineChildGovernor(
|
|
|
615
653
|
root.acquireProviderPermit(toolId, opts),
|
|
616
654
|
suggestedParallelism: (toolId, fallback) =>
|
|
617
655
|
root.suggestedParallelism(toolId, fallback),
|
|
656
|
+
suggestedDirectToolParallelism: (toolId, fallback) =>
|
|
657
|
+
root.suggestedDirectToolParallelism(toolId, fallback),
|
|
618
658
|
chargeBudget: (kind, amount) => root.chargeBudget(kind, amount),
|
|
619
659
|
resolveRowConcurrency: (requested) => root.resolveRowConcurrency(requested),
|
|
620
660
|
reportProviderBackpressure: async (input) =>
|
|
@@ -80,7 +80,11 @@ export interface ResolvedExecutionPolicy {
|
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
export const DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS = 64;
|
|
83
|
-
|
|
83
|
+
// A 1,000-row Play may legitimately need 1,000 resident provider calls to
|
|
84
|
+
// sustain a declared start rate while responses are slow. This is a validated
|
|
85
|
+
// opt-in ceiling, not the default transport load: short runtime-to-app fetches
|
|
86
|
+
// remain guarded independently by `integrationRequests` below.
|
|
87
|
+
export const MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS = 1_000;
|
|
84
88
|
export const MAX_CONFIGURABLE_CONCURRENT_ROWS = 1_000;
|
|
85
89
|
// Dispatch groups are only a scheduler envelope. The external-call semaphore
|
|
86
90
|
// remains the resource bound, so do not impose a lower hidden ceiling here.
|
|
@@ -146,7 +150,7 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
|
|
|
146
150
|
rowMax: 1_000,
|
|
147
151
|
// Global logical-call backstop. The default is intentionally close to the
|
|
148
152
|
// standard row cohort so slow small responses do not serialize most rows.
|
|
149
|
-
// Runs may request a value through the validated 1..
|
|
153
|
+
// Runs may request a value through the validated 1..1000 launch contract;
|
|
150
154
|
// provider pacing and the runner-owned hard ceiling still apply.
|
|
151
155
|
toolCalls: DEFAULT_MAX_CONCURRENT_EXTERNAL_CALLS,
|
|
152
156
|
// Physical runtime-to-app fetch/body admission. Keep short socket bursts
|
|
@@ -173,7 +177,11 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
|
|
|
173
177
|
pacing: {
|
|
174
178
|
// Undeclared providers; declared providers (rate-limit-definitions.ts) win.
|
|
175
179
|
defaultProviderRequestsPerSecond: 10,
|
|
176
|
-
|
|
180
|
+
// This is the scheduler's resident-call envelope, not an RPS target.
|
|
181
|
+
// Keep it aligned with the explicit run ceiling so an RPS-only provider
|
|
182
|
+
// can retain the calls required by its observed residence time. A declared
|
|
183
|
+
// provider maxConcurrency still tightens this per lane.
|
|
184
|
+
suggestedMaxParallelism: MAX_CONFIGURABLE_CONCURRENT_EXTERNAL_CALLS,
|
|
177
185
|
// Worker isolate pacing knobs. The budget module owns platform accounting;
|
|
178
186
|
// the Governor owns the policy values that decide when work is chunked.
|
|
179
187
|
workerYieldElapsedMs: 45_000,
|
|
@@ -138,8 +138,6 @@ export interface PlayRunnerContextConfig {
|
|
|
138
138
|
docflowEnabled?: boolean;
|
|
139
139
|
/** Validated internal fixture-only provider response simulation. */
|
|
140
140
|
fixtureBehavior?: FixtureBehavior | null;
|
|
141
|
-
/** Preview/dev test seam that applies provider pacing to fixture responses. */
|
|
142
|
-
enforceFixtureProviderPacing?: boolean;
|
|
143
141
|
/** Validated per-run ceiling for provider-tool executions and ctx.fetch. */
|
|
144
142
|
maxConcurrentExternalCalls?: number | null;
|
|
145
143
|
maxConcurrentRows?: number | null;
|
|
@@ -60,6 +60,10 @@ export interface RuntimeResourceGovernor {
|
|
|
60
60
|
signal?: AbortSignal;
|
|
61
61
|
}): Promise<RuntimeResourceLease>;
|
|
62
62
|
suggestedToolParallelism(toolId: string, fallback: number): Promise<number>;
|
|
63
|
+
suggestedDirectToolParallelism(
|
|
64
|
+
toolId: string,
|
|
65
|
+
fallback: number,
|
|
66
|
+
): Promise<number>;
|
|
63
67
|
resolveRowConcurrency(requested?: number): number;
|
|
64
68
|
reportProviderBackpressure(input: {
|
|
65
69
|
provider: string;
|
|
@@ -228,6 +232,13 @@ export function createRuntimeResourceGovernor(input: {
|
|
|
228
232
|
return input.executionGovernor.suggestedParallelism(toolId, fallback);
|
|
229
233
|
},
|
|
230
234
|
|
|
235
|
+
suggestedDirectToolParallelism(toolId, fallback) {
|
|
236
|
+
return input.executionGovernor.suggestedDirectToolParallelism(
|
|
237
|
+
toolId,
|
|
238
|
+
fallback,
|
|
239
|
+
);
|
|
240
|
+
},
|
|
241
|
+
|
|
231
242
|
resolveRowConcurrency(requested) {
|
|
232
243
|
return input.executionGovernor.resolveRowConcurrency(requested);
|
|
233
244
|
},
|
|
@@ -42,15 +42,7 @@ export type { PlayRunInputPayload };
|
|
|
42
42
|
export type PlaySchedulerSubmitInput = {
|
|
43
43
|
runId: string;
|
|
44
44
|
playId: string;
|
|
45
|
-
/** Canonical definition name used by the Convex read-model projector. */
|
|
46
|
-
definitionName?: string | null;
|
|
47
45
|
playName: string;
|
|
48
|
-
/** Immutable display/lookup name selected at admission, if distinct. */
|
|
49
|
-
playReference?: string | null;
|
|
50
|
-
/** Definition ownership selected at admission; never re-resolve on replay. */
|
|
51
|
-
definitionScope?: 'org' | 'system' | null;
|
|
52
|
-
/** Immutable published revision selected at admission. */
|
|
53
|
-
revisionId?: string | null;
|
|
54
46
|
workflowFamilyKey?: string | null;
|
|
55
47
|
artifactStorageKey: string;
|
|
56
48
|
/** Optional inline artifact for the Node scheduler. */
|
|
@@ -60,10 +52,6 @@ export type PlaySchedulerSubmitInput = {
|
|
|
60
52
|
/** Immutable scheduler/worker/runner ABI snapshot (not a Git SHA). */
|
|
61
53
|
runtimeProtocolVersion?: PlayRuntimeProtocolVersion | null;
|
|
62
54
|
input: PlayRunInputPayload;
|
|
63
|
-
/** Immutable secret authority selected at admission; never re-resolve on projection. */
|
|
64
|
-
secretRefs?: unknown[];
|
|
65
|
-
/** Display-only, size-bounded projection of the admitted input. */
|
|
66
|
-
inputSummary?: Record<string, unknown>;
|
|
67
55
|
/** Convex metadata for the exact input saved before scheduler submission. */
|
|
68
56
|
inputFileId?: string;
|
|
69
57
|
inputBytes?: number;
|
|
@@ -83,8 +71,6 @@ export type PlaySchedulerSubmitInput = {
|
|
|
83
71
|
storageKey?: string;
|
|
84
72
|
fileName?: string;
|
|
85
73
|
logicalPath?: string;
|
|
86
|
-
storageKind?: 'r2';
|
|
87
|
-
contentHash?: string;
|
|
88
74
|
contentType?: string;
|
|
89
75
|
bytes?: number;
|
|
90
76
|
} | null;
|
|
@@ -101,8 +87,6 @@ export type PlaySchedulerSubmitInput = {
|
|
|
101
87
|
logicalPath?: string;
|
|
102
88
|
fileName?: string;
|
|
103
89
|
storageKey: string;
|
|
104
|
-
storageKind?: 'r2';
|
|
105
|
-
contentHash?: string;
|
|
106
90
|
contentType?: string;
|
|
107
91
|
bytes?: number;
|
|
108
92
|
inlineText?: string;
|
|
@@ -152,16 +136,6 @@ export type PlaySchedulerSubmitInput = {
|
|
|
152
136
|
*/
|
|
153
137
|
queuePriority?: number | null;
|
|
154
138
|
executionProfile?: string | null;
|
|
155
|
-
/** Concrete Runtime Postgres namespace selected at admission. */
|
|
156
|
-
schedulerSchema?: string | null;
|
|
157
|
-
/** Immutable release tuple for the Convex read-model projection. */
|
|
158
|
-
runtimeReleaseId?: string | null;
|
|
159
|
-
runtimeCallbackBaseUrl?: string | null;
|
|
160
|
-
coordinatorWorkerName?: string | null;
|
|
161
|
-
runtimeWorkflowName?: string | null;
|
|
162
|
-
runtimeHarnessWorkerName?: string | null;
|
|
163
|
-
/** Billing admission cap from the immutable contract snapshot. */
|
|
164
|
-
maxCreditsPerRun?: number | null;
|
|
165
139
|
/** runner backend to use for executing attempts */
|
|
166
140
|
runtimeBackend: string;
|
|
167
141
|
/**
|
|
@@ -36,8 +36,6 @@ type ValidatedRuntimeTestFaultHeader =
|
|
|
36
36
|
export type RuntimeTestPolicyOverrides = {
|
|
37
37
|
/** Opt-in bounded runner map latency profile for local/preview diagnosis. */
|
|
38
38
|
mapLatencyProfile?: boolean;
|
|
39
|
-
/** Exercise provider pacing during fixture runs without dispatching provider traffic. */
|
|
40
|
-
enforceFixtureProviderPacing?: boolean;
|
|
41
39
|
receiptLeaseTtlMs?: number;
|
|
42
40
|
sheetAttemptLeaseMs?: number;
|
|
43
41
|
heartbeatIntervalMs?: number;
|
|
@@ -258,8 +256,7 @@ function parseRuntimeTestPolicyOverrides(
|
|
|
258
256
|
(key) =>
|
|
259
257
|
!RUNTIME_TEST_POLICY_MS_FIELDS.has(key) &&
|
|
260
258
|
key !== 'workBudgetYieldLimits' &&
|
|
261
|
-
key !== 'mapLatencyProfile'
|
|
262
|
-
key !== 'enforceFixtureProviderPacing',
|
|
259
|
+
key !== 'mapLatencyProfile',
|
|
263
260
|
);
|
|
264
261
|
if (unknownKeys.length > 0) {
|
|
265
262
|
return {
|
|
@@ -276,16 +273,6 @@ function parseRuntimeTestPolicyOverrides(
|
|
|
276
273
|
}
|
|
277
274
|
overrides.mapLatencyProfile = record.mapLatencyProfile;
|
|
278
275
|
}
|
|
279
|
-
if ('enforceFixtureProviderPacing' in record) {
|
|
280
|
-
if (typeof record.enforceFixtureProviderPacing !== 'boolean') {
|
|
281
|
-
return {
|
|
282
|
-
error:
|
|
283
|
-
'testPolicyOverrides.enforceFixtureProviderPacing must be a boolean.',
|
|
284
|
-
};
|
|
285
|
-
}
|
|
286
|
-
overrides.enforceFixtureProviderPacing =
|
|
287
|
-
record.enforceFixtureProviderPacing;
|
|
288
|
-
}
|
|
289
276
|
for (const field of RUNTIME_TEST_POLICY_MS_FIELDS) {
|
|
290
277
|
if (!(field in record)) continue;
|
|
291
278
|
const parsed = readPositiveIntegerField({
|
|
@@ -105,15 +105,25 @@ export type ToolResultReceipts = {
|
|
|
105
105
|
}): Promise<void>;
|
|
106
106
|
};
|
|
107
107
|
|
|
108
|
+
export type AlwaysFreshToolCallDispatch = {
|
|
109
|
+
call: ToolCallInput;
|
|
110
|
+
/** An always-fresh call must not claim a provider idempotency identity. */
|
|
111
|
+
providerOperationKey: null;
|
|
112
|
+
/** Receiptless means structurally absent, not a nullable lease. */
|
|
113
|
+
receipt?: never;
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
export type ReceiptBackedToolCallDispatch = {
|
|
117
|
+
call: ToolCallInput;
|
|
118
|
+
providerOperationKey: string;
|
|
119
|
+
receipt: ToolCallReceiptLease;
|
|
120
|
+
};
|
|
121
|
+
|
|
108
122
|
/** Direct and map execution are Adapter choices, never separate public jobs. */
|
|
109
123
|
export type ToolCallDispatcher = {
|
|
110
|
-
dispatch(
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
providerOperationKey: string | null;
|
|
114
|
-
/** Null means this is a receiptless, always-fresh physical call. */
|
|
115
|
-
receipt: ToolCallReceiptLease | null;
|
|
116
|
-
}): Promise<ToolExecuteResult>;
|
|
124
|
+
dispatch(
|
|
125
|
+
input: AlwaysFreshToolCallDispatch | ReceiptBackedToolCallDispatch,
|
|
126
|
+
): Promise<ToolExecuteResult>;
|
|
117
127
|
};
|
|
118
128
|
|
|
119
129
|
export type ToolCallDependencies = {
|