deepline 0.1.220 → 0.1.222
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +190 -48
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +7 -9
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +1 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +62 -3
- package/dist/cli/index.js +238 -42
- package/dist/cli/index.mjs +238 -42
- package/dist/index.d.mts +21 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/dist/viewer/viewer.css +11 -0
- package/dist/viewer/viewer.js +44 -2
- package/package.json +1 -1
|
@@ -108,10 +108,10 @@ export const SDK_RELEASE = {
|
|
|
108
108
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
109
109
|
// 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
|
|
110
110
|
// automatically without blocking their current command.
|
|
111
|
-
version: '0.1.
|
|
111
|
+
version: '0.1.222',
|
|
112
112
|
apiContract: '2026-06-dataset-handle-results-hard-cutover',
|
|
113
113
|
supportPolicy: {
|
|
114
|
-
latest: '0.1.
|
|
114
|
+
latest: '0.1.222',
|
|
115
115
|
minimumSupported: '0.1.53',
|
|
116
116
|
deprecatedBelow: '0.1.219',
|
|
117
117
|
commandMinimumSupported: [
|
|
@@ -997,6 +997,27 @@ export interface PlayCheckResult {
|
|
|
997
997
|
summary?: string;
|
|
998
998
|
artifactHash?: string | null;
|
|
999
999
|
graphHash?: string | null;
|
|
1000
|
+
/** SHA-256 of the exact source bytes checked by Deepline. */
|
|
1001
|
+
sourceHash?: string | null;
|
|
1002
|
+
/** Enforceable byte budgets measured during cloud preflight. */
|
|
1003
|
+
limits?: {
|
|
1004
|
+
revisionStorage: {
|
|
1005
|
+
usedBytes: number;
|
|
1006
|
+
limitBytes: number;
|
|
1007
|
+
withinLimit: boolean;
|
|
1008
|
+
breakdown: {
|
|
1009
|
+
sourceCodeBytes: number;
|
|
1010
|
+
staticPipelineBytes: number;
|
|
1011
|
+
bindingsBytes: number;
|
|
1012
|
+
descriptionBytes: number;
|
|
1013
|
+
};
|
|
1014
|
+
};
|
|
1015
|
+
bundle: {
|
|
1016
|
+
usedBytes: number;
|
|
1017
|
+
limitBytes: number;
|
|
1018
|
+
withinLimit: boolean;
|
|
1019
|
+
};
|
|
1020
|
+
};
|
|
1000
1021
|
}
|
|
1001
1022
|
|
|
1002
1023
|
/** Severity of a {@link PlayCheckIssue}. `error` fails the check; `warning` does not. */
|
|
@@ -288,6 +288,9 @@ const MAP_INCREMENTAL_PERSIST_CHUNK_ROWS = 100;
|
|
|
288
288
|
const MAP_INCREMENTAL_PERSIST_CHUNK_BYTES = 1 * 1024 * 1024;
|
|
289
289
|
const MAP_INCREMENTAL_PERSIST_INTERVAL_MS = 100;
|
|
290
290
|
const MAP_FRAME_FLUSH_INTERVAL_MS = 250;
|
|
291
|
+
// Batchable calls wait at most this long for same-key row continuations. The
|
|
292
|
+
// window is fixed from the first item and never resets on later arrivals.
|
|
293
|
+
const TOOL_BATCH_COALESCE_WINDOW_MS = 5;
|
|
291
294
|
const TOOL_RETRY_AFTER_FALLBACK_MS = 1_000;
|
|
292
295
|
const TOOL_RETRY_HEARTBEAT_INTERVAL_MS = 30_000;
|
|
293
296
|
const DEEPLINEAGENT_TOOL_RUNTIME_TIMEOUT_MS = 15 * 60 * 1000;
|
|
@@ -1196,6 +1199,14 @@ function createPacingResolver(
|
|
|
1196
1199
|
export class PlayContextImpl {
|
|
1197
1200
|
private rowStates = new Map<number, RowState>();
|
|
1198
1201
|
private toolCallQueue: ToolCallRequest[] = [];
|
|
1202
|
+
/**
|
|
1203
|
+
* Fixed, non-resetting coalescing deadlines for batchable ready work. A
|
|
1204
|
+
* deadline starts when the first request enters an empty batch bucket. New
|
|
1205
|
+
* arrivals never push it back, so sustained traffic cannot starve dispatch.
|
|
1206
|
+
*/
|
|
1207
|
+
private readonly toolBatchQueuedAtByKey = new Map<string, number>();
|
|
1208
|
+
private readonly toolDispatcherWakeWaiters = new Set<() => void>();
|
|
1209
|
+
private toolDispatcherFailure: unknown | null = null;
|
|
1199
1210
|
private toolCallResolvers = new Map<
|
|
1200
1211
|
string,
|
|
1201
1212
|
{ resolve: (value: unknown) => void; reject: (reason: unknown) => void }
|
|
@@ -6064,33 +6075,172 @@ export class PlayContextImpl {
|
|
|
6064
6075
|
}
|
|
6065
6076
|
}
|
|
6066
6077
|
|
|
6078
|
+
private toolBatchCoalesceKey(request: ToolCallRequest): string | null {
|
|
6079
|
+
const strategy =
|
|
6080
|
+
this.#options.getBatchOperationStrategy?.(request.toolId) ?? null;
|
|
6081
|
+
if (!strategy) return null;
|
|
6082
|
+
return [
|
|
6083
|
+
request.toolId,
|
|
6084
|
+
request.executionAuthScopeDigest ?? '',
|
|
6085
|
+
String(strategy.toBucketKey(request.input)),
|
|
6086
|
+
].join('\u0000');
|
|
6087
|
+
}
|
|
6088
|
+
|
|
6089
|
+
private wakeToolDispatcher(): void {
|
|
6090
|
+
const waiters = [...this.toolDispatcherWakeWaiters];
|
|
6091
|
+
this.toolDispatcherWakeWaiters.clear();
|
|
6092
|
+
for (const wake of waiters) wake();
|
|
6093
|
+
}
|
|
6094
|
+
|
|
6095
|
+
private enqueueToolCall(request: ToolCallRequest): void {
|
|
6096
|
+
if (this.toolDispatcherFailure != null) {
|
|
6097
|
+
const resolver = this.toolCallResolvers.get(request.callId);
|
|
6098
|
+
if (resolver) {
|
|
6099
|
+
resolver.reject(this.toolDispatcherFailure);
|
|
6100
|
+
this.toolCallResolvers.delete(request.callId);
|
|
6101
|
+
}
|
|
6102
|
+
return;
|
|
6103
|
+
}
|
|
6104
|
+
this.toolCallQueue.push(request);
|
|
6105
|
+
const batchKey = this.toolBatchCoalesceKey(request);
|
|
6106
|
+
if (batchKey && !this.toolBatchQueuedAtByKey.has(batchKey)) {
|
|
6107
|
+
this.toolBatchQueuedAtByKey.set(batchKey, Date.now());
|
|
6108
|
+
}
|
|
6109
|
+
this.wakeToolDispatcher();
|
|
6110
|
+
}
|
|
6111
|
+
|
|
6112
|
+
private rejectQueuedToolCalls(error: unknown): void {
|
|
6113
|
+
const queued = this.toolCallQueue;
|
|
6114
|
+
this.toolCallQueue = [];
|
|
6115
|
+
this.toolBatchQueuedAtByKey.clear();
|
|
6116
|
+
for (const request of queued) {
|
|
6117
|
+
const resolver = this.toolCallResolvers.get(request.callId);
|
|
6118
|
+
if (!resolver) continue;
|
|
6119
|
+
resolver.reject(error);
|
|
6120
|
+
this.toolCallResolvers.delete(request.callId);
|
|
6121
|
+
}
|
|
6122
|
+
}
|
|
6123
|
+
|
|
6124
|
+
private takeDispatchableToolCalls(nowMs: number): {
|
|
6125
|
+
requests: ToolCallRequest[];
|
|
6126
|
+
nextDeadlineMs: number | null;
|
|
6127
|
+
} {
|
|
6128
|
+
if (this.toolCallQueue.length === 0) {
|
|
6129
|
+
return { requests: [], nextDeadlineMs: null };
|
|
6130
|
+
}
|
|
6131
|
+
|
|
6132
|
+
const batchCounts = new Map<string, number>();
|
|
6133
|
+
const batchMaxSizes = new Map<string, number>();
|
|
6134
|
+
for (const request of this.toolCallQueue) {
|
|
6135
|
+
const batchKey = this.toolBatchCoalesceKey(request);
|
|
6136
|
+
if (!batchKey) continue;
|
|
6137
|
+
batchCounts.set(batchKey, (batchCounts.get(batchKey) ?? 0) + 1);
|
|
6138
|
+
const strategy =
|
|
6139
|
+
this.#options.getBatchOperationStrategy?.(request.toolId) ?? null;
|
|
6140
|
+
if (strategy) batchMaxSizes.set(batchKey, strategy.maxBatchSize);
|
|
6141
|
+
}
|
|
6142
|
+
|
|
6143
|
+
const readyBatchKeys = new Set<string>();
|
|
6144
|
+
let nextDeadlineMs: number | null = null;
|
|
6145
|
+
for (const [batchKey, count] of batchCounts) {
|
|
6146
|
+
const queuedAt = this.toolBatchQueuedAtByKey.get(batchKey) ?? nowMs;
|
|
6147
|
+
if (!this.toolBatchQueuedAtByKey.has(batchKey)) {
|
|
6148
|
+
this.toolBatchQueuedAtByKey.set(batchKey, queuedAt);
|
|
6149
|
+
}
|
|
6150
|
+
const deadline = queuedAt + TOOL_BATCH_COALESCE_WINDOW_MS;
|
|
6151
|
+
const maxBatchSize = batchMaxSizes.get(batchKey) ?? 1;
|
|
6152
|
+
if (count >= maxBatchSize || deadline <= nowMs) {
|
|
6153
|
+
readyBatchKeys.add(batchKey);
|
|
6154
|
+
} else {
|
|
6155
|
+
nextDeadlineMs =
|
|
6156
|
+
nextDeadlineMs == null
|
|
6157
|
+
? deadline
|
|
6158
|
+
: Math.min(nextDeadlineMs, deadline);
|
|
6159
|
+
}
|
|
6160
|
+
}
|
|
6161
|
+
|
|
6162
|
+
const requests: ToolCallRequest[] = [];
|
|
6163
|
+
const remaining: ToolCallRequest[] = [];
|
|
6164
|
+
for (const request of this.toolCallQueue) {
|
|
6165
|
+
const batchKey = this.toolBatchCoalesceKey(request);
|
|
6166
|
+
if (!batchKey || readyBatchKeys.has(batchKey)) {
|
|
6167
|
+
requests.push(request);
|
|
6168
|
+
} else {
|
|
6169
|
+
remaining.push(request);
|
|
6170
|
+
}
|
|
6171
|
+
}
|
|
6172
|
+
this.toolCallQueue = remaining;
|
|
6173
|
+
for (const batchKey of readyBatchKeys) {
|
|
6174
|
+
this.toolBatchQueuedAtByKey.delete(batchKey);
|
|
6175
|
+
}
|
|
6176
|
+
return { requests, nextDeadlineMs };
|
|
6177
|
+
}
|
|
6178
|
+
|
|
6179
|
+
private waitForToolDispatcherWake(deadlineMs: number | null): Promise<void> {
|
|
6180
|
+
return new Promise((resolve) => {
|
|
6181
|
+
let timer: ReturnType<typeof setTimeout> | null = null;
|
|
6182
|
+
const wake = () => {
|
|
6183
|
+
if (timer) clearTimeout(timer);
|
|
6184
|
+
this.toolDispatcherWakeWaiters.delete(wake);
|
|
6185
|
+
resolve();
|
|
6186
|
+
};
|
|
6187
|
+
this.toolDispatcherWakeWaiters.add(wake);
|
|
6188
|
+
if (deadlineMs != null) {
|
|
6189
|
+
timer = setTimeout(wake, Math.max(0, deadlineMs - Date.now()));
|
|
6190
|
+
}
|
|
6191
|
+
});
|
|
6192
|
+
}
|
|
6193
|
+
|
|
6067
6194
|
private async drainQueuedWork<T>(promises: Promise<T>[]): Promise<void> {
|
|
6068
|
-
//
|
|
6069
|
-
//
|
|
6070
|
-
//
|
|
6071
|
-
|
|
6072
|
-
|
|
6073
|
-
|
|
6074
|
-
|
|
6075
|
-
|
|
6076
|
-
|
|
6077
|
-
//
|
|
6078
|
-
//
|
|
6079
|
-
//
|
|
6080
|
-
|
|
6081
|
-
|
|
6082
|
-
|
|
6083
|
-
|
|
6084
|
-
|
|
6085
|
-
|
|
6086
|
-
|
|
6087
|
-
|
|
6088
|
-
|
|
6089
|
-
|
|
6090
|
-
|
|
6091
|
-
|
|
6092
|
-
|
|
6093
|
-
|
|
6195
|
+
// One dispatcher owns ready work, fixed batch coalescing deadlines, and
|
|
6196
|
+
// the in-flight registry. A row resumes after its own receipt persists; it
|
|
6197
|
+
// never waits for unrelated sibling calls from the previous column.
|
|
6198
|
+
const inFlightToolExecutions = new Set<Promise<void>>();
|
|
6199
|
+
let rowsSettled = false;
|
|
6200
|
+
void Promise.allSettled(promises).then(() => {
|
|
6201
|
+
rowsSettled = true;
|
|
6202
|
+
this.wakeToolDispatcher();
|
|
6203
|
+
});
|
|
6204
|
+
// A pending Promise does not keep a short-lived Node runner alive. Keep one
|
|
6205
|
+
// handle open without polling scheduler state; actual work wakes the
|
|
6206
|
+
// dispatcher through queue/in-flight/row-settlement signals.
|
|
6207
|
+
const keepAlive = setInterval(() => undefined, 1_000);
|
|
6208
|
+
try {
|
|
6209
|
+
let pass = 0;
|
|
6210
|
+
while (true) {
|
|
6211
|
+
if (this.toolDispatcherFailure != null) {
|
|
6212
|
+
this.rejectQueuedToolCalls(this.toolDispatcherFailure);
|
|
6213
|
+
await Promise.allSettled([...inFlightToolExecutions]);
|
|
6214
|
+
throw this.toolDispatcherFailure;
|
|
6215
|
+
}
|
|
6216
|
+
|
|
6217
|
+
const dispatchable = this.takeDispatchableToolCalls(Date.now());
|
|
6218
|
+
if (dispatchable.requests.length > 0) {
|
|
6219
|
+
pass += 1;
|
|
6220
|
+
this.log(` Batch pass ${pass}`);
|
|
6221
|
+
this.log(
|
|
6222
|
+
` Dispatcher launch: ready=${dispatchable.requests.length} ` +
|
|
6223
|
+
`queued=${this.toolCallQueue.length} ` +
|
|
6224
|
+
`in_flight_groups=${inFlightToolExecutions.size}`,
|
|
6225
|
+
);
|
|
6226
|
+
const tracked = this.executeBatchedToolCalls(dispatchable.requests)
|
|
6227
|
+
.catch((error) => {
|
|
6228
|
+
this.toolDispatcherFailure ??= error;
|
|
6229
|
+
this.rejectQueuedToolCalls(error);
|
|
6230
|
+
})
|
|
6231
|
+
.finally(() => {
|
|
6232
|
+
inFlightToolExecutions.delete(tracked);
|
|
6233
|
+
this.wakeToolDispatcher();
|
|
6234
|
+
});
|
|
6235
|
+
inFlightToolExecutions.add(tracked);
|
|
6236
|
+
continue;
|
|
6237
|
+
}
|
|
6238
|
+
|
|
6239
|
+
if (
|
|
6240
|
+
rowsSettled &&
|
|
6241
|
+
this.toolCallQueue.length === 0 &&
|
|
6242
|
+
inFlightToolExecutions.size === 0
|
|
6243
|
+
) {
|
|
6094
6244
|
if (!this.activeDatasetStep) {
|
|
6095
6245
|
const childPlaySuspension =
|
|
6096
6246
|
this.consumePendingChildPlaySuspension();
|
|
@@ -6101,23 +6251,10 @@ export class PlayContextImpl {
|
|
|
6101
6251
|
break;
|
|
6102
6252
|
}
|
|
6103
6253
|
|
|
6104
|
-
|
|
6105
|
-
// batch. Give resolver microtasks a turn, then loop back and drain any
|
|
6106
|
-
// newly queued work. This timer is deliberately tiny; it is here for
|
|
6107
|
-
// correctness/liveness, not throughput throttling.
|
|
6108
|
-
await new Promise((r) => setTimeout(r, 10));
|
|
6109
|
-
continue;
|
|
6110
|
-
}
|
|
6111
|
-
|
|
6112
|
-
pass++;
|
|
6113
|
-
this.log(` Batch pass ${pass}`);
|
|
6114
|
-
this.log(
|
|
6115
|
-
` Queue depth before drain: tool_calls=${this.toolCallQueue.length}`,
|
|
6116
|
-
);
|
|
6117
|
-
|
|
6118
|
-
if (this.toolCallQueue.length > 0) {
|
|
6119
|
-
await this.executeBatchedToolCalls();
|
|
6254
|
+
await this.waitForToolDispatcherWake(dispatchable.nextDeadlineMs);
|
|
6120
6255
|
}
|
|
6256
|
+
} finally {
|
|
6257
|
+
clearInterval(keepAlive);
|
|
6121
6258
|
}
|
|
6122
6259
|
}
|
|
6123
6260
|
|
|
@@ -6392,7 +6529,7 @@ export class PlayContextImpl {
|
|
|
6392
6529
|
toolId,
|
|
6393
6530
|
options?.timeoutMs,
|
|
6394
6531
|
);
|
|
6395
|
-
this.
|
|
6532
|
+
this.enqueueToolCall({
|
|
6396
6533
|
callId,
|
|
6397
6534
|
cacheKey: toolResultCacheKey,
|
|
6398
6535
|
cacheable: cacheableToolResult,
|
|
@@ -7407,10 +7544,9 @@ export class PlayContextImpl {
|
|
|
7407
7544
|
|
|
7408
7545
|
// ——— Batched tool call execution ———
|
|
7409
7546
|
|
|
7410
|
-
private async executeBatchedToolCalls(
|
|
7411
|
-
|
|
7412
|
-
|
|
7413
|
-
|
|
7547
|
+
private async executeBatchedToolCalls(
|
|
7548
|
+
queuedToolCalls: ToolCallRequest[],
|
|
7549
|
+
): Promise<void> {
|
|
7414
7550
|
// Group by toolId
|
|
7415
7551
|
const byTool = new Map<string, ToolCallRequest[]>();
|
|
7416
7552
|
for (const req of queuedToolCalls) {
|
|
@@ -7418,7 +7554,7 @@ export class PlayContextImpl {
|
|
|
7418
7554
|
byTool.get(req.toolId)!.push(req);
|
|
7419
7555
|
}
|
|
7420
7556
|
|
|
7421
|
-
await Promise.
|
|
7557
|
+
const toolSettlements = await Promise.allSettled(
|
|
7422
7558
|
[...byTool.entries()].map(async ([toolId, requests]) => {
|
|
7423
7559
|
this.log(`Executing tool batch ${toolId}: ${requests.length} calls`);
|
|
7424
7560
|
const liveStepResults = new Map<string, unknown>();
|
|
@@ -8341,6 +8477,11 @@ export class PlayContextImpl {
|
|
|
8341
8477
|
}
|
|
8342
8478
|
}),
|
|
8343
8479
|
);
|
|
8480
|
+
const rejected = toolSettlements.find(
|
|
8481
|
+
(settlement): settlement is PromiseRejectedResult =>
|
|
8482
|
+
settlement.status === 'rejected',
|
|
8483
|
+
);
|
|
8484
|
+
if (rejected) throw rejected.reason;
|
|
8344
8485
|
}
|
|
8345
8486
|
|
|
8346
8487
|
private async executeResolvedPlay(
|
|
@@ -8741,6 +8882,7 @@ export class PlayContextImpl {
|
|
|
8741
8882
|
const httpFailureAttempt = httpFailureAttempts.next({
|
|
8742
8883
|
toolId,
|
|
8743
8884
|
status: response.status,
|
|
8885
|
+
bodyText: text,
|
|
8744
8886
|
transientHttpRetrySafe: retrySafeTransientHttp,
|
|
8745
8887
|
});
|
|
8746
8888
|
const failure = classifyToolExecuteHttpFailure({
|
package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts
CHANGED
|
@@ -7,17 +7,15 @@ import { isIsolatedRuntimeSchedulerSchema } from '@shared_libs/play-runtime/runt
|
|
|
7
7
|
|
|
8
8
|
const DAYTONA_CREATE_TIMEOUT_SECONDS = 10;
|
|
9
9
|
const DAYTONA_CREATE_RETRY_DELAYS_MS = [0, 500, 1_500] as const;
|
|
10
|
-
|
|
10
|
+
// A detached runner is bounded by both the play runtime ceiling and Daytona's
|
|
11
|
+
// own inactivity stop. No background activity refresh keeps a wedged sandbox
|
|
12
|
+
// alive past those limits.
|
|
13
|
+
const DAYTONA_AUTO_STOP_INTERVAL_MINUTES = 30;
|
|
11
14
|
const DAYTONA_AUTO_ARCHIVE_INTERVAL_MINUTES = 15;
|
|
12
15
|
const DAYTONA_SANDBOX_LABEL_SOURCE = 'deepline-play-runner';
|
|
13
|
-
// The
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
// POST /sandbox/:id/resize (404 "Cannot POST"). The resource boundary is
|
|
17
|
-
// therefore the default snapshot's grant, verified against these pins after
|
|
18
|
-
// create. If Daytona changes the default snapshot's resources, sandbox
|
|
19
|
-
// acquisition fails loudly instead of silently running plays on a different
|
|
20
|
-
// compute boundary; bump these pins deliberately when that happens.
|
|
16
|
+
// The sandbox must be created at its final resource boundary. Do not add a
|
|
17
|
+
// resize call here: sandbox acquisition is launch-critical, and every extra
|
|
18
|
+
// control-plane round trip raises cold-start latency.
|
|
21
19
|
export const DAYTONA_SANDBOX_CPU = 1;
|
|
22
20
|
export const DAYTONA_SANDBOX_MEMORY_GIB = 1;
|
|
23
21
|
export const DAYTONA_SANDBOX_DISK_GIB = 3;
|
|
@@ -132,10 +132,7 @@ function redactDaytonaErrorDetail(value: string): string {
|
|
|
132
132
|
'$1[redacted]',
|
|
133
133
|
)
|
|
134
134
|
.replace(/(bearer\s+)[^\s,;]+/gi, '$1[redacted]')
|
|
135
|
-
.replace(
|
|
136
|
-
/((?:api[_-]?key|token)\s*[=:]\s*)[^\s,;]+/gi,
|
|
137
|
-
'$1[redacted]',
|
|
138
|
-
)
|
|
135
|
+
.replace(/((?:api[_-]?key|token)\s*[=:]\s*)[^\s,;]+/gi, '$1[redacted]')
|
|
139
136
|
.replace(/\s+/g, ' ')
|
|
140
137
|
.trim()
|
|
141
138
|
.slice(0, 1_000);
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
/** Maximum active user-code runtime for a standard play, in seconds. */
|
|
2
|
-
export const STANDARD_PLAY_RUNTIME_LIMIT_SECONDS =
|
|
3
|
-
export const STANDARD_PLAY_RUNTIME_LIMIT_LABEL = '
|
|
2
|
+
export const STANDARD_PLAY_RUNTIME_LIMIT_SECONDS = 30 * 60;
|
|
3
|
+
export const STANDARD_PLAY_RUNTIME_LIMIT_LABEL = '30 minutes';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Runner timeout includes setup, cleanup, and billing headroom after the
|
|
7
7
|
* user-code runtime cap.
|
|
8
8
|
*/
|
|
9
|
-
export const PLAY_RUNNER_TIMEOUT_SECONDS =
|
|
9
|
+
export const PLAY_RUNNER_TIMEOUT_SECONDS = 40 * 60;
|
|
10
10
|
|
|
11
11
|
/** TTL for workflow executor tokens, in seconds. */
|
|
12
12
|
export const WORKFLOW_EXECUTOR_TOKEN_TTL_SECONDS = PLAY_RUNNER_TIMEOUT_SECONDS;
|
|
@@ -173,6 +173,7 @@ export type PlaySchedulerSubmitInput = {
|
|
|
173
173
|
* case the worker falls back to the dev-collapse id (today's queue).
|
|
174
174
|
*/
|
|
175
175
|
absurdReleaseId?: string | null;
|
|
176
|
+
absurdReleaseEnvironment?: 'preview' | 'production';
|
|
176
177
|
/** Request-scoped Vercel Deployment Protection bypass for preview runtime callbacks. */
|
|
177
178
|
vercelProtectionBypassToken?: string | null;
|
|
178
179
|
/** Request-scoped, dev-only runtime fault injection header for black-box durability tests. */
|
|
@@ -5,6 +5,14 @@ import {
|
|
|
5
5
|
} from './tool-http-errors';
|
|
6
6
|
|
|
7
7
|
export const TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS = 2;
|
|
8
|
+
// A provider may need to finish the original request before it can replay the
|
|
9
|
+
// result for this key. With no Retry-After signal, these two polls give the
|
|
10
|
+
// original request ten seconds to finish without repeatedly hammering it.
|
|
11
|
+
export const TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS = [
|
|
12
|
+
5_000, 5_000,
|
|
13
|
+
] as const;
|
|
14
|
+
export const TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_MAX_ATTEMPTS =
|
|
15
|
+
TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS.length + 1;
|
|
8
16
|
export const TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS = 8;
|
|
9
17
|
export const TOOL_EXECUTE_BARE_RATE_LIMIT_MAX_ATTEMPTS = 2;
|
|
10
18
|
export const TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS = 3;
|
|
@@ -13,6 +21,13 @@ export const TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS = 1_000;
|
|
|
13
21
|
export const TOOL_EXECUTE_RETRY_DELAY_MAX_MS = 5_000;
|
|
14
22
|
export const TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS = 60_000;
|
|
15
23
|
export const TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE = 'AUTH_SCOPE_CHANGED';
|
|
24
|
+
/**
|
|
25
|
+
* A provider/action-local outcome contract emits this only when a provider has
|
|
26
|
+
* explicitly said that the same idempotency key is still executing. It is not
|
|
27
|
+
* a generic HTTP 409 policy.
|
|
28
|
+
*/
|
|
29
|
+
export const TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_CODE =
|
|
30
|
+
'UPSTREAM_IDEMPOTENCY_IN_PROGRESS';
|
|
16
31
|
|
|
17
32
|
export class ToolExecuteAuthScopeChangedError extends Error {
|
|
18
33
|
readonly code = TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE;
|
|
@@ -29,6 +44,7 @@ export type ToolExecuteHttpRetryDecision = {
|
|
|
29
44
|
attemptCap: number;
|
|
30
45
|
reason:
|
|
31
46
|
| 'rate_limit'
|
|
47
|
+
| 'idempotency_in_progress'
|
|
32
48
|
| 'retry_safe_transient_5xx'
|
|
33
49
|
| 'unsafe_transient_5xx'
|
|
34
50
|
| 'hard_billing_error'
|
|
@@ -49,6 +65,7 @@ export type ToolExecuteHttpFailureAttemptTracker = {
|
|
|
49
65
|
next(input: {
|
|
50
66
|
toolId: string;
|
|
51
67
|
status: number;
|
|
68
|
+
bodyText?: string;
|
|
52
69
|
transientHttpRetrySafe?: boolean;
|
|
53
70
|
}): number;
|
|
54
71
|
};
|
|
@@ -65,6 +82,29 @@ function parseJsonObject(text: string): Record<string, unknown> | null {
|
|
|
65
82
|
}
|
|
66
83
|
}
|
|
67
84
|
|
|
85
|
+
function isIdempotencyInProgressResponse(input: {
|
|
86
|
+
status: number;
|
|
87
|
+
bodyText: string;
|
|
88
|
+
}): boolean {
|
|
89
|
+
if (input.status !== 409) return false;
|
|
90
|
+
const body = parseJsonObject(input.bodyText);
|
|
91
|
+
return (
|
|
92
|
+
body?.code === TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_CODE &&
|
|
93
|
+
body.error_category === 'idempotency_in_progress'
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function idempotencyInProgressRetryDelayMs(attempt: number): number {
|
|
98
|
+
return (
|
|
99
|
+
TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS[
|
|
100
|
+
Math.min(
|
|
101
|
+
Math.max(0, attempt - 1),
|
|
102
|
+
TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS.length - 1,
|
|
103
|
+
)
|
|
104
|
+
] ?? TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
|
|
68
108
|
export function parseToolExecuteAuthScopeChangedError(input: {
|
|
69
109
|
status: number;
|
|
70
110
|
bodyText: string;
|
|
@@ -84,6 +124,7 @@ export function parseToolExecuteAuthScopeChangedError(input: {
|
|
|
84
124
|
|
|
85
125
|
function decideToolExecuteHttpRetry(input: {
|
|
86
126
|
status: number;
|
|
127
|
+
idempotencyInProgress?: boolean;
|
|
87
128
|
hardBillingFailure?: boolean;
|
|
88
129
|
hasRetryAfterHeader?: boolean;
|
|
89
130
|
transientHttpRetrySafe?: boolean;
|
|
@@ -110,6 +151,13 @@ function decideToolExecuteHttpRetry(input: {
|
|
|
110
151
|
reason: 'rate_limit',
|
|
111
152
|
};
|
|
112
153
|
}
|
|
154
|
+
if (input.idempotencyInProgress) {
|
|
155
|
+
return {
|
|
156
|
+
retryable: true,
|
|
157
|
+
attemptCap: TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_MAX_ATTEMPTS,
|
|
158
|
+
reason: 'idempotency_in_progress',
|
|
159
|
+
};
|
|
160
|
+
}
|
|
113
161
|
if (input.status >= 500 && input.status < 600) {
|
|
114
162
|
if (!input.transientHttpRetrySafe) {
|
|
115
163
|
return {
|
|
@@ -137,6 +185,7 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
|
|
|
137
185
|
number
|
|
138
186
|
> = {
|
|
139
187
|
rate_limit: 0,
|
|
188
|
+
idempotency_in_progress: 0,
|
|
140
189
|
retry_safe_transient_5xx: 0,
|
|
141
190
|
unsafe_transient_5xx: 0,
|
|
142
191
|
hard_billing_error: 0,
|
|
@@ -147,6 +196,10 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
|
|
|
147
196
|
next(input) {
|
|
148
197
|
const decision = decideToolExecuteHttpRetry({
|
|
149
198
|
status: input.status,
|
|
199
|
+
idempotencyInProgress: isIdempotencyInProgressResponse({
|
|
200
|
+
status: input.status,
|
|
201
|
+
bodyText: input.bodyText ?? '',
|
|
202
|
+
}),
|
|
150
203
|
hasRetryAfterHeader: true,
|
|
151
204
|
transientHttpRetrySafe: input.transientHttpRetrySafe === true,
|
|
152
205
|
});
|
|
@@ -186,8 +239,10 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
186
239
|
const hasRetryAfterHeader =
|
|
187
240
|
typeof input.retryAfterHeader === 'string' &&
|
|
188
241
|
input.retryAfterHeader.trim().length > 0;
|
|
242
|
+
const idempotencyInProgress = isIdempotencyInProgressResponse(input);
|
|
189
243
|
const initialRetryDecision = decideToolExecuteHttpRetry({
|
|
190
244
|
status: input.status,
|
|
245
|
+
idempotencyInProgress,
|
|
191
246
|
hasRetryAfterHeader,
|
|
192
247
|
transientHttpRetrySafe,
|
|
193
248
|
});
|
|
@@ -208,6 +263,7 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
208
263
|
const hardBillingFailure = isHardBillingToolHttpError(error);
|
|
209
264
|
const retryDecision = decideToolExecuteHttpRetry({
|
|
210
265
|
status: input.status,
|
|
266
|
+
idempotencyInProgress,
|
|
211
267
|
hardBillingFailure,
|
|
212
268
|
hasRetryAfterHeader,
|
|
213
269
|
transientHttpRetrySafe,
|
|
@@ -238,9 +294,12 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
238
294
|
TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt,
|
|
239
295
|
),
|
|
240
296
|
)
|
|
241
|
-
:
|
|
242
|
-
|
|
243
|
-
|
|
297
|
+
: retryDecision.reason === 'idempotency_in_progress' &&
|
|
298
|
+
!hasRetryAfterHeader
|
|
299
|
+
? idempotencyInProgressRetryDelayMs(input.attempt)
|
|
300
|
+
: retryAfterMs > 0
|
|
301
|
+
? Math.min(TOOL_EXECUTE_RETRY_DELAY_MAX_MS, retryAfterMs)
|
|
302
|
+
: TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
|
|
244
303
|
return {
|
|
245
304
|
...retryDecision,
|
|
246
305
|
error,
|