deepline 0.1.139 → 0.1.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +58 -18
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +46 -22
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/live-progress.ts +4 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-http-errors.ts +7 -262
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +15 -1
- package/dist/bundling-sources/sdk/src/client.ts +24 -0
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +32 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +54 -34
- package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/live-events.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/live-state-contract.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +59 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +55 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +248 -0
- package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +4 -0
- package/dist/cli/index.js +165 -42
- package/dist/cli/index.mjs +165 -42
- package/dist/index.d.mts +44 -0
- package/dist/index.d.ts +44 -0
- package/dist/index.js +36 -2
- package/dist/index.mjs +36 -2
- package/package.json +1 -1
|
@@ -53,6 +53,7 @@ import {
|
|
|
53
53
|
import {
|
|
54
54
|
decideWorkflowPlatformRetry,
|
|
55
55
|
PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
|
|
56
|
+
shouldPublishWorkflowRuntimeFailure,
|
|
56
57
|
} from './workflow-retry';
|
|
57
58
|
import {
|
|
58
59
|
WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES,
|
|
@@ -274,6 +275,7 @@ interface CoordinatorEnv {
|
|
|
274
275
|
DEEPLINE_INTERNAL_TOKEN?: string;
|
|
275
276
|
DEEPLINE_TAIL_LOG_TOKEN?: string;
|
|
276
277
|
DEEPLINE_COORDINATOR_DEPLOY_MARKER?: string;
|
|
278
|
+
CF_VERSION_METADATA?: WorkerVersionMetadata;
|
|
277
279
|
VERCEL_PROTECTION_BYPASS_TOKEN?: string;
|
|
278
280
|
DEEPLINE_PLAY_PREVIEW_SLUG?: string;
|
|
279
281
|
/**
|
|
@@ -3038,27 +3040,44 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
|
|
|
3038
3040
|
: null,
|
|
3039
3041
|
},
|
|
3040
3042
|
);
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3043
|
+
if (
|
|
3044
|
+
shouldPublishWorkflowRuntimeFailure({
|
|
3045
|
+
error: innerError,
|
|
3046
|
+
retryAttempts: 0,
|
|
3047
|
+
})
|
|
3048
|
+
) {
|
|
3049
|
+
await markWorkflowRuntimeFailure({
|
|
3050
|
+
env,
|
|
3051
|
+
event: innerEvent,
|
|
3052
|
+
error: innerError,
|
|
3053
|
+
}).catch((markError) => {
|
|
3054
|
+
console.error(
|
|
3055
|
+
'[coordinator] failed to forward DynamicWorkflow runner error',
|
|
3056
|
+
{
|
|
3057
|
+
graphHash,
|
|
3058
|
+
message:
|
|
3059
|
+
markError instanceof Error
|
|
3060
|
+
? markError.message
|
|
3061
|
+
: String(markError),
|
|
3062
|
+
},
|
|
3063
|
+
);
|
|
3064
|
+
});
|
|
3065
|
+
await writeCoordinatorTerminalState(env, {
|
|
3066
|
+
runId: runIdForTrace,
|
|
3067
|
+
status: 'failed',
|
|
3068
|
+
error: failure.message,
|
|
3069
|
+
}).catch(() => undefined);
|
|
3070
|
+
} else {
|
|
3071
|
+
console.warn(
|
|
3072
|
+
'[coordinator] DynamicWorkflow platform reset will be retried; not publishing durable run.failed',
|
|
3048
3073
|
{
|
|
3049
3074
|
graphHash,
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
: String(markError),
|
|
3075
|
+
runId: runIdForTrace,
|
|
3076
|
+
errorCode: failure.code,
|
|
3077
|
+
retryLimit: PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
|
|
3054
3078
|
},
|
|
3055
3079
|
);
|
|
3056
|
-
}
|
|
3057
|
-
await writeCoordinatorTerminalState(env, {
|
|
3058
|
-
runId: runIdForTrace,
|
|
3059
|
-
status: 'failed',
|
|
3060
|
-
error: failure.message,
|
|
3061
|
-
}).catch(() => undefined);
|
|
3080
|
+
}
|
|
3062
3081
|
throw innerError;
|
|
3063
3082
|
}
|
|
3064
3083
|
},
|
|
@@ -3136,7 +3155,20 @@ async function coordinatorRouteFetch(
|
|
|
3136
3155
|
): Promise<Response> {
|
|
3137
3156
|
const url = new URL(request.url);
|
|
3138
3157
|
if (url.pathname === '/health') {
|
|
3139
|
-
|
|
3158
|
+
const headers = new Headers();
|
|
3159
|
+
headers.set(
|
|
3160
|
+
'x-deepline-runtime-deploy-version',
|
|
3161
|
+
env.CF_VERSION_METADATA?.id ??
|
|
3162
|
+
env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
|
|
3163
|
+
'',
|
|
3164
|
+
);
|
|
3165
|
+
if (env.DEEPLINE_COORDINATOR_DEPLOY_MARKER) {
|
|
3166
|
+
headers.set(
|
|
3167
|
+
'x-deepline-deploy-marker',
|
|
3168
|
+
env.DEEPLINE_COORDINATOR_DEPLOY_MARKER,
|
|
3169
|
+
);
|
|
3170
|
+
}
|
|
3171
|
+
return new Response('ok', { status: 200, headers });
|
|
3140
3172
|
}
|
|
3141
3173
|
if (url.pathname === '/warmup/submit') {
|
|
3142
3174
|
const authError = authorizeCoordinatorControlRequest({ request, env });
|
|
@@ -3174,6 +3206,10 @@ async function coordinatorRouteFetch(
|
|
|
3174
3206
|
return Response.json({
|
|
3175
3207
|
ok: true,
|
|
3176
3208
|
deployMarker: env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ?? null,
|
|
3209
|
+
runtimeDeployVersion:
|
|
3210
|
+
env.CF_VERSION_METADATA?.id ??
|
|
3211
|
+
env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
|
|
3212
|
+
null,
|
|
3177
3213
|
});
|
|
3178
3214
|
}
|
|
3179
3215
|
if (url.pathname === '/staged-files/put') {
|
|
@@ -3683,6 +3719,10 @@ async function handleWorkflowRoute(input: {
|
|
|
3683
3719
|
status: 'submitted',
|
|
3684
3720
|
workflowInstanceId: instance.id,
|
|
3685
3721
|
instanceState,
|
|
3722
|
+
runtimeDeployVersion:
|
|
3723
|
+
env.CF_VERSION_METADATA?.id ??
|
|
3724
|
+
env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
|
|
3725
|
+
null,
|
|
3686
3726
|
coordinatorTimings,
|
|
3687
3727
|
});
|
|
3688
3728
|
} finally {
|
|
@@ -76,6 +76,12 @@ import {
|
|
|
76
76
|
type ToolExecuteResult,
|
|
77
77
|
type ToolResultMetadataInput,
|
|
78
78
|
} from '../../../shared_libs/play-runtime/tool-result';
|
|
79
|
+
import {
|
|
80
|
+
TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS,
|
|
81
|
+
TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
|
|
82
|
+
TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS,
|
|
83
|
+
decideToolExecuteHttpRetry,
|
|
84
|
+
} from '../../../shared_libs/play-runtime/tool-execute-retry-policy';
|
|
79
85
|
import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
80
86
|
import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
|
|
81
87
|
import type { PlayRuntimeManifestMap } from '../../../shared_libs/plays/compiler-manifest';
|
|
@@ -1223,12 +1229,11 @@ async function callToolDirect(
|
|
|
1223
1229
|
): Promise<ToolExecuteResult> {
|
|
1224
1230
|
const { id, toolId, input } = args;
|
|
1225
1231
|
const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
|
|
1226
|
-
const maxAttempts = 3;
|
|
1227
1232
|
let lastError: Error | null = null;
|
|
1228
1233
|
|
|
1229
1234
|
for (
|
|
1230
1235
|
let attempt = 1;
|
|
1231
|
-
attempt <=
|
|
1236
|
+
attempt <= TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS;
|
|
1232
1237
|
attempt += 1
|
|
1233
1238
|
) {
|
|
1234
1239
|
let res: Response;
|
|
@@ -1247,18 +1252,18 @@ async function callToolDirect(
|
|
|
1247
1252
|
} catch (error) {
|
|
1248
1253
|
const message = error instanceof Error ? error.message : String(error);
|
|
1249
1254
|
lastError = new Error(
|
|
1250
|
-
`Tool ${toolId} transport failed calling ${path} for run ${req.runId} on attempt ${attempt}/${
|
|
1255
|
+
`Tool ${toolId} transport failed calling ${path} for run ${req.runId} on attempt ${attempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}: ${message}`,
|
|
1251
1256
|
);
|
|
1252
1257
|
if (
|
|
1253
|
-
attempt >=
|
|
1258
|
+
attempt >= TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS ||
|
|
1254
1259
|
!isRetryableRuntimeApiError(error)
|
|
1255
1260
|
) {
|
|
1256
1261
|
throw lastError;
|
|
1257
1262
|
}
|
|
1258
1263
|
onRetryAttempt?.();
|
|
1259
|
-
const delayMs =
|
|
1264
|
+
const delayMs = TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS * attempt;
|
|
1260
1265
|
console.warn(
|
|
1261
|
-
`[deepline-run:${req.runId}] tool transport retry tool=${toolId} path=${path} attempt=${attempt}/${
|
|
1266
|
+
`[deepline-run:${req.runId}] tool transport retry tool=${toolId} path=${path} attempt=${attempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} retryAfterMs=${delayMs} error=${redactSecretsFromLogString(message)}`,
|
|
1262
1267
|
);
|
|
1263
1268
|
await sleepWorkerMs(delayMs);
|
|
1264
1269
|
continue;
|
|
@@ -1276,18 +1281,25 @@ async function callToolDirect(
|
|
|
1276
1281
|
|
|
1277
1282
|
const text = await res.text().catch(() => '');
|
|
1278
1283
|
const isRateLimited = res.status === 429;
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
: maxAttempts;
|
|
1284
|
+
const initialRetryDecision = decideToolExecuteHttpRetry({
|
|
1285
|
+
toolId,
|
|
1286
|
+
status: res.status,
|
|
1287
|
+
});
|
|
1284
1288
|
lastError = normalizeToolHttpErrorMessage({
|
|
1285
1289
|
toolId,
|
|
1286
1290
|
status: res.status,
|
|
1287
1291
|
attempt,
|
|
1288
|
-
maxAttempts: attemptCap,
|
|
1292
|
+
maxAttempts: initialRetryDecision.attemptCap,
|
|
1289
1293
|
bodyText: text,
|
|
1290
1294
|
});
|
|
1295
|
+
// Rate-limit pushback gets the larger 429-specific retry budget, unless the
|
|
1296
|
+
// current response body is a hard Deepline billing denial.
|
|
1297
|
+
const retryDecision = decideToolExecuteHttpRetry({
|
|
1298
|
+
toolId,
|
|
1299
|
+
status: res.status,
|
|
1300
|
+
hardBillingFailure: isHardBillingToolHttpError(lastError),
|
|
1301
|
+
});
|
|
1302
|
+
const attemptCap = retryDecision.attemptCap;
|
|
1291
1303
|
const retryAfterSeconds = Number(res.headers.get('retry-after'));
|
|
1292
1304
|
const retryAfterMs =
|
|
1293
1305
|
Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0
|
|
@@ -1298,10 +1310,7 @@ async function callToolDirect(
|
|
|
1298
1310
|
// final attempt so the (org, provider) bucket backs off across isolates.
|
|
1299
1311
|
onProviderBackpressure?.(retryAfterMs > 0 ? retryAfterMs : 1_000);
|
|
1300
1312
|
}
|
|
1301
|
-
|
|
1302
|
-
(isRateLimited && !isHardBillingToolHttpError(lastError)) ||
|
|
1303
|
-
(res.status >= 500 && WORKER_RETRY_SAFE_5XX_TOOLS.has(toolId));
|
|
1304
|
-
if (!retryable || attempt >= attemptCap) {
|
|
1313
|
+
if (!retryDecision.retryable || attempt >= attemptCap) {
|
|
1305
1314
|
throw lastError;
|
|
1306
1315
|
}
|
|
1307
1316
|
// Charge the retry budget per attempt, matching the cjs runner's
|
|
@@ -1490,7 +1499,6 @@ const MAP_ROW_FAILURE_SAMPLE_LIMIT = 3;
|
|
|
1490
1499
|
// their previous batching behavior; declared providers tighten via the
|
|
1491
1500
|
// Governor's suggestedParallelism.
|
|
1492
1501
|
const WORKER_TOOL_BATCH_DEFAULT_PARALLELISM = 4;
|
|
1493
|
-
const WORKER_RETRY_SAFE_5XX_TOOLS = new Set(['test_transient_500']);
|
|
1494
1502
|
/**
|
|
1495
1503
|
* In-process retry budget for HTTP 429 tool responses. Rate-limit pushback is
|
|
1496
1504
|
* throughput pacing (provider or Deepline limiter), not a tool defect, so it
|
|
@@ -1499,9 +1507,6 @@ const WORKER_RETRY_SAFE_5XX_TOOLS = new Set(['test_transient_500']);
|
|
|
1499
1507
|
* throttling before the call fails. Every retry still charges the Governor's
|
|
1500
1508
|
* retry budget, so a runaway storm stays bounded and loud.
|
|
1501
1509
|
*/
|
|
1502
|
-
const WORKER_TOOL_RATE_LIMIT_MAX_ATTEMPTS = 8;
|
|
1503
|
-
const WORKER_TOOL_TRANSPORT_MAX_ATTEMPTS = 3;
|
|
1504
|
-
const WORKER_TOOL_TRANSPORT_RETRY_DELAY_MS = 1_000;
|
|
1505
1510
|
|
|
1506
1511
|
function sleepWorkerMs(ms: number): Promise<void> {
|
|
1507
1512
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
@@ -2473,7 +2478,7 @@ async function postRuntimeEgressFetch(
|
|
|
2473
2478
|
let lastError: Error | null = null;
|
|
2474
2479
|
for (
|
|
2475
2480
|
let attempt = 1;
|
|
2476
|
-
attempt <=
|
|
2481
|
+
attempt <= TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS;
|
|
2477
2482
|
attempt += 1
|
|
2478
2483
|
) {
|
|
2479
2484
|
const response = await fetchRuntimeApi(
|
|
@@ -2519,7 +2524,7 @@ async function postRuntimeEgressFetch(
|
|
|
2519
2524
|
? Math.ceil(retryAfterSeconds * 1000)
|
|
2520
2525
|
: 1_000;
|
|
2521
2526
|
onProviderBackpressure?.(retryAfterMs);
|
|
2522
|
-
if (attempt >=
|
|
2527
|
+
if (attempt >= TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS) {
|
|
2523
2528
|
throw lastError;
|
|
2524
2529
|
}
|
|
2525
2530
|
onRetryAttempt?.();
|
|
@@ -3943,6 +3948,13 @@ function createMinimalWorkerCtx(
|
|
|
3943
3948
|
{
|
|
3944
3949
|
completed: completedRowsForProgress(),
|
|
3945
3950
|
total: progressTotalRows,
|
|
3951
|
+
startedRows: startedExecutedRows,
|
|
3952
|
+
activeRows: activeExecutedRows,
|
|
3953
|
+
waitingRows: Math.max(
|
|
3954
|
+
0,
|
|
3955
|
+
rowsToExecute.length - startedExecutedRows,
|
|
3956
|
+
),
|
|
3957
|
+
completedRows: completedExecutedRows,
|
|
3946
3958
|
startedAt: mapStartedAt,
|
|
3947
3959
|
message: formatMapExecutionHeartbeatMessage({
|
|
3948
3960
|
rowsToExecute: rowsToExecute.length,
|
|
@@ -6147,6 +6159,18 @@ async function executeRunRequest(
|
|
|
6147
6159
|
...(typeof progress.failed === 'number'
|
|
6148
6160
|
? { failed: progress.failed }
|
|
6149
6161
|
: {}),
|
|
6162
|
+
...(typeof progress.startedRows === 'number'
|
|
6163
|
+
? { startedRows: progress.startedRows }
|
|
6164
|
+
: {}),
|
|
6165
|
+
...(typeof progress.activeRows === 'number'
|
|
6166
|
+
? { activeRows: progress.activeRows }
|
|
6167
|
+
: {}),
|
|
6168
|
+
...(typeof progress.waitingRows === 'number'
|
|
6169
|
+
? { waitingRows: progress.waitingRows }
|
|
6170
|
+
: {}),
|
|
6171
|
+
...(typeof progress.completedRows === 'number'
|
|
6172
|
+
? { completedRows: progress.completedRows }
|
|
6173
|
+
: {}),
|
|
6150
6174
|
...(typeof progress.message === 'string' && progress.message
|
|
6151
6175
|
? { message: progress.message }
|
|
6152
6176
|
: {}),
|
|
@@ -1,262 +1,7 @@
|
|
|
1
|
-
export
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
billing: Record<string, unknown> | null,
|
|
9
|
-
status: number,
|
|
10
|
-
) {
|
|
11
|
-
super(message);
|
|
12
|
-
this.name = 'ToolHttpError';
|
|
13
|
-
this.billing = billing;
|
|
14
|
-
this.status = status;
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
function formatCreditAmount(value: unknown): string {
|
|
19
|
-
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
20
|
-
return String(value ?? '-');
|
|
21
|
-
}
|
|
22
|
-
return Number(value.toFixed(8)).toString();
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
26
|
-
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
function getStringField(value: unknown, key: string): string | null {
|
|
30
|
-
if (!isRecord(value)) return null;
|
|
31
|
-
const field = value[key];
|
|
32
|
-
return typeof field === 'string' && field.trim() ? field : null;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function getObjectField(
|
|
36
|
-
value: unknown,
|
|
37
|
-
key: string,
|
|
38
|
-
): Record<string, unknown> | null {
|
|
39
|
-
if (!isRecord(value)) return null;
|
|
40
|
-
const field = value[key];
|
|
41
|
-
return isRecord(field) ? field : null;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
function isInsufficientCreditsBilling(
|
|
45
|
-
billing: Record<string, unknown> | null,
|
|
46
|
-
): billing is Record<string, unknown> {
|
|
47
|
-
return billing?.kind === 'insufficient_credits';
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
function isHardBillingFailurePayload(
|
|
51
|
-
payload: Record<string, unknown> | null,
|
|
52
|
-
): payload is Record<string, unknown> {
|
|
53
|
-
if (!payload) return false;
|
|
54
|
-
const category = String(
|
|
55
|
-
payload.error_category ?? payload.errorCategory ?? '',
|
|
56
|
-
).toLowerCase();
|
|
57
|
-
const code = String(payload.code ?? payload.error_code ?? '').toUpperCase();
|
|
58
|
-
const message = String(
|
|
59
|
-
payload.error ?? payload.message ?? payload.failure_description ?? '',
|
|
60
|
-
).toLowerCase();
|
|
61
|
-
if (category === 'billing') return true;
|
|
62
|
-
if (
|
|
63
|
-
code === 'INSUFFICIENT_CREDITS' ||
|
|
64
|
-
code === 'BILLING_CAP_EXCEEDED' ||
|
|
65
|
-
code === 'MONTHLY_BILLING_LIMIT_EXCEEDED'
|
|
66
|
-
) {
|
|
67
|
-
return true;
|
|
68
|
-
}
|
|
69
|
-
return (
|
|
70
|
-
(message.includes('billing cap') ||
|
|
71
|
-
message.includes('monthly billing limit') ||
|
|
72
|
-
message.includes('rolling 30-day organization billing cap') ||
|
|
73
|
-
message.includes('insufficient credits')) &&
|
|
74
|
-
!message.includes('rate limit')
|
|
75
|
-
);
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
function normalizeHardBillingPayload(
|
|
79
|
-
payload: Record<string, unknown>,
|
|
80
|
-
): Record<string, unknown> {
|
|
81
|
-
return {
|
|
82
|
-
kind: 'billing_cap_exceeded',
|
|
83
|
-
code:
|
|
84
|
-
typeof payload.code === 'string' && payload.code.trim()
|
|
85
|
-
? payload.code
|
|
86
|
-
: 'MONTHLY_BILLING_LIMIT_EXCEEDED',
|
|
87
|
-
error_category: 'billing',
|
|
88
|
-
failure_origin:
|
|
89
|
-
typeof payload.failure_origin === 'string' &&
|
|
90
|
-
payload.failure_origin.trim()
|
|
91
|
-
? payload.failure_origin
|
|
92
|
-
: 'deepline_billing',
|
|
93
|
-
message:
|
|
94
|
-
typeof payload.error === 'string' && payload.error.trim()
|
|
95
|
-
? payload.error
|
|
96
|
-
: typeof payload.message === 'string' && payload.message.trim()
|
|
97
|
-
? payload.message
|
|
98
|
-
: 'Deepline billing cap exceeded.',
|
|
99
|
-
...payload,
|
|
100
|
-
};
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
function formatHardBillingFailureMessage(input: {
|
|
104
|
-
billing: Record<string, unknown>;
|
|
105
|
-
toolId: string;
|
|
106
|
-
status: number;
|
|
107
|
-
attempt: number;
|
|
108
|
-
maxAttempts: number;
|
|
109
|
-
}): string {
|
|
110
|
-
const code = getStringField(input.billing, 'code');
|
|
111
|
-
const message =
|
|
112
|
-
getStringField(input.billing, 'message') ??
|
|
113
|
-
getStringField(input.billing, 'error') ??
|
|
114
|
-
'Deepline billing cap exceeded.';
|
|
115
|
-
return [
|
|
116
|
-
`tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}:`,
|
|
117
|
-
'Deepline billing cap exceeded.',
|
|
118
|
-
'Run halted before marking remaining rows processed.',
|
|
119
|
-
code ? `code=${code}.` : '',
|
|
120
|
-
message,
|
|
121
|
-
]
|
|
122
|
-
.filter(Boolean)
|
|
123
|
-
.join(' ');
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
function formatInsufficientCreditsMessage(input: {
|
|
127
|
-
billing: Record<string, unknown>;
|
|
128
|
-
toolId: string;
|
|
129
|
-
}): string {
|
|
130
|
-
const operation =
|
|
131
|
-
getStringField(input.billing, 'operation_id') ??
|
|
132
|
-
getStringField(input.billing, 'operation') ??
|
|
133
|
-
input.toolId;
|
|
134
|
-
const balance = formatCreditAmount(input.billing.balance_credits);
|
|
135
|
-
const required = formatCreditAmount(input.billing.required_credits);
|
|
136
|
-
const recommended = formatCreditAmount(
|
|
137
|
-
input.billing.recommended_add_credits ?? input.billing.needed_credits,
|
|
138
|
-
);
|
|
139
|
-
const billingUrl = getStringField(input.billing, 'billing_url');
|
|
140
|
-
const addSuffix =
|
|
141
|
-
billingUrl && recommended !== '-'
|
|
142
|
-
? ` Add >=${recommended} at ${billingUrl}.`
|
|
143
|
-
: billingUrl
|
|
144
|
-
? ` Add credits at ${billingUrl}.`
|
|
145
|
-
: '';
|
|
146
|
-
return `Workspace balance ${balance} < required ${required} for ${operation}.${addSuffix}`;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
function formatPublicToolErrorPayload(input: {
|
|
150
|
-
parsed: Record<string, unknown> | null;
|
|
151
|
-
bodyText: string;
|
|
152
|
-
}): string {
|
|
153
|
-
if (!input.parsed) {
|
|
154
|
-
return input.bodyText.slice(0, 500);
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
const selected: Record<string, unknown> = {};
|
|
158
|
-
for (const key of [
|
|
159
|
-
'error',
|
|
160
|
-
'message',
|
|
161
|
-
'code',
|
|
162
|
-
'failure_origin',
|
|
163
|
-
'error_category',
|
|
164
|
-
'failure_description',
|
|
165
|
-
'operator_hint',
|
|
166
|
-
'failure_hint',
|
|
167
|
-
'details',
|
|
168
|
-
'provider',
|
|
169
|
-
'operation',
|
|
170
|
-
'request_id',
|
|
171
|
-
'requestId',
|
|
172
|
-
'credential_source',
|
|
173
|
-
'credential_owner',
|
|
174
|
-
]) {
|
|
175
|
-
const value = input.parsed[key];
|
|
176
|
-
if (typeof value === 'string' && value.trim()) {
|
|
177
|
-
selected[key] = value;
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
return JSON.stringify(
|
|
182
|
-
Object.keys(selected).length > 0 ? selected : input.parsed,
|
|
183
|
-
).slice(0, 1_500);
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
export function normalizeToolHttpErrorMessage(input: {
|
|
187
|
-
toolId: string;
|
|
188
|
-
status: number;
|
|
189
|
-
attempt: number;
|
|
190
|
-
maxAttempts: number;
|
|
191
|
-
bodyText: string;
|
|
192
|
-
}): ToolHttpError {
|
|
193
|
-
let parsed: Record<string, unknown> | null = null;
|
|
194
|
-
try {
|
|
195
|
-
const candidate = JSON.parse(input.bodyText);
|
|
196
|
-
parsed = isRecord(candidate) ? candidate : null;
|
|
197
|
-
} catch {
|
|
198
|
-
parsed = null;
|
|
199
|
-
}
|
|
200
|
-
const billing = getObjectField(parsed, 'billing');
|
|
201
|
-
if (isInsufficientCreditsBilling(billing)) {
|
|
202
|
-
return new ToolHttpError(
|
|
203
|
-
`tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage(
|
|
204
|
-
{
|
|
205
|
-
billing,
|
|
206
|
-
toolId: input.toolId,
|
|
207
|
-
},
|
|
208
|
-
)}`,
|
|
209
|
-
billing,
|
|
210
|
-
input.status,
|
|
211
|
-
);
|
|
212
|
-
}
|
|
213
|
-
const hardBillingPayload = isHardBillingFailurePayload(billing)
|
|
214
|
-
? normalizeHardBillingPayload(billing)
|
|
215
|
-
: isHardBillingFailurePayload(parsed)
|
|
216
|
-
? normalizeHardBillingPayload(parsed)
|
|
217
|
-
: null;
|
|
218
|
-
if (hardBillingPayload) {
|
|
219
|
-
return new ToolHttpError(
|
|
220
|
-
formatHardBillingFailureMessage({
|
|
221
|
-
billing: hardBillingPayload,
|
|
222
|
-
toolId: input.toolId,
|
|
223
|
-
status: input.status,
|
|
224
|
-
attempt: input.attempt,
|
|
225
|
-
maxAttempts: input.maxAttempts,
|
|
226
|
-
}),
|
|
227
|
-
hardBillingPayload,
|
|
228
|
-
input.status,
|
|
229
|
-
);
|
|
230
|
-
}
|
|
231
|
-
return new ToolHttpError(
|
|
232
|
-
`tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatPublicToolErrorPayload(
|
|
233
|
-
{
|
|
234
|
-
parsed,
|
|
235
|
-
bodyText: input.bodyText,
|
|
236
|
-
},
|
|
237
|
-
)}`,
|
|
238
|
-
billing,
|
|
239
|
-
input.status,
|
|
240
|
-
);
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
export function extractErrorBilling(
|
|
244
|
-
error: unknown,
|
|
245
|
-
): Record<string, unknown> | null {
|
|
246
|
-
return error instanceof ToolHttpError ? error.billing : null;
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
export function isHardBillingToolHttpError(error: unknown): boolean {
|
|
250
|
-
return (
|
|
251
|
-
error instanceof ToolHttpError && isHardBillingFailurePayload(error.billing)
|
|
252
|
-
);
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
/**
|
|
256
|
-
* A tool call that ultimately failed with HTTP 429 — provider or
|
|
257
|
-
* Deepline-internal rate-limit pushback that survived the in-process retry
|
|
258
|
-
* budget. This is run-level throughput pressure, never a row-specific defect.
|
|
259
|
-
*/
|
|
260
|
-
export function isRateLimitToolHttpError(error: unknown): boolean {
|
|
261
|
-
return error instanceof ToolHttpError && error.status === 429;
|
|
262
|
-
}
|
|
1
|
+
export {
|
|
2
|
+
ToolHttpError,
|
|
3
|
+
extractErrorBilling,
|
|
4
|
+
isHardBillingToolHttpError,
|
|
5
|
+
isRateLimitToolHttpError,
|
|
6
|
+
normalizeToolHttpErrorMessage,
|
|
7
|
+
} from '../../../../shared_libs/play-runtime/tool-http-errors';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { normalizePlayRunFailure } from '../../../shared_libs/play-runtime/run-failure';
|
|
2
2
|
|
|
3
|
-
export const PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT =
|
|
3
|
+
export const PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT = 5;
|
|
4
4
|
|
|
5
5
|
export type WorkflowRetryDecision =
|
|
6
6
|
| {
|
|
@@ -44,3 +44,17 @@ export function decideWorkflowPlatformRetry(input: {
|
|
|
44
44
|
message: failure.message,
|
|
45
45
|
};
|
|
46
46
|
}
|
|
47
|
+
|
|
48
|
+
export function shouldPublishWorkflowRuntimeFailure(input: {
|
|
49
|
+
error: unknown;
|
|
50
|
+
retryAttempts: number;
|
|
51
|
+
}): boolean {
|
|
52
|
+
const message =
|
|
53
|
+
input.error instanceof Error ? input.error.message : String(input.error);
|
|
54
|
+
const decision = decideWorkflowPlatformRetry({
|
|
55
|
+
workflowStatus: 'errored',
|
|
56
|
+
error: message,
|
|
57
|
+
retryAttempts: input.retryAttempts,
|
|
58
|
+
});
|
|
59
|
+
return decision.action !== 'retry';
|
|
60
|
+
}
|
|
@@ -75,6 +75,7 @@ import type {
|
|
|
75
75
|
ToolSearchResult,
|
|
76
76
|
ToolMetadata,
|
|
77
77
|
CustomerDbQueryResult,
|
|
78
|
+
DeeplineAgentModelDescription,
|
|
78
79
|
} from './types.js';
|
|
79
80
|
import type { PlayStagedFileRef } from './plays/local-file-discovery.js';
|
|
80
81
|
import type { PlayCompilerManifest } from '../../shared_libs/plays/compiler-manifest.js';
|
|
@@ -1113,6 +1114,29 @@ export class DeeplineClient {
|
|
|
1113
1114
|
);
|
|
1114
1115
|
}
|
|
1115
1116
|
|
|
1117
|
+
/**
|
|
1118
|
+
* Describe a Deepline Agent model and its provider-specific option surface.
|
|
1119
|
+
*
|
|
1120
|
+
* Combines live AI Gateway model metadata with Deepline's generated AI SDK
|
|
1121
|
+
* provider option registry so agents can construct `providerOptions`
|
|
1122
|
+
* payloads before executing `deeplineagent`.
|
|
1123
|
+
*
|
|
1124
|
+
* The returned option schemas describe accepted provider option shapes, not
|
|
1125
|
+
* guaranteed support for every model. Runtime AI SDK/Gateway errors remain
|
|
1126
|
+
* authoritative for model-gated values.
|
|
1127
|
+
*
|
|
1128
|
+
* @param model - Gateway model id such as `"openai/gpt-5.5"`
|
|
1129
|
+
* @returns Model metadata, provider option shapes, and runnable examples
|
|
1130
|
+
*/
|
|
1131
|
+
async describeModel(model: string): Promise<DeeplineAgentModelDescription> {
|
|
1132
|
+
return this.http.request<DeeplineAgentModelDescription>(
|
|
1133
|
+
`/api/v2/models/describe?model=${encodeURIComponent(model)}`,
|
|
1134
|
+
{
|
|
1135
|
+
method: 'GET',
|
|
1136
|
+
},
|
|
1137
|
+
);
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1116
1140
|
/**
|
|
1117
1141
|
* Execute a tool and return the standard execution envelope.
|
|
1118
1142
|
*
|
|
@@ -101,10 +101,10 @@ export const SDK_RELEASE = {
|
|
|
101
101
|
// 0.1.108 ships explicit dataset column/tool recompute policy and removes
|
|
102
102
|
// the SDK enrich generator's one-second stale policy.
|
|
103
103
|
// 0.1.110 ships authored V2 prebuilts and required top-level play descriptions.
|
|
104
|
-
version: '0.1.
|
|
104
|
+
version: '0.1.141',
|
|
105
105
|
apiContract: '2026-06-dataset-column-cell-stale-hard-cutover',
|
|
106
106
|
supportPolicy: {
|
|
107
|
-
latest: '0.1.
|
|
107
|
+
latest: '0.1.141',
|
|
108
108
|
minimumSupported: '0.1.53',
|
|
109
109
|
deprecatedBelow: '0.1.53',
|
|
110
110
|
commandMinimumSupported: [
|
|
@@ -247,6 +247,38 @@ export interface ToolDefinition {
|
|
|
247
247
|
}>;
|
|
248
248
|
}
|
|
249
249
|
|
|
250
|
+
export interface ModelProviderOptionField {
|
|
251
|
+
name: string;
|
|
252
|
+
type: 'string' | 'number' | 'boolean' | 'object' | 'array';
|
|
253
|
+
enumValues?: string[];
|
|
254
|
+
description: string;
|
|
255
|
+
caveat?: string;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export interface ModelProviderOptionNamespace {
|
|
259
|
+
provider: string;
|
|
260
|
+
sourcePackage: string;
|
|
261
|
+
sourceSymbol: string;
|
|
262
|
+
fields: ModelProviderOptionField[];
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
export interface DeeplineAgentModelDescription {
|
|
266
|
+
schemaVersion: 1;
|
|
267
|
+
model: string;
|
|
268
|
+
provider: string;
|
|
269
|
+
modelMetadata: Record<string, unknown> | null;
|
|
270
|
+
providerOptions: {
|
|
271
|
+
gateway: ModelProviderOptionNamespace;
|
|
272
|
+
selectedProvider?: ModelProviderOptionNamespace;
|
|
273
|
+
};
|
|
274
|
+
exampleInput: {
|
|
275
|
+
model: string;
|
|
276
|
+
providerOptions: Record<string, unknown>;
|
|
277
|
+
};
|
|
278
|
+
caveats: string[];
|
|
279
|
+
sources: string[];
|
|
280
|
+
}
|
|
281
|
+
|
|
250
282
|
/**
|
|
251
283
|
* Query options for ranked tool/provider discovery.
|
|
252
284
|
*/
|