deepline 0.1.300 → 0.1.302
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +8 -0
- package/dist/bundling-sources/sdk/src/errors.ts +57 -15
- package/dist/bundling-sources/sdk/src/http.ts +82 -5
- package/dist/bundling-sources/sdk/src/index.ts +4 -0
- package/dist/bundling-sources/sdk/src/play.ts +29 -1
- package/dist/bundling-sources/sdk/src/release.ts +5 -3
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +120 -14
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +59 -11
- package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/execution-plan.ts +0 -15
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +136 -36
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +78 -5
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +17 -12
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +210 -5
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +4 -0
- package/dist/bundling-sources/shared_libs/plays/artifact-contract-version.ts +6 -0
- package/dist/bundling-sources/shared_libs/plays/artifact-types.ts +4 -0
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +159 -12
- package/dist/bundling-sources/shared_libs/plays/contracts.ts +69 -1
- package/dist/bundling-sources/shared_libs/tool-execution-error.ts +428 -0
- package/dist/cli/index.js +501 -179
- package/dist/cli/index.mjs +501 -179
- package/dist/index.d.mts +39 -18
- package/dist/index.d.ts +39 -18
- package/dist/index.js +394 -59
- package/dist/index.mjs +391 -59
- package/dist/plays/bundle-play-file.d.mts +4 -2
- package/dist/plays/bundle-play-file.d.ts +4 -2
- package/dist/plays/bundle-play-file.mjs +116 -19
- package/dist/{compiler-manifest-BjopNIqx.d.mts → tool-execution-error-9mH4i-mJ.d.mts} +79 -1
- package/dist/{compiler-manifest-BjopNIqx.d.ts → tool-execution-error-9mH4i-mJ.d.ts} +79 -1
- package/package.json +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/completed-receipt-cache.ts +0 -166
|
@@ -124,6 +124,7 @@ export class RuntimeReceiptStoreAdapter {
|
|
|
124
124
|
maxBatchBytes?: number;
|
|
125
125
|
maxFlushMs?: number;
|
|
126
126
|
maxBufferedBytes?: number;
|
|
127
|
+
onRetryTelemetry?: (line: string) => void;
|
|
127
128
|
}) {
|
|
128
129
|
const context: WorkerRuntimeApiContext = {
|
|
129
130
|
...options.context,
|
|
@@ -137,6 +138,14 @@ export class RuntimeReceiptStoreAdapter {
|
|
|
137
138
|
maxFlushMs: options.maxFlushMs,
|
|
138
139
|
maxBufferedBytes: options.maxBufferedBytes,
|
|
139
140
|
classifyRetryableError: classifyReceiptRetry,
|
|
141
|
+
onRetryEvent: (event) =>
|
|
142
|
+
emitReceiptWriterRetryTelemetry({
|
|
143
|
+
event,
|
|
144
|
+
playName: options.playName,
|
|
145
|
+
runId: options.runId,
|
|
146
|
+
gatewayHost: runtimeGatewayHost(context.baseUrl),
|
|
147
|
+
onTelemetry: options.onRetryTelemetry,
|
|
148
|
+
}),
|
|
140
149
|
send: async (commands, { signal }) =>
|
|
141
150
|
await sendReceiptCommands({
|
|
142
151
|
context: { ...context, signal },
|
|
@@ -260,6 +269,100 @@ export class RuntimeReceiptStoreAdapter {
|
|
|
260
269
|
}
|
|
261
270
|
}
|
|
262
271
|
|
|
272
|
+
const RECEIPT_WRITER_RETRY_TELEMETRY_TAG =
|
|
273
|
+
'[perf][worker.receipt_writer.retry]';
|
|
274
|
+
|
|
275
|
+
function isPowerOfTwo(value: number): boolean {
|
|
276
|
+
return value > 0 && (value & (value - 1)) === 0;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function runtimeGatewayHost(baseUrl: string): string | null {
|
|
280
|
+
try {
|
|
281
|
+
return new URL(baseUrl).host || null;
|
|
282
|
+
} catch {
|
|
283
|
+
return null;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function receiptCommandTelemetryAction(command: ReceiptCommand): string {
|
|
288
|
+
return command.kind === 'settle'
|
|
289
|
+
? `${command.kind}_${command.outcome}`
|
|
290
|
+
: command.kind;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function receiptRetryErrorTelemetry(error: unknown): Record<string, unknown> {
|
|
294
|
+
if (isAppRuntimeApiCapacityError(error)) {
|
|
295
|
+
return {
|
|
296
|
+
failureKind: 'capacity',
|
|
297
|
+
errorName: error.name,
|
|
298
|
+
status: error.status,
|
|
299
|
+
code: error.code,
|
|
300
|
+
requestId: error.requestId,
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
if (error instanceof AppRuntimeApiTransportError) {
|
|
304
|
+
return {
|
|
305
|
+
failureKind: 'transport',
|
|
306
|
+
errorName: error.name,
|
|
307
|
+
transportAttempts: error.attempts,
|
|
308
|
+
transportAction: error.action,
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
if (error instanceof AppRuntimeApiResponseError) {
|
|
312
|
+
return {
|
|
313
|
+
failureKind: 'http',
|
|
314
|
+
errorName: error.name,
|
|
315
|
+
status: error.status,
|
|
316
|
+
code: error.code,
|
|
317
|
+
requestId: error.requestId,
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
return {
|
|
321
|
+
failureKind: 'unknown',
|
|
322
|
+
errorName: error instanceof Error ? error.name : typeof error,
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
function emitReceiptWriterRetryTelemetry(input: {
|
|
327
|
+
event: import('./runtime-receipt-writer').RuntimeReceiptWriterRetryEvent<ReceiptCommand>;
|
|
328
|
+
playName: string;
|
|
329
|
+
runId: string;
|
|
330
|
+
gatewayHost: string | null;
|
|
331
|
+
onTelemetry?: (line: string) => void;
|
|
332
|
+
}): void {
|
|
333
|
+
const { event } = input;
|
|
334
|
+
if (event.phase === 'retry' && !isPowerOfTwo(event.attempt)) return;
|
|
335
|
+
const payload = {
|
|
336
|
+
phase: event.phase,
|
|
337
|
+
playName: input.playName,
|
|
338
|
+
runId: input.runId,
|
|
339
|
+
gatewayHost: input.gatewayHost,
|
|
340
|
+
action: receiptCommandTelemetryAction(event.firstInput),
|
|
341
|
+
batchId: event.batchId,
|
|
342
|
+
batchSize: event.batchSize,
|
|
343
|
+
attempt: event.attempt,
|
|
344
|
+
elapsedMs: event.elapsedMs,
|
|
345
|
+
retryAfterMs: event.retryAfterMs,
|
|
346
|
+
queued: event.queued,
|
|
347
|
+
blocked: event.blocked,
|
|
348
|
+
bufferedBytes: event.bufferedBytes,
|
|
349
|
+
...(event.error === undefined
|
|
350
|
+
? { failureKind: null }
|
|
351
|
+
: receiptRetryErrorTelemetry(event.error)),
|
|
352
|
+
};
|
|
353
|
+
try {
|
|
354
|
+
const serialized = JSON.stringify(payload);
|
|
355
|
+
input.onTelemetry?.(`${RECEIPT_WRITER_RETRY_TELEMETRY_TAG} ${serialized}`);
|
|
356
|
+
if (event.phase === 'retry') {
|
|
357
|
+
console.warn(RECEIPT_WRITER_RETRY_TELEMETRY_TAG, serialized);
|
|
358
|
+
} else {
|
|
359
|
+
console.info(RECEIPT_WRITER_RETRY_TELEMETRY_TAG, serialized);
|
|
360
|
+
}
|
|
361
|
+
} catch {
|
|
362
|
+
// Receipt telemetry must never affect durable delivery.
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
263
366
|
function receiptCommandBatchKey(command: ReceiptCommand): string {
|
|
264
367
|
switch (command.kind) {
|
|
265
368
|
case 'get':
|
|
@@ -22,6 +22,20 @@ export type RuntimeReceiptWriterStats = {
|
|
|
22
22
|
bufferedBytes: number;
|
|
23
23
|
};
|
|
24
24
|
|
|
25
|
+
export type RuntimeReceiptWriterRetryEvent<Input> = {
|
|
26
|
+
phase: 'retry' | 'recovered';
|
|
27
|
+
attempt: number;
|
|
28
|
+
batchId: number;
|
|
29
|
+
batchSize: number;
|
|
30
|
+
firstInput: Input;
|
|
31
|
+
elapsedMs: number;
|
|
32
|
+
retryAfterMs: number;
|
|
33
|
+
queued: number;
|
|
34
|
+
blocked: number;
|
|
35
|
+
bufferedBytes: number;
|
|
36
|
+
error?: unknown;
|
|
37
|
+
};
|
|
38
|
+
|
|
25
39
|
export class RuntimeReceiptWriterBufferLimitError extends Error {
|
|
26
40
|
constructor(
|
|
27
41
|
readonly inputBytes: number,
|
|
@@ -76,11 +90,15 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
76
90
|
readonly #maxBatchBytes: number;
|
|
77
91
|
readonly #maxBufferedBytes: number;
|
|
78
92
|
readonly #maxFlushMs: number;
|
|
93
|
+
readonly #onRetryEvent:
|
|
94
|
+
| ((event: RuntimeReceiptWriterRetryEvent<Input>) => void)
|
|
95
|
+
| null;
|
|
79
96
|
readonly #queued: Array<PendingWrite<Input, Output>> = [];
|
|
80
97
|
readonly #blocked: Array<PendingWrite<Input, Output>> = [];
|
|
81
98
|
readonly #closeController = new AbortController();
|
|
82
99
|
|
|
83
100
|
#bufferedBytes = 0;
|
|
101
|
+
#nextBatchId = 0;
|
|
84
102
|
#activeBatch: Array<PendingWrite<Input, Output>> | null = null;
|
|
85
103
|
#pump: Promise<void> | null = null;
|
|
86
104
|
#flushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
@@ -103,6 +121,7 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
103
121
|
maxBatchBytes?: number;
|
|
104
122
|
maxBufferedBytes?: number;
|
|
105
123
|
maxFlushMs?: number;
|
|
124
|
+
onRetryEvent?: (event: RuntimeReceiptWriterRetryEvent<Input>) => void;
|
|
106
125
|
}) {
|
|
107
126
|
this.#batchKey = options.batchKey;
|
|
108
127
|
this.#send = options.send;
|
|
@@ -119,6 +138,7 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
119
138
|
this.#maxBufferedBytes,
|
|
120
139
|
);
|
|
121
140
|
this.#maxFlushMs = Math.max(0, Math.floor(options.maxFlushMs ?? 5));
|
|
141
|
+
this.#onRetryEvent = options.onRetryEvent ?? null;
|
|
122
142
|
}
|
|
123
143
|
|
|
124
144
|
write(input: Input, options: { signal?: AbortSignal } = {}): Promise<Output> {
|
|
@@ -243,9 +263,10 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
243
263
|
while (this.#queued.length > 0) {
|
|
244
264
|
const batch = this.#takeBatch();
|
|
245
265
|
if (batch.length === 0) continue;
|
|
266
|
+
const batchId = ++this.#nextBatchId;
|
|
246
267
|
this.#activeBatch = batch;
|
|
247
268
|
try {
|
|
248
|
-
const outputs = await this.#sendWithRetry(batch);
|
|
269
|
+
const outputs = await this.#sendWithRetry(batch, batchId);
|
|
249
270
|
if (outputs.length !== batch.length) {
|
|
250
271
|
throw new RuntimeReceiptWriterResultCountError(
|
|
251
272
|
batch.length,
|
|
@@ -305,9 +326,13 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
305
326
|
|
|
306
327
|
async #sendWithRetry(
|
|
307
328
|
batch: readonly PendingWrite<Input, Output>[],
|
|
329
|
+
batchId: number,
|
|
308
330
|
): Promise<readonly Output[]> {
|
|
309
331
|
const inputs = batch.map((pending) => pending.input);
|
|
332
|
+
const startedAt = Date.now();
|
|
333
|
+
let attempt = 0;
|
|
310
334
|
while (true) {
|
|
335
|
+
attempt += 1;
|
|
311
336
|
if (this.#closeController.signal.aborted) {
|
|
312
337
|
throw this.#closeReason;
|
|
313
338
|
}
|
|
@@ -315,20 +340,68 @@ export class RuntimeReceiptWriter<Input, Output> {
|
|
|
315
340
|
throw batch[0]?.aborted;
|
|
316
341
|
}
|
|
317
342
|
try {
|
|
318
|
-
|
|
343
|
+
const outputs = await this.#send(inputs, {
|
|
319
344
|
signal: this.#closeController.signal,
|
|
320
345
|
});
|
|
346
|
+
if (attempt > 1) {
|
|
347
|
+
this.#emitRetryEvent({
|
|
348
|
+
phase: 'recovered',
|
|
349
|
+
attempt,
|
|
350
|
+
batchId,
|
|
351
|
+
batch,
|
|
352
|
+
startedAt,
|
|
353
|
+
retryAfterMs: 0,
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
return outputs;
|
|
321
357
|
} catch (error) {
|
|
322
358
|
const retry = this.#classifyRetryableError(error);
|
|
323
359
|
if (!retry) throw error;
|
|
324
|
-
|
|
325
|
-
|
|
360
|
+
const retryAfterMs = jitteredRetryDelayMs(retry.retryAfterMs);
|
|
361
|
+
this.#emitRetryEvent({
|
|
362
|
+
phase: 'retry',
|
|
363
|
+
attempt,
|
|
364
|
+
batchId,
|
|
326
365
|
batch,
|
|
327
|
-
|
|
366
|
+
startedAt,
|
|
367
|
+
retryAfterMs,
|
|
368
|
+
error,
|
|
369
|
+
});
|
|
370
|
+
await this.#waitForRetry(retryAfterMs, batch);
|
|
328
371
|
}
|
|
329
372
|
}
|
|
330
373
|
}
|
|
331
374
|
|
|
375
|
+
#emitRetryEvent(input: {
|
|
376
|
+
phase: 'retry' | 'recovered';
|
|
377
|
+
attempt: number;
|
|
378
|
+
batchId: number;
|
|
379
|
+
batch: readonly PendingWrite<Input, Output>[];
|
|
380
|
+
startedAt: number;
|
|
381
|
+
retryAfterMs: number;
|
|
382
|
+
error?: unknown;
|
|
383
|
+
}): void {
|
|
384
|
+
const firstInput = input.batch[0]?.input;
|
|
385
|
+
if (!this.#onRetryEvent || firstInput === undefined) return;
|
|
386
|
+
try {
|
|
387
|
+
this.#onRetryEvent({
|
|
388
|
+
phase: input.phase,
|
|
389
|
+
attempt: input.attempt,
|
|
390
|
+
batchId: input.batchId,
|
|
391
|
+
batchSize: input.batch.length,
|
|
392
|
+
firstInput,
|
|
393
|
+
elapsedMs: Date.now() - input.startedAt,
|
|
394
|
+
retryAfterMs: input.retryAfterMs,
|
|
395
|
+
queued: this.#queued.length,
|
|
396
|
+
blocked: this.#blocked.length,
|
|
397
|
+
bufferedBytes: this.#bufferedBytes,
|
|
398
|
+
...(input.error === undefined ? {} : { error: input.error }),
|
|
399
|
+
});
|
|
400
|
+
} catch {
|
|
401
|
+
// Receipt telemetry must never affect durable delivery.
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
332
405
|
async #waitForRetry(
|
|
333
406
|
delayMs: number,
|
|
334
407
|
batch: readonly PendingWrite<Input, Output>[],
|
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
normalizeToolHttpErrorMessage,
|
|
4
4
|
type ToolHttpError,
|
|
5
5
|
} from './tool-http-errors';
|
|
6
|
+
import type { ToolExecutionErrorSchemaVersion } from '../tool-execution-error';
|
|
6
7
|
|
|
7
8
|
export const TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS = 2;
|
|
8
9
|
// A provider may need to finish the original request before it can replay the
|
|
@@ -296,6 +297,7 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
296
297
|
status: number;
|
|
297
298
|
attempt: number;
|
|
298
299
|
bodyText: string;
|
|
300
|
+
schemaVersion?: ToolExecutionErrorSchemaVersion;
|
|
299
301
|
retryAfterHeader?: string | null;
|
|
300
302
|
transientHttpRetrySafe?: boolean;
|
|
301
303
|
providerLatencyMs?: number;
|
|
@@ -318,12 +320,20 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
318
320
|
hasRetryAfterHeader,
|
|
319
321
|
transientHttpRetrySafe,
|
|
320
322
|
});
|
|
323
|
+
const retryAfterMs = parseToolExecuteRetryAfterMs(
|
|
324
|
+
input.retryAfterHeader,
|
|
325
|
+
input.nowMs,
|
|
326
|
+
);
|
|
321
327
|
const error = normalizeToolHttpErrorMessage({
|
|
322
328
|
toolId: input.toolId,
|
|
323
329
|
status: input.status,
|
|
324
330
|
attempt: input.attempt,
|
|
325
331
|
maxAttempts: initialRetryDecision.attemptCap,
|
|
326
332
|
bodyText: input.bodyText,
|
|
333
|
+
schemaVersion: input.schemaVersion,
|
|
334
|
+
retryable: initialRetryDecision.retryable,
|
|
335
|
+
retryAfterMs:
|
|
336
|
+
input.status === 429 && hasRetryAfterHeader ? retryAfterMs : null,
|
|
327
337
|
});
|
|
328
338
|
if (
|
|
329
339
|
typeof input.providerLatencyMs === 'number' &&
|
|
@@ -355,10 +365,6 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
355
365
|
) {
|
|
356
366
|
error.receiptFailureKind = 'repairable';
|
|
357
367
|
}
|
|
358
|
-
const retryAfterMs = parseToolExecuteRetryAfterMs(
|
|
359
|
-
input.retryAfterHeader,
|
|
360
|
-
input.nowMs,
|
|
361
|
-
);
|
|
362
368
|
const retryDelayMs =
|
|
363
369
|
input.status === 429
|
|
364
370
|
? Math.min(
|
|
@@ -369,14 +375,14 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
369
375
|
),
|
|
370
376
|
)
|
|
371
377
|
: retryDecision.reason === 'idempotency_in_progress' &&
|
|
372
|
-
!hasRetryAfterHeader
|
|
373
|
-
? idempotencyInProgressRetryDelayMs(input.attempt)
|
|
374
|
-
: retryDecision.reason === 'gateway_invocation_in_progress' &&
|
|
375
378
|
!hasRetryAfterHeader
|
|
376
379
|
? idempotencyInProgressRetryDelayMs(input.attempt)
|
|
377
|
-
:
|
|
378
|
-
|
|
379
|
-
|
|
380
|
+
: retryDecision.reason === 'gateway_invocation_in_progress' &&
|
|
381
|
+
!hasRetryAfterHeader
|
|
382
|
+
? idempotencyInProgressRetryDelayMs(input.attempt)
|
|
383
|
+
: retryAfterMs > 0
|
|
384
|
+
? Math.min(TOOL_EXECUTE_RETRY_DELAY_MAX_MS, retryAfterMs)
|
|
385
|
+
: TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
|
|
380
386
|
return {
|
|
381
387
|
...retryDecision,
|
|
382
388
|
error,
|
|
@@ -395,7 +401,6 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
395
401
|
: TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS
|
|
396
402
|
: null,
|
|
397
403
|
chargeRetryBudget:
|
|
398
|
-
shouldRetry &&
|
|
399
|
-
retryDecision.reason !== 'gateway_invocation_in_progress',
|
|
404
|
+
shouldRetry && retryDecision.reason !== 'gateway_invocation_in_progress',
|
|
400
405
|
};
|
|
401
406
|
}
|
|
@@ -1,5 +1,50 @@
|
|
|
1
1
|
import type { WorkReceiptFailureKind } from './work-receipts';
|
|
2
|
+
import {
|
|
3
|
+
LEGACY_TOOL_EXECUTION_ERROR_SCHEMA_VERSION,
|
|
4
|
+
ToolExecutionError,
|
|
5
|
+
deserializeToolExecutionFailure,
|
|
6
|
+
isProviderTransientFailure,
|
|
7
|
+
normalizeToolExecutionCategory,
|
|
8
|
+
normalizeToolExecutionOrigin,
|
|
9
|
+
type ToolExecutionErrorSchemaVersion,
|
|
10
|
+
type ToolExecutionErrorOptions,
|
|
11
|
+
} from '../tool-execution-error';
|
|
2
12
|
|
|
13
|
+
const TOOL_HTTP_ERROR_BRAND = Symbol.for('deepline.tool-http-error.v1');
|
|
14
|
+
|
|
15
|
+
type ToolHttpErrorConstructorInput = {
|
|
16
|
+
message: string;
|
|
17
|
+
billing: Record<string, unknown> | null;
|
|
18
|
+
status: number;
|
|
19
|
+
receiptFailureKind: WorkReceiptFailureKind;
|
|
20
|
+
options: Partial<ToolExecutionErrorOptions>;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
function applyToolHttpErrorFields(
|
|
24
|
+
error: {
|
|
25
|
+
name: string;
|
|
26
|
+
billing: Record<string, unknown> | null;
|
|
27
|
+
status: number;
|
|
28
|
+
receiptFailureKind: WorkReceiptFailureKind;
|
|
29
|
+
},
|
|
30
|
+
input: ToolHttpErrorConstructorInput,
|
|
31
|
+
): void {
|
|
32
|
+
error.name = 'ToolHttpError';
|
|
33
|
+
error.billing = input.billing;
|
|
34
|
+
error.status = input.status;
|
|
35
|
+
error.receiptFailureKind = input.receiptFailureKind;
|
|
36
|
+
Object.defineProperty(error, TOOL_HTTP_ERROR_BRAND, {
|
|
37
|
+
configurable: false,
|
|
38
|
+
enumerable: false,
|
|
39
|
+
writable: false,
|
|
40
|
+
value: true,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Historical play-runtime HTTP error. Existing artifacts must keep this exact
|
|
46
|
+
* Error prototype; typed errors are an explicit artifact-contract opt-in.
|
|
47
|
+
*/
|
|
3
48
|
export class ToolHttpError extends Error {
|
|
4
49
|
readonly billing: Record<string, unknown> | null;
|
|
5
50
|
/** HTTP status of the failed tool-execute response (e.g. 429, 502). */
|
|
@@ -11,15 +56,89 @@ export class ToolHttpError extends Error {
|
|
|
11
56
|
billing: Record<string, unknown> | null,
|
|
12
57
|
status: number,
|
|
13
58
|
receiptFailureKind: WorkReceiptFailureKind = 'terminal',
|
|
59
|
+
_options: Partial<ToolExecutionErrorOptions> = {},
|
|
14
60
|
) {
|
|
15
61
|
super(message);
|
|
16
|
-
this.name = 'ToolHttpError';
|
|
17
62
|
this.billing = billing;
|
|
18
63
|
this.status = status;
|
|
19
64
|
this.receiptFailureKind = receiptFailureKind;
|
|
65
|
+
applyToolHttpErrorFields(this, {
|
|
66
|
+
message,
|
|
67
|
+
billing,
|
|
68
|
+
status,
|
|
69
|
+
receiptFailureKind,
|
|
70
|
+
options: _options,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
static [Symbol.hasInstance](value: unknown): boolean {
|
|
75
|
+
return Boolean(
|
|
76
|
+
value &&
|
|
77
|
+
typeof value === 'object' &&
|
|
78
|
+
(value as Record<symbol, unknown>)[TOOL_HTTP_ERROR_BRAND] === true,
|
|
79
|
+
);
|
|
20
80
|
}
|
|
21
81
|
}
|
|
22
82
|
|
|
83
|
+
class StructuredToolHttpError extends ToolExecutionError {
|
|
84
|
+
readonly billing: Record<string, unknown> | null;
|
|
85
|
+
readonly status: number;
|
|
86
|
+
receiptFailureKind: WorkReceiptFailureKind;
|
|
87
|
+
|
|
88
|
+
constructor(input: ToolHttpErrorConstructorInput) {
|
|
89
|
+
super(input.message, {
|
|
90
|
+
toolId: input.options.toolId ?? 'unknown_tool',
|
|
91
|
+
provider: input.options.provider ?? null,
|
|
92
|
+
operation: input.options.operation ?? null,
|
|
93
|
+
code: input.options.code ?? null,
|
|
94
|
+
origin: input.options.origin ?? 'unknown',
|
|
95
|
+
category: input.options.category ?? 'unknown',
|
|
96
|
+
retryable: input.options.retryable === true,
|
|
97
|
+
statusCode:
|
|
98
|
+
input.options.statusCode === undefined
|
|
99
|
+
? input.status
|
|
100
|
+
: input.options.statusCode,
|
|
101
|
+
requestId: input.options.requestId ?? null,
|
|
102
|
+
retryAfterMs: input.options.retryAfterMs ?? null,
|
|
103
|
+
networkKind: input.options.networkKind ?? null,
|
|
104
|
+
networkScope: input.options.networkScope ?? null,
|
|
105
|
+
});
|
|
106
|
+
this.billing = input.billing;
|
|
107
|
+
this.status = input.status;
|
|
108
|
+
this.receiptFailureKind = input.receiptFailureKind;
|
|
109
|
+
applyToolHttpErrorFields(this, input);
|
|
110
|
+
this.name = isProviderTransientFailure(this)
|
|
111
|
+
? 'ProviderTransientError'
|
|
112
|
+
: 'ToolExecutionError';
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function createToolHttpError(
|
|
117
|
+
schemaVersion: ToolExecutionErrorSchemaVersion,
|
|
118
|
+
message: string,
|
|
119
|
+
billing: Record<string, unknown> | null,
|
|
120
|
+
status: number,
|
|
121
|
+
receiptFailureKind: WorkReceiptFailureKind = 'terminal',
|
|
122
|
+
options: Partial<ToolExecutionErrorOptions> = {},
|
|
123
|
+
): ToolHttpError {
|
|
124
|
+
if (schemaVersion === LEGACY_TOOL_EXECUTION_ERROR_SCHEMA_VERSION) {
|
|
125
|
+
return new ToolHttpError(
|
|
126
|
+
message,
|
|
127
|
+
billing,
|
|
128
|
+
status,
|
|
129
|
+
receiptFailureKind,
|
|
130
|
+
options,
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
return new StructuredToolHttpError({
|
|
134
|
+
message,
|
|
135
|
+
billing,
|
|
136
|
+
status,
|
|
137
|
+
receiptFailureKind,
|
|
138
|
+
options,
|
|
139
|
+
}) as ToolHttpError;
|
|
140
|
+
}
|
|
141
|
+
|
|
23
142
|
function formatCreditAmount(value: unknown): string {
|
|
24
143
|
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
25
144
|
return String(value ?? '-');
|
|
@@ -222,7 +341,12 @@ export function normalizeToolHttpErrorMessage(input: {
|
|
|
222
341
|
attempt: number;
|
|
223
342
|
maxAttempts: number;
|
|
224
343
|
bodyText: string;
|
|
344
|
+
retryable?: boolean;
|
|
345
|
+
retryAfterMs?: number | null;
|
|
346
|
+
schemaVersion?: ToolExecutionErrorSchemaVersion;
|
|
225
347
|
}): ToolHttpError {
|
|
348
|
+
const schemaVersion =
|
|
349
|
+
input.schemaVersion ?? LEGACY_TOOL_EXECUTION_ERROR_SCHEMA_VERSION;
|
|
226
350
|
let parsed: Record<string, unknown> | null = null;
|
|
227
351
|
try {
|
|
228
352
|
const candidate = JSON.parse(input.bodyText);
|
|
@@ -230,9 +354,64 @@ export function normalizeToolHttpErrorMessage(input: {
|
|
|
230
354
|
} catch {
|
|
231
355
|
parsed = null;
|
|
232
356
|
}
|
|
357
|
+
const serializedFailure = parsed?.tool_error;
|
|
358
|
+
const hydratedFailure = deserializeToolExecutionFailure(
|
|
359
|
+
'',
|
|
360
|
+
serializedFailure,
|
|
361
|
+
schemaVersion,
|
|
362
|
+
);
|
|
363
|
+
const provider =
|
|
364
|
+
getStringField(parsed, 'provider') ??
|
|
365
|
+
hydratedFailure?.provider ??
|
|
366
|
+
input.toolId.split(/[._:./-]+/)[0]?.trim() ??
|
|
367
|
+
null;
|
|
368
|
+
const operation =
|
|
369
|
+
getStringField(parsed, 'operation') ??
|
|
370
|
+
hydratedFailure?.operation ??
|
|
371
|
+
input.toolId;
|
|
372
|
+
const code = getStringField(parsed, 'code') ?? hydratedFailure?.code ?? null;
|
|
373
|
+
const rawOrigin =
|
|
374
|
+
getStringField(parsed, 'failure_origin') ?? hydratedFailure?.origin;
|
|
375
|
+
const rawCategory =
|
|
376
|
+
getStringField(parsed, 'error_category') ?? hydratedFailure?.category;
|
|
377
|
+
const origin = normalizeToolExecutionOrigin(rawOrigin, {
|
|
378
|
+
provider,
|
|
379
|
+
operation,
|
|
380
|
+
});
|
|
381
|
+
const category = normalizeToolExecutionCategory(rawCategory);
|
|
382
|
+
const trustworthy = origin !== 'unknown' && category !== 'unknown';
|
|
383
|
+
const publicOptions: Partial<ToolExecutionErrorOptions> = {
|
|
384
|
+
toolId: input.toolId,
|
|
385
|
+
provider,
|
|
386
|
+
operation,
|
|
387
|
+
code,
|
|
388
|
+
origin,
|
|
389
|
+
category,
|
|
390
|
+
retryable: trustworthy
|
|
391
|
+
? (hydratedFailure?.retryable ?? input.retryable === true)
|
|
392
|
+
: false,
|
|
393
|
+
statusCode: input.status,
|
|
394
|
+
requestId:
|
|
395
|
+
getStringField(parsed, 'request_id') ??
|
|
396
|
+
getStringField(parsed, 'requestId') ??
|
|
397
|
+
hydratedFailure?.requestId ??
|
|
398
|
+
null,
|
|
399
|
+
retryAfterMs: hydratedFailure?.retryAfterMs ?? input.retryAfterMs ?? null,
|
|
400
|
+
networkKind:
|
|
401
|
+
hydratedFailure?.networkKind ??
|
|
402
|
+
(category === 'network' && code?.toUpperCase().includes('TIMEOUT')
|
|
403
|
+
? 'timeout'
|
|
404
|
+
: null),
|
|
405
|
+
networkScope:
|
|
406
|
+
hydratedFailure?.networkScope ??
|
|
407
|
+
(origin === 'provider' && category === 'network'
|
|
408
|
+
? 'deepline_to_provider'
|
|
409
|
+
: null),
|
|
410
|
+
};
|
|
233
411
|
const billing = getObjectField(parsed, 'billing');
|
|
234
412
|
if (isInsufficientCreditsBilling(billing)) {
|
|
235
|
-
return
|
|
413
|
+
return createToolHttpError(
|
|
414
|
+
schemaVersion,
|
|
236
415
|
`tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatInsufficientCreditsMessage(
|
|
237
416
|
{
|
|
238
417
|
billing,
|
|
@@ -241,6 +420,14 @@ export function normalizeToolHttpErrorMessage(input: {
|
|
|
241
420
|
)}`,
|
|
242
421
|
billing,
|
|
243
422
|
input.status,
|
|
423
|
+
'terminal',
|
|
424
|
+
{
|
|
425
|
+
...publicOptions,
|
|
426
|
+
origin: 'deepline',
|
|
427
|
+
category: 'billing',
|
|
428
|
+
code: publicOptions.code ?? 'INSUFFICIENT_CREDITS',
|
|
429
|
+
retryable: false,
|
|
430
|
+
},
|
|
244
431
|
);
|
|
245
432
|
}
|
|
246
433
|
const hardBillingPayload = isHardBillingFailurePayload(billing)
|
|
@@ -249,7 +436,8 @@ export function normalizeToolHttpErrorMessage(input: {
|
|
|
249
436
|
? normalizeHardBillingPayload(parsed)
|
|
250
437
|
: null;
|
|
251
438
|
if (hardBillingPayload) {
|
|
252
|
-
return
|
|
439
|
+
return createToolHttpError(
|
|
440
|
+
schemaVersion,
|
|
253
441
|
formatHardBillingFailureMessage({
|
|
254
442
|
billing: hardBillingPayload,
|
|
255
443
|
toolId: input.toolId,
|
|
@@ -259,9 +447,17 @@ export function normalizeToolHttpErrorMessage(input: {
|
|
|
259
447
|
}),
|
|
260
448
|
hardBillingPayload,
|
|
261
449
|
input.status,
|
|
450
|
+
'terminal',
|
|
451
|
+
{
|
|
452
|
+
...publicOptions,
|
|
453
|
+
origin: 'deepline',
|
|
454
|
+
category: 'billing',
|
|
455
|
+
retryable: false,
|
|
456
|
+
},
|
|
262
457
|
);
|
|
263
458
|
}
|
|
264
|
-
return
|
|
459
|
+
return createToolHttpError(
|
|
460
|
+
schemaVersion,
|
|
265
461
|
`tool ${input.toolId} ${input.status} attempt ${input.attempt}/${input.maxAttempts}: ${formatPublicToolErrorPayload(
|
|
266
462
|
{
|
|
267
463
|
parsed,
|
|
@@ -271,6 +467,7 @@ export function normalizeToolHttpErrorMessage(input: {
|
|
|
271
467
|
billing,
|
|
272
468
|
input.status,
|
|
273
469
|
receiptFailureKindForToolErrorPayload(parsed),
|
|
470
|
+
publicOptions,
|
|
274
471
|
);
|
|
275
472
|
}
|
|
276
473
|
|
|
@@ -281,10 +478,18 @@ export function extractErrorBilling(
|
|
|
281
478
|
}
|
|
282
479
|
|
|
283
480
|
export function isHardBillingToolHttpError(error: unknown): boolean {
|
|
284
|
-
|
|
481
|
+
if (
|
|
285
482
|
error instanceof ToolHttpError &&
|
|
286
483
|
(isInsufficientCreditsBilling(error.billing) ||
|
|
287
484
|
isHardBillingFailurePayload(error.billing))
|
|
485
|
+
) {
|
|
486
|
+
return true;
|
|
487
|
+
}
|
|
488
|
+
return (
|
|
489
|
+
error instanceof ToolExecutionError &&
|
|
490
|
+
error.origin === 'deepline' &&
|
|
491
|
+
error.category === 'billing' &&
|
|
492
|
+
error.code !== 'BILLING_UNAVAILABLE'
|
|
288
493
|
);
|
|
289
494
|
}
|
|
290
495
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { isReusableWorkReceiptState } from './work-receipt-state-machine';
|
|
2
|
+
import type { ToolExecutionFailureV1 } from '../tool-execution-error';
|
|
2
3
|
|
|
3
4
|
export type WorkReceiptFailureKind = 'terminal' | 'repairable';
|
|
4
5
|
|
|
@@ -37,6 +38,7 @@ export type WorkReceipt = {
|
|
|
37
38
|
status: WorkReceiptStatus;
|
|
38
39
|
output?: unknown;
|
|
39
40
|
error?: string | null;
|
|
41
|
+
errorPayload?: ToolExecutionFailureV1 | null;
|
|
40
42
|
failureKind?: WorkReceiptFailureKind | null;
|
|
41
43
|
runId?: string | null;
|
|
42
44
|
leaseId?: string | null;
|
|
@@ -89,6 +91,7 @@ export type WorkReceiptBatchFailCommand = {
|
|
|
89
91
|
runId: string;
|
|
90
92
|
key: string;
|
|
91
93
|
error: string;
|
|
94
|
+
errorPayload?: ToolExecutionFailureV1 | null;
|
|
92
95
|
failureKind?: WorkReceiptFailureKind | null;
|
|
93
96
|
leaseId?: string | null;
|
|
94
97
|
runAttempt?: number | null;
|
|
@@ -161,6 +164,7 @@ export type WorkReceiptStore = {
|
|
|
161
164
|
failReceipt(
|
|
162
165
|
input: WorkReceiptCommand & {
|
|
163
166
|
error: string;
|
|
167
|
+
errorPayload?: ToolExecutionFailureV1 | null;
|
|
164
168
|
failureKind?: WorkReceiptFailureKind | null;
|
|
165
169
|
},
|
|
166
170
|
): Promise<WorkReceipt | null>;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Immutable Play artifact ABI version shared by bundling and SDK release
|
|
3
|
+
* policy. Keep this module dependency-free: shared Play code must not import
|
|
4
|
+
* the SDK release manifest or create a package cycle.
|
|
5
|
+
*/
|
|
6
|
+
export const CURRENT_PLAY_ARTIFACT_CONTRACT_VERSION = 2 as const;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { ToolExecutionErrorSchemaVersion } from '../tool-execution-error';
|
|
2
|
+
|
|
1
3
|
export type PlayPackageImport = {
|
|
2
4
|
name: string;
|
|
3
5
|
version: string | null;
|
|
@@ -21,6 +23,8 @@ export type PlayArtifactCompatibility = {
|
|
|
21
23
|
minRunnerVersion: number;
|
|
22
24
|
runtimeFeatures: PlayRuntimeFeature[];
|
|
23
25
|
runtimeBackend?: string | null;
|
|
26
|
+
/** Missing preserves the legacy schema used before this field existed. */
|
|
27
|
+
toolErrorSchemaVersion?: ToolExecutionErrorSchemaVersion;
|
|
24
28
|
};
|
|
25
29
|
|
|
26
30
|
/** The only executable Play artifact contract. */
|