deepline 0.1.149 → 0.1.151
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +157 -140
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/csv-rows.ts +2 -19
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/row-isolation.ts +5 -53
- package/dist/bundling-sources/sdk/src/client.ts +5 -0
- package/dist/bundling-sources/sdk/src/config.ts +2 -2
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/tool-output.ts +63 -17
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +100 -158
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +54 -0
- package/dist/bundling-sources/shared_libs/play-runtime/map-row-outcome.ts +167 -0
- package/dist/bundling-sources/shared_libs/play-runtime/pacing.ts +79 -0
- package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +39 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +19 -86
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-transition.ts +90 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-session.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +142 -11
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result-types.ts +17 -4
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +343 -26
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +20 -23
- package/dist/cli/index.js +186 -105
- package/dist/cli/index.mjs +193 -106
- package/dist/index.d.mts +12 -9
- package/dist/index.d.ts +12 -9
- package/dist/index.js +33 -20
- package/dist/index.mjs +40 -21
- package/dist/plays/bundle-play-file.mjs +22 -19
- package/package.json +1 -1
|
@@ -1,53 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Thrown by `assertNotAborted` and surfaced through ctx.step / ctx.sleep / map
|
|
8
|
-
* processing when the workflow has been terminated externally. Cooperatively
|
|
9
|
-
* cancels in-flight user code: the play must check `ctx.signal.aborted` (or
|
|
10
|
-
* await one of the abort-aware ctx methods) before doing more work.
|
|
11
|
-
*/
|
|
12
|
-
export class WorkflowAbortError extends Error {
|
|
13
|
-
override readonly name = 'WorkflowAbort';
|
|
14
|
-
constructor(message = 'Play run cancelled.') {
|
|
15
|
-
super(message);
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export function isAbortLikeError(error: unknown): boolean {
|
|
20
|
-
if (!error) return false;
|
|
21
|
-
if (error instanceof WorkflowAbortError) return true;
|
|
22
|
-
if (error instanceof Error) {
|
|
23
|
-
if (error.name === 'WorkflowAbort' || error.name === 'AbortError')
|
|
24
|
-
return true;
|
|
25
|
-
return /\b(cancell?ed|aborted|terminate[d]?)\b/i.test(error.message);
|
|
26
|
-
}
|
|
27
|
-
return false;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Errors that must stay run-fatal even under the default map row failure
|
|
32
|
-
* isolation:
|
|
33
|
-
*
|
|
34
|
-
* - Cancellation/abort must stop the run.
|
|
35
|
-
* - Governor budget exhaustion is a run-level invariant — isolating it per
|
|
36
|
-
* row would silently convert "this run exceeded its execution budget" into
|
|
37
|
-
* thousands of identical row failures.
|
|
38
|
-
* - Rate-limit pushback (a tool call that still got HTTP 429 after the
|
|
39
|
-
* in-process retry budget) is run-level throughput pressure that applies to
|
|
40
|
-
* every row equally, not a row defect. Isolating it silently drops healthy
|
|
41
|
-
* rows from the output dataset whenever a provider throttles — the durable
|
|
42
|
-
* chunk step's retries (and, if the storm persists, a loud run failure with
|
|
43
|
-
* recoverable persisted rows) are the correct response.
|
|
44
|
-
* - Hard billing failures (billing cap / insufficient credits) promise "run
|
|
45
|
-
* halted before marking remaining rows processed"; isolating them would
|
|
46
|
-
* complete the run while silently failing every remaining row.
|
|
47
|
-
*/
|
|
48
|
-
export function isRowIsolationExemptError(error: unknown): boolean {
|
|
49
|
-
if (isAbortLikeError(error)) return true;
|
|
50
|
-
if (error instanceof Error && error.name === 'GovernorBudgetError')
|
|
51
|
-
return true;
|
|
52
|
-
return isRateLimitToolHttpError(error) || isHardBillingToolHttpError(error);
|
|
53
|
-
}
|
|
1
|
+
export {
|
|
2
|
+
WorkflowAbortError,
|
|
3
|
+
isAbortLikeError,
|
|
4
|
+
isRowIsolationExemptError,
|
|
5
|
+
} from '../../../../shared_libs/play-runtime/row-isolation';
|
|
@@ -84,6 +84,7 @@ import type { EnrichCompiledConfig } from './cli/enrich-play-compiler.js';
|
|
|
84
84
|
const TERMINAL_PLAY_STATUSES = new Set(['completed', 'failed', 'cancelled']);
|
|
85
85
|
const INCLUDE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
|
|
86
86
|
const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
|
|
87
|
+
const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
|
|
87
88
|
const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-response';
|
|
88
89
|
const COMPILE_MANIFEST_RETRY_DELAYS_MS = [250, 1_000];
|
|
89
90
|
const REGISTER_PLAY_ARTIFACTS_COMPILE_CONCURRENCY = 3;
|
|
@@ -163,6 +164,7 @@ function chunkRegisterPlayArtifacts<T>(artifacts: T[]): T[][] {
|
|
|
163
164
|
|
|
164
165
|
type ExecuteToolRawOptions = {
|
|
165
166
|
includeToolMetadata?: boolean;
|
|
167
|
+
responseIntent?: 'raw' | 'row_artifact';
|
|
166
168
|
};
|
|
167
169
|
|
|
168
170
|
/**
|
|
@@ -1156,6 +1158,9 @@ export class DeeplineClient {
|
|
|
1156
1158
|
...(options?.includeToolMetadata
|
|
1157
1159
|
? { [INCLUDE_TOOL_METADATA_HEADER]: 'true' }
|
|
1158
1160
|
: {}),
|
|
1161
|
+
...(options?.responseIntent
|
|
1162
|
+
? { [EXECUTE_RESPONSE_INTENT_HEADER]: options.responseIntent }
|
|
1163
|
+
: {}),
|
|
1159
1164
|
};
|
|
1160
1165
|
return this.http.post<ToolExecution<TData, TMeta>>(
|
|
1161
1166
|
`/api/v2/integrations/${encodeURIComponent(toolId)}/execute`,
|
|
@@ -321,8 +321,8 @@ function loadProjectDeeplineEnv(startDir = process.cwd()): EnvValues {
|
|
|
321
321
|
return loadProjectEnvCandidates(startDir)[0]?.env ?? {};
|
|
322
322
|
}
|
|
323
323
|
|
|
324
|
-
function normalizeBaseUrl(baseUrl: string): string {
|
|
325
|
-
const trimmed = baseUrl
|
|
324
|
+
function normalizeBaseUrl(baseUrl: string | null | undefined): string {
|
|
325
|
+
const trimmed = baseUrl?.trim().replace(/\/+$/, '') ?? '';
|
|
326
326
|
if (!trimmed) return '';
|
|
327
327
|
try {
|
|
328
328
|
const parsed = new URL(trimmed);
|
|
@@ -102,10 +102,10 @@ export const SDK_RELEASE = {
|
|
|
102
102
|
// the SDK enrich generator's one-second stale policy.
|
|
103
103
|
// 0.1.110 ships authored V2 prebuilts and required top-level play descriptions.
|
|
104
104
|
// 0.1.111 ships dataset-native tool list getters and result row datasets.
|
|
105
|
-
version: '0.1.
|
|
105
|
+
version: '0.1.151',
|
|
106
106
|
apiContract: '2026-06-dataset-handle-results-hard-cutover',
|
|
107
107
|
supportPolicy: {
|
|
108
|
-
latest: '0.1.
|
|
108
|
+
latest: '0.1.151',
|
|
109
109
|
minimumSupported: '0.1.53',
|
|
110
110
|
deprecatedBelow: '0.1.53',
|
|
111
111
|
commandMinimumSupported: [
|
|
@@ -23,7 +23,13 @@
|
|
|
23
23
|
*
|
|
24
24
|
* @module
|
|
25
25
|
*/
|
|
26
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
closeSync,
|
|
28
|
+
mkdirSync,
|
|
29
|
+
openSync,
|
|
30
|
+
writeFileSync,
|
|
31
|
+
writeSync,
|
|
32
|
+
} from 'node:fs';
|
|
27
33
|
import { homedir } from 'node:os';
|
|
28
34
|
import { dirname, join } from 'node:path';
|
|
29
35
|
|
|
@@ -54,6 +60,22 @@ export type ListConversionResult = {
|
|
|
54
60
|
sourcePath: string | null;
|
|
55
61
|
};
|
|
56
62
|
|
|
63
|
+
export type RowOutputProjection = {
|
|
64
|
+
rows: Array<Record<string, unknown>>;
|
|
65
|
+
rowCount: number;
|
|
66
|
+
columns: string[];
|
|
67
|
+
previewRows: Array<Record<string, unknown>>;
|
|
68
|
+
strategy: ListConversionResult['strategy'];
|
|
69
|
+
sourcePath: string | null;
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
export type CsvOutputArtifact = {
|
|
73
|
+
path: string;
|
|
74
|
+
rowCount: number;
|
|
75
|
+
columns: string[];
|
|
76
|
+
preview: string;
|
|
77
|
+
};
|
|
78
|
+
|
|
57
79
|
type Scalar = string | number | boolean | null;
|
|
58
80
|
|
|
59
81
|
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
|
@@ -109,6 +131,20 @@ function normalizeRows(value: unknown): Array<Record<string, unknown>> | null {
|
|
|
109
131
|
});
|
|
110
132
|
}
|
|
111
133
|
|
|
134
|
+
function columnsForRows(rows: readonly Record<string, unknown>[]): string[] {
|
|
135
|
+
const seen = new Set<string>();
|
|
136
|
+
const columns: string[] = [];
|
|
137
|
+
for (const row of rows) {
|
|
138
|
+
for (const key of Object.keys(row)) {
|
|
139
|
+
if (!seen.has(key)) {
|
|
140
|
+
seen.add(key);
|
|
141
|
+
columns.push(key);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return columns;
|
|
146
|
+
}
|
|
147
|
+
|
|
112
148
|
/**
|
|
113
149
|
* Generate candidate root objects to search for lists.
|
|
114
150
|
* Tries: raw payload → V2 toolResponse.raw → legacy payload.output.body → legacy payload.result → legacy payload.result.data.
|
|
@@ -271,6 +307,19 @@ export function tryConvertToList(
|
|
|
271
307
|
return null;
|
|
272
308
|
}
|
|
273
309
|
|
|
310
|
+
export function projectRowOutput(
|
|
311
|
+
conversion: ListConversionResult,
|
|
312
|
+
): RowOutputProjection {
|
|
313
|
+
return {
|
|
314
|
+
rows: conversion.rows,
|
|
315
|
+
rowCount: conversion.rows.length,
|
|
316
|
+
columns: columnsForRows(conversion.rows),
|
|
317
|
+
previewRows: conversion.rows.slice(0, 5),
|
|
318
|
+
strategy: conversion.strategy,
|
|
319
|
+
sourcePath: conversion.sourcePath,
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
|
|
274
323
|
/** Ensure the shared output directory exists. Returns its path. */
|
|
275
324
|
function ensureOutputDir(): string {
|
|
276
325
|
const outputDir = join(homedir(), '.local', 'share', 'deepline', 'data');
|
|
@@ -330,21 +379,12 @@ export function writeCsvOutputFile(
|
|
|
330
379
|
rows: Array<Record<string, unknown>>,
|
|
331
380
|
stem: string,
|
|
332
381
|
options?: { outPath?: string },
|
|
333
|
-
):
|
|
382
|
+
): CsvOutputArtifact {
|
|
334
383
|
const outputPath = options?.outPath
|
|
335
384
|
? options.outPath
|
|
336
385
|
: join(ensureOutputDir(), `${stem}_${Date.now()}.csv`);
|
|
337
386
|
mkdirSync(dirname(outputPath), { recursive: true });
|
|
338
|
-
const
|
|
339
|
-
const columns: string[] = [];
|
|
340
|
-
for (const row of rows) {
|
|
341
|
-
for (const key of Object.keys(row)) {
|
|
342
|
-
if (!seen.has(key)) {
|
|
343
|
-
seen.add(key);
|
|
344
|
-
columns.push(key);
|
|
345
|
-
}
|
|
346
|
-
}
|
|
347
|
-
}
|
|
387
|
+
const columns = columnsForRows(rows);
|
|
348
388
|
|
|
349
389
|
const escapeCell = (value: unknown): string => {
|
|
350
390
|
const normalized =
|
|
@@ -361,12 +401,18 @@ export function writeCsvOutputFile(
|
|
|
361
401
|
return normalized;
|
|
362
402
|
};
|
|
363
403
|
|
|
364
|
-
const
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
404
|
+
const fd = openSync(outputPath, 'w');
|
|
405
|
+
try {
|
|
406
|
+
writeSync(fd, `${columns.map(escapeCell).join(',')}\n`);
|
|
407
|
+
for (const row of rows) {
|
|
408
|
+
writeSync(
|
|
409
|
+
fd,
|
|
410
|
+
`${columns.map((column) => escapeCell(row[column])).join(',')}\n`,
|
|
411
|
+
);
|
|
412
|
+
}
|
|
413
|
+
} finally {
|
|
414
|
+
closeSync(fd);
|
|
368
415
|
}
|
|
369
|
-
writeFileSync(outputPath, `${lines.join('\n')}\n`, 'utf-8');
|
|
370
416
|
|
|
371
417
|
const previewRows = rows.slice(0, 5);
|
|
372
418
|
const previewColumns = columns.slice(0, 5);
|
|
@@ -28,6 +28,12 @@ import {
|
|
|
28
28
|
} from './batch-runtime';
|
|
29
29
|
import type { PlayQueueHint } from './governor/rate-state-backend';
|
|
30
30
|
import type { MapRowOutcome } from './durability-store';
|
|
31
|
+
import {
|
|
32
|
+
completedMapRowOutcome,
|
|
33
|
+
failedMapRowOutcome,
|
|
34
|
+
mapRowOutcomeRuntimeFields,
|
|
35
|
+
resolveMapRowOutcomeKey,
|
|
36
|
+
} from './map-row-outcome';
|
|
31
37
|
import {
|
|
32
38
|
createDefaultGovernanceSnapshot,
|
|
33
39
|
createPlayExecutionGovernor,
|
|
@@ -35,12 +41,9 @@ import {
|
|
|
35
41
|
type PacingResolver,
|
|
36
42
|
type PlayExecutionGovernor,
|
|
37
43
|
} from './governor/governor';
|
|
38
|
-
import {
|
|
39
|
-
CTX_FETCH_EGRESS_TOOL_ID,
|
|
40
|
-
resolveBuiltinPacing,
|
|
41
|
-
} from './builtin-pacing';
|
|
44
|
+
import { CTX_FETCH_EGRESS_TOOL_ID } from './builtin-pacing';
|
|
42
45
|
import { InMemoryRateStateBackend } from './governor/in-memory-rate-state-backend';
|
|
43
|
-
import
|
|
46
|
+
import { pacingPolicyForTool } from './pacing';
|
|
44
47
|
import {
|
|
45
48
|
cloneToolExecuteResultWithExecution,
|
|
46
49
|
createToolExecuteResult,
|
|
@@ -54,13 +57,11 @@ import {
|
|
|
54
57
|
type ToolResultMetadataInput,
|
|
55
58
|
} from './tool-result';
|
|
56
59
|
import {
|
|
57
|
-
|
|
58
|
-
|
|
60
|
+
TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS,
|
|
61
|
+
classifyToolExecuteHttpFailure,
|
|
62
|
+
createToolExecuteHttpFailureAttemptTracker,
|
|
59
63
|
} from './tool-execute-retry-policy';
|
|
60
|
-
import {
|
|
61
|
-
isHardBillingToolHttpError,
|
|
62
|
-
normalizeToolHttpErrorMessage,
|
|
63
|
-
} from './tool-http-errors';
|
|
64
|
+
import { isRowIsolationExemptError } from './row-isolation';
|
|
64
65
|
import { sqlSafePlayColumnName } from '@shared_libs/plays/static-pipeline';
|
|
65
66
|
import { createRuntimeDatasetId } from './dataset-id';
|
|
66
67
|
import { dedupeExplicitMapKeyRows } from './map-row-identity';
|
|
@@ -577,24 +578,6 @@ function normalizeFetchHeaders(
|
|
|
577
578
|
);
|
|
578
579
|
}
|
|
579
580
|
|
|
580
|
-
function parseRetryAfterMs(header: string | null): number {
|
|
581
|
-
if (!header) {
|
|
582
|
-
return TOOL_RETRY_AFTER_FALLBACK_MS;
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
const seconds = Number(header);
|
|
586
|
-
if (Number.isFinite(seconds) && seconds > 0) {
|
|
587
|
-
return Math.ceil(seconds * 1000);
|
|
588
|
-
}
|
|
589
|
-
|
|
590
|
-
const retryAt = Date.parse(header);
|
|
591
|
-
if (Number.isFinite(retryAt)) {
|
|
592
|
-
return Math.max(1, retryAt - Date.now());
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
return TOOL_RETRY_AFTER_FALLBACK_MS;
|
|
596
|
-
}
|
|
597
|
-
|
|
598
581
|
function parseJsonOrNull(bodyText: string): unknown | null {
|
|
599
582
|
if (!bodyText.trim()) return null;
|
|
600
583
|
try {
|
|
@@ -652,20 +635,10 @@ function createPacingResolver(
|
|
|
652
635
|
getToolQueueHints: ContextOptions['getToolQueueHints'],
|
|
653
636
|
): PacingResolver {
|
|
654
637
|
return async (toolId: string) => {
|
|
655
|
-
const builtin =
|
|
638
|
+
const builtin = pacingPolicyForTool(toolId, []);
|
|
656
639
|
if (builtin) return builtin;
|
|
657
640
|
const hints = getToolQueueHints ? await getToolQueueHints(toolId) : [];
|
|
658
|
-
|
|
659
|
-
return null;
|
|
660
|
-
}
|
|
661
|
-
const provider = hints[0]!.provider;
|
|
662
|
-
const rules: PacingRule[] = hints.map((hint) => ({
|
|
663
|
-
ruleId: hint.ruleId,
|
|
664
|
-
requestsPerWindow: hint.requestsPerWindow,
|
|
665
|
-
windowMs: hint.windowMs,
|
|
666
|
-
maxConcurrency: hint.maxConcurrency,
|
|
667
|
-
}));
|
|
668
|
-
return { provider, rules };
|
|
641
|
+
return pacingPolicyForTool(toolId, hints);
|
|
669
642
|
};
|
|
670
643
|
}
|
|
671
644
|
|
|
@@ -2030,9 +2003,8 @@ export class PlayContextImpl {
|
|
|
2030
2003
|
}
|
|
2031
2004
|
|
|
2032
2005
|
const rowIdentity = (row: Record<string, unknown>, index = 0) => {
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
}
|
|
2006
|
+
const runtimeKey = resolveMapRowOutcomeKey(row);
|
|
2007
|
+
if (runtimeKey) return runtimeKey;
|
|
2036
2008
|
return explicitKeyResolver
|
|
2037
2009
|
? derivePlayRowIdentityFromKey(
|
|
2038
2010
|
explicitKeyResolver(row, index),
|
|
@@ -2081,7 +2053,9 @@ export class PlayContextImpl {
|
|
|
2081
2053
|
const mapStartRows = shouldPassRowKey
|
|
2082
2054
|
? rawItems.map((row, index) => ({
|
|
2083
2055
|
...toSerializableCsvAliasedRow(row),
|
|
2084
|
-
|
|
2056
|
+
...mapRowOutcomeRuntimeFields({
|
|
2057
|
+
key: rowIdentity(row, index),
|
|
2058
|
+
}),
|
|
2085
2059
|
}))
|
|
2086
2060
|
: rawItems.map((row) => toSerializableCsvAliasedRow(row));
|
|
2087
2061
|
const mapStartResult = await this.#options.onMapStart(
|
|
@@ -2099,9 +2073,7 @@ export class PlayContextImpl {
|
|
|
2099
2073
|
mapStartResult.tableNamespace,
|
|
2100
2074
|
);
|
|
2101
2075
|
const persistedRowIdentity = (row: Record<string, unknown>, index = 0) =>
|
|
2102
|
-
|
|
2103
|
-
? row.__deeplineRowKey
|
|
2104
|
-
: rowIdentity(row, index);
|
|
2076
|
+
resolveMapRowOutcomeKey(row) ?? rowIdentity(row, index);
|
|
2105
2077
|
const pendingKeys = new Set(
|
|
2106
2078
|
mapStartResult.pendingRows.map((row, index) =>
|
|
2107
2079
|
persistedRowIdentity(row, index),
|
|
@@ -2336,14 +2308,16 @@ export class PlayContextImpl {
|
|
|
2336
2308
|
if (incrementalPersistence?.isPersisted(row)) continue;
|
|
2337
2309
|
const rowKey = row.key;
|
|
2338
2310
|
const meta = mapCellMeta?.get(rowKey);
|
|
2339
|
-
persistRows.push(
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2311
|
+
persistRows.push(
|
|
2312
|
+
completedMapRowOutcome({
|
|
2313
|
+
key: rowKey,
|
|
2314
|
+
data: row.data,
|
|
2315
|
+
cellMetaPatch: {
|
|
2316
|
+
...(row.cellMetaPatch ?? {}),
|
|
2317
|
+
...(meta ?? {}),
|
|
2318
|
+
},
|
|
2319
|
+
}),
|
|
2320
|
+
);
|
|
2347
2321
|
}
|
|
2348
2322
|
persistRows.push(
|
|
2349
2323
|
...mapResult.failedRows.filter(
|
|
@@ -2441,20 +2415,19 @@ export class PlayContextImpl {
|
|
|
2441
2415
|
.filter((fieldName) => shouldPersistMapCellField(fieldName));
|
|
2442
2416
|
const normalizedTableNamespace = mapScope.artifactTableNamespace;
|
|
2443
2417
|
const rowIdentity = (row: Record<string, unknown>, index = 0) =>
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
!fieldName.startsWith('__deepline'),
|
|
2453
|
-
),
|
|
2454
|
-
),
|
|
2418
|
+
resolveMapRowOutcomeKey(row) ??
|
|
2419
|
+
mapScope.rowIdentity(
|
|
2420
|
+
stripCsvProjectedFields(
|
|
2421
|
+
Object.fromEntries(
|
|
2422
|
+
Object.entries(row).filter(
|
|
2423
|
+
([fieldName]) =>
|
|
2424
|
+
!datasetColumnNames.includes(fieldName) &&
|
|
2425
|
+
!fieldName.startsWith('__deepline'),
|
|
2455
2426
|
),
|
|
2456
|
-
|
|
2457
|
-
|
|
2427
|
+
),
|
|
2428
|
+
),
|
|
2429
|
+
index,
|
|
2430
|
+
);
|
|
2458
2431
|
const executionRowKey = (row: Record<string, unknown>, index: number) =>
|
|
2459
2432
|
runtimeOptions?.executionRowKeys?.[index] ?? rowIdentity(row, index);
|
|
2460
2433
|
const executionRowIndex = (index: number) =>
|
|
@@ -2645,14 +2618,16 @@ export class PlayContextImpl {
|
|
|
2645
2618
|
`Map completed: ${results.length + completedRows} results (${results.length} executed, ${completedRows} already satisfied)`,
|
|
2646
2619
|
);
|
|
2647
2620
|
return {
|
|
2648
|
-
completedRows: results.map((row, index) =>
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2621
|
+
completedRows: results.map((row, index) =>
|
|
2622
|
+
completedMapRowOutcome({
|
|
2623
|
+
key: executionRowKey(
|
|
2624
|
+
this.toOutputRow(items[index] as Record<string, unknown>),
|
|
2625
|
+
index,
|
|
2626
|
+
),
|
|
2627
|
+
inputIndex: executionRowIndex(index),
|
|
2628
|
+
data: this.toPersistedOutputRow(row),
|
|
2629
|
+
}),
|
|
2630
|
+
),
|
|
2656
2631
|
failedRows: [],
|
|
2657
2632
|
};
|
|
2658
2633
|
}
|
|
@@ -2767,6 +2742,9 @@ export class PlayContextImpl {
|
|
|
2767
2742
|
) {
|
|
2768
2743
|
throw error;
|
|
2769
2744
|
}
|
|
2745
|
+
if (isRowIsolationExemptError(error)) {
|
|
2746
|
+
throw error;
|
|
2747
|
+
}
|
|
2770
2748
|
|
|
2771
2749
|
value = null;
|
|
2772
2750
|
computedFields[fieldName] = value;
|
|
@@ -2787,7 +2765,7 @@ export class PlayContextImpl {
|
|
|
2787
2765
|
: {},
|
|
2788
2766
|
});
|
|
2789
2767
|
if (failFastOnRowError) {
|
|
2790
|
-
const failedRow: PersistableMapRow = {
|
|
2768
|
+
const failedRow: PersistableMapRow = failedMapRowOutcome({
|
|
2791
2769
|
key: rowKey,
|
|
2792
2770
|
inputIndex: rowIndex,
|
|
2793
2771
|
data: this.toPersistedOutputRow(
|
|
@@ -2796,9 +2774,8 @@ export class PlayContextImpl {
|
|
|
2796
2774
|
...(this.activeMapCellMeta?.get(rowKey)
|
|
2797
2775
|
? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) }
|
|
2798
2776
|
: {}),
|
|
2799
|
-
status: 'failed',
|
|
2800
2777
|
error: formattedError,
|
|
2801
|
-
};
|
|
2778
|
+
});
|
|
2802
2779
|
failedRowsToPersist.push(failedRow);
|
|
2803
2780
|
throw error;
|
|
2804
2781
|
}
|
|
@@ -2806,14 +2783,13 @@ export class PlayContextImpl {
|
|
|
2806
2783
|
cloneCsvAliasedRow(baseRow, computedFields),
|
|
2807
2784
|
);
|
|
2808
2785
|
const cellMetaPatch = this.activeMapCellMeta?.get(rowKey);
|
|
2809
|
-
const failedRow: PersistableMapRow = {
|
|
2786
|
+
const failedRow: PersistableMapRow = failedMapRowOutcome({
|
|
2810
2787
|
key: rowKey,
|
|
2811
2788
|
inputIndex: rowIndex,
|
|
2812
2789
|
data: failedData,
|
|
2813
2790
|
...(cellMetaPatch ? { cellMetaPatch } : {}),
|
|
2814
|
-
status: 'failed',
|
|
2815
2791
|
error: formattedError,
|
|
2816
|
-
};
|
|
2792
|
+
});
|
|
2817
2793
|
failedRowsToPersist.push(failedRow);
|
|
2818
2794
|
updateMapFrameProgress({
|
|
2819
2795
|
failedRowKey: rowKey,
|
|
@@ -2867,14 +2843,14 @@ export class PlayContextImpl {
|
|
|
2867
2843
|
dataPatch: {},
|
|
2868
2844
|
});
|
|
2869
2845
|
const publicRow = this.toPublicOutputRow(merged);
|
|
2870
|
-
const completedRow: PersistableMapRow = {
|
|
2846
|
+
const completedRow: PersistableMapRow = completedMapRowOutcome({
|
|
2871
2847
|
key: rowKey,
|
|
2872
2848
|
inputIndex: rowIndex,
|
|
2873
2849
|
data: this.toPersistedOutputRow(merged),
|
|
2874
2850
|
...(this.activeMapCellMeta?.get(rowKey)
|
|
2875
2851
|
? { cellMetaPatch: this.activeMapCellMeta.get(rowKey) }
|
|
2876
2852
|
: {}),
|
|
2877
|
-
};
|
|
2853
|
+
});
|
|
2878
2854
|
completedRowsToPersist.push(completedRow);
|
|
2879
2855
|
enqueueIncrementalPersist(completedRow);
|
|
2880
2856
|
return publicRow;
|
|
@@ -5538,7 +5514,13 @@ export class PlayContextImpl {
|
|
|
5538
5514
|
},
|
|
5539
5515
|
},
|
|
5540
5516
|
async (span) => {
|
|
5541
|
-
|
|
5517
|
+
const httpFailureAttempts =
|
|
5518
|
+
createToolExecuteHttpFailureAttemptTracker();
|
|
5519
|
+
const retryPolicy = await this.#options
|
|
5520
|
+
.getToolRetryPolicy?.(toolId)
|
|
5521
|
+
.catch(() => null);
|
|
5522
|
+
const retrySafeTransientHttp =
|
|
5523
|
+
retryPolicy?.retrySafeTransientHttp === true;
|
|
5542
5524
|
let transportAttempt = 0;
|
|
5543
5525
|
|
|
5544
5526
|
while (true) {
|
|
@@ -5568,7 +5550,7 @@ export class PlayContextImpl {
|
|
|
5568
5550
|
transportAttempt += 1;
|
|
5569
5551
|
const message =
|
|
5570
5552
|
error instanceof Error ? error.message : String(error);
|
|
5571
|
-
if (transportAttempt <
|
|
5553
|
+
if (transportAttempt < TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS) {
|
|
5572
5554
|
this.governor.chargeBudget('retry');
|
|
5573
5555
|
const retryAfterMs =
|
|
5574
5556
|
TOOL_RETRY_AFTER_FALLBACK_MS * transportAttempt;
|
|
@@ -5577,105 +5559,65 @@ export class PlayContextImpl {
|
|
|
5577
5559
|
transportAttempt,
|
|
5578
5560
|
);
|
|
5579
5561
|
this.log(
|
|
5580
|
-
`Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${
|
|
5562
|
+
`Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${message}`,
|
|
5581
5563
|
);
|
|
5582
5564
|
await this.sleepWithCheckpointHeartbeat(retryAfterMs);
|
|
5583
5565
|
continue;
|
|
5584
5566
|
}
|
|
5585
5567
|
throw new Error(
|
|
5586
|
-
`Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${
|
|
5568
|
+
`Tool ${toolId} transport failed calling ${url} after ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS} attempts: ${message}`,
|
|
5587
5569
|
);
|
|
5588
5570
|
}
|
|
5589
5571
|
|
|
5590
5572
|
span.setAttribute('plays.http_status_code', response.status);
|
|
5591
5573
|
|
|
5592
|
-
if (response.status === 429) {
|
|
5593
|
-
rateLimitAttempt += 1;
|
|
5594
|
-
const text = await response.text();
|
|
5595
|
-
const initialRetryDecision = decideToolExecuteHttpRetry({
|
|
5596
|
-
toolId,
|
|
5597
|
-
status: response.status,
|
|
5598
|
-
});
|
|
5599
|
-
const error = normalizeToolHttpErrorMessage({
|
|
5600
|
-
toolId,
|
|
5601
|
-
status: response.status,
|
|
5602
|
-
attempt: rateLimitAttempt,
|
|
5603
|
-
maxAttempts: initialRetryDecision.attemptCap,
|
|
5604
|
-
bodyText: text,
|
|
5605
|
-
});
|
|
5606
|
-
const retryDecision = decideToolExecuteHttpRetry({
|
|
5607
|
-
toolId,
|
|
5608
|
-
status: response.status,
|
|
5609
|
-
hardBillingFailure: isHardBillingToolHttpError(error),
|
|
5610
|
-
});
|
|
5611
|
-
if (
|
|
5612
|
-
!retryDecision.retryable ||
|
|
5613
|
-
rateLimitAttempt >= retryDecision.attemptCap
|
|
5614
|
-
) {
|
|
5615
|
-
throw error;
|
|
5616
|
-
}
|
|
5617
|
-
this.governor.chargeBudget('retry');
|
|
5618
|
-
const retryAfterMs = parseRetryAfterMs(
|
|
5619
|
-
response.headers.get('retry-after'),
|
|
5620
|
-
);
|
|
5621
|
-
// Feed the server-observed Retry-After back into the shared pacer
|
|
5622
|
-
// so subsequent acquires for this provider back off.
|
|
5623
|
-
await this.reportToolBackpressure(toolId, retryAfterMs);
|
|
5624
|
-
span.setAttribute(
|
|
5625
|
-
'plays.rate_limit_retry_after_ms',
|
|
5626
|
-
retryAfterMs,
|
|
5627
|
-
);
|
|
5628
|
-
span.setAttribute('plays.rate_limit_attempt', rateLimitAttempt);
|
|
5629
|
-
this.log(
|
|
5630
|
-
`Tool ${toolId} rate limited; retrying after ${retryAfterMs}ms`,
|
|
5631
|
-
);
|
|
5632
|
-
await this.sleepWithCheckpointHeartbeat(retryAfterMs);
|
|
5633
|
-
continue;
|
|
5634
|
-
}
|
|
5635
|
-
|
|
5636
5574
|
if (!response.ok) {
|
|
5637
5575
|
const text = await response.text();
|
|
5638
|
-
const
|
|
5576
|
+
const httpFailureAttempt = httpFailureAttempts.next({
|
|
5639
5577
|
toolId,
|
|
5640
5578
|
status: response.status,
|
|
5579
|
+
transientHttpRetrySafe: retrySafeTransientHttp,
|
|
5641
5580
|
});
|
|
5642
|
-
const
|
|
5581
|
+
const failure = classifyToolExecuteHttpFailure({
|
|
5643
5582
|
toolId,
|
|
5644
5583
|
status: response.status,
|
|
5645
|
-
attempt:
|
|
5646
|
-
maxAttempts: initialRetryDecision.attemptCap,
|
|
5584
|
+
attempt: httpFailureAttempt,
|
|
5647
5585
|
bodyText: text,
|
|
5586
|
+
retryAfterHeader: response.headers.get('retry-after'),
|
|
5587
|
+
transientHttpRetrySafe: retrySafeTransientHttp,
|
|
5648
5588
|
});
|
|
5649
|
-
|
|
5650
|
-
|
|
5651
|
-
|
|
5652
|
-
|
|
5653
|
-
|
|
5654
|
-
|
|
5655
|
-
|
|
5656
|
-
rateLimitAttempt + 1 < retryDecision.attemptCap
|
|
5657
|
-
) {
|
|
5658
|
-
rateLimitAttempt += 1;
|
|
5659
|
-
this.governor.chargeBudget('retry');
|
|
5660
|
-
const retryAfterMs = parseRetryAfterMs(
|
|
5661
|
-
response.headers.get('retry-after'),
|
|
5589
|
+
if (failure.backpressureDelayMs !== null) {
|
|
5590
|
+
// Feed the server-observed Retry-After back into the shared
|
|
5591
|
+
// pacer even on the final attempt so later provider calls back
|
|
5592
|
+
// off instead of retrying the whole map chunk.
|
|
5593
|
+
await this.reportToolBackpressure(
|
|
5594
|
+
toolId,
|
|
5595
|
+
failure.backpressureDelayMs,
|
|
5662
5596
|
);
|
|
5597
|
+
}
|
|
5598
|
+
if (failure.shouldRetry) {
|
|
5599
|
+
if (failure.chargeRetryBudget) {
|
|
5600
|
+
this.governor.chargeBudget('retry');
|
|
5601
|
+
}
|
|
5602
|
+
const retryAttributePrefix = failure.isRateLimit
|
|
5603
|
+
? 'rate_limit'
|
|
5604
|
+
: 'transient_http';
|
|
5663
5605
|
span.setAttribute(
|
|
5664
|
-
|
|
5665
|
-
|
|
5606
|
+
`plays.${retryAttributePrefix}_retry_after_ms`,
|
|
5607
|
+
failure.retryDelayMs,
|
|
5666
5608
|
);
|
|
5667
5609
|
span.setAttribute(
|
|
5668
|
-
|
|
5669
|
-
|
|
5610
|
+
`plays.${retryAttributePrefix}_attempt`,
|
|
5611
|
+
httpFailureAttempt,
|
|
5670
5612
|
);
|
|
5671
5613
|
this.log(
|
|
5672
|
-
`Tool ${toolId} returned ${response.status}; retrying after ${
|
|
5614
|
+
`Tool ${toolId} returned ${response.status}; retrying after ${failure.retryDelayMs}ms`,
|
|
5673
5615
|
);
|
|
5674
|
-
await this.sleepWithCheckpointHeartbeat(
|
|
5616
|
+
await this.sleepWithCheckpointHeartbeat(failure.retryDelayMs);
|
|
5675
5617
|
continue;
|
|
5676
5618
|
}
|
|
5677
|
-
this.log(error.message);
|
|
5678
|
-
throw error;
|
|
5619
|
+
this.log(failure.error.message);
|
|
5620
|
+
throw failure.error;
|
|
5679
5621
|
}
|
|
5680
5622
|
|
|
5681
5623
|
const data = (await response.json()) as Record<string, unknown>;
|
|
@@ -487,6 +487,9 @@ export interface ContextOptions {
|
|
|
487
487
|
runId?: string;
|
|
488
488
|
resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
|
|
489
489
|
getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
|
|
490
|
+
getToolRetryPolicy?: (toolId: string) => Promise<{
|
|
491
|
+
retrySafeTransientHttp?: boolean;
|
|
492
|
+
} | null>;
|
|
490
493
|
getToolTargetGetters?: (
|
|
491
494
|
toolId: string,
|
|
492
495
|
output: string,
|