deepline 0.1.163 → 0.1.164
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +13 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +302 -119
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/cli/index.js +144 -21
- package/dist/cli/index.mjs +144 -21
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/package.json +1 -1
|
@@ -3559,6 +3559,18 @@ function resolveSheetContractFromReq(
|
|
|
3559
3559
|
}
|
|
3560
3560
|
}
|
|
3561
3561
|
|
|
3562
|
+
function staticPipelineFromReq(req: RunRequest): PlayStaticPipeline | null {
|
|
3563
|
+
const snapshot = req.contractSnapshot as
|
|
3564
|
+
| { staticPipeline?: unknown }
|
|
3565
|
+
| undefined
|
|
3566
|
+
| null;
|
|
3567
|
+
const pipeline = snapshot?.staticPipeline;
|
|
3568
|
+
if (!pipeline || typeof pipeline !== 'object' || Array.isArray(pipeline)) {
|
|
3569
|
+
return null;
|
|
3570
|
+
}
|
|
3571
|
+
return pipeline as PlayStaticPipeline;
|
|
3572
|
+
}
|
|
3573
|
+
|
|
3562
3574
|
/**
|
|
3563
3575
|
* Direct-Neon writes from the shared harness Worker. Resolves the org runtime
|
|
3564
3576
|
* Postgres URL via `create_db_session` inside that long-lived Worker, then
|
|
@@ -4290,6 +4302,7 @@ function createMinimalWorkerCtx(
|
|
|
4290
4302
|
mapName: name,
|
|
4291
4303
|
rowCountHint,
|
|
4292
4304
|
executionPlan: plan,
|
|
4305
|
+
staticPipeline: staticPipelineFromReq(req),
|
|
4293
4306
|
});
|
|
4294
4307
|
const outputFields = fieldEntries.map(([field]) => field);
|
|
4295
4308
|
const updateMapProgress = (
|
|
@@ -1,83 +1,319 @@
|
|
|
1
1
|
import {
|
|
2
2
|
chooseMapChunkSize,
|
|
3
|
-
EXECUTION_PLAN_DEFAULTS,
|
|
4
3
|
type ExecutionPlan,
|
|
5
4
|
} from '../../../../shared_libs/play-runtime/execution-plan';
|
|
5
|
+
import { getPlayRuntimeBatchStrategy } from '../../../../shared_libs/play-runtime/play-runtime-batching-registry';
|
|
6
6
|
import { TOOL_CALLING_MAP_CHUNK_SIZE } from '../../../../shared_libs/play-runtime/map-chunk-limits';
|
|
7
|
+
import {
|
|
8
|
+
flattenStaticSubsteps,
|
|
9
|
+
getTopLevelPipelineSubsteps,
|
|
10
|
+
type PlayStaticColumnProducer,
|
|
11
|
+
type PlayStaticPipeline,
|
|
12
|
+
type PlayStaticSubstep,
|
|
13
|
+
} from '../../../../shared_libs/plays/static-pipeline';
|
|
7
14
|
|
|
8
15
|
export const CACHE_ENABLED_SIMPLE_MAP_CHUNK_SIZE = 10_000;
|
|
9
16
|
export { TOOL_CALLING_MAP_CHUNK_SIZE };
|
|
10
|
-
|
|
11
|
-
|
|
17
|
+
// Cloudflare preview Workers enforce the 50-subrequest invocation limit. Leave
|
|
18
|
+
// headroom for coordinator/storage calls around row-level unbatched tool RPCs.
|
|
19
|
+
export const UNBATCHED_TOOL_SUBREQUESTS_PER_CHUNK_BUDGET = 40;
|
|
20
|
+
// Fresh unbatched tool calls use one RUNTIME_API integration execute RPC and
|
|
21
|
+
// one HARNESS durable-receipt completion RPC. Batch-cap rows by both.
|
|
22
|
+
export const SUBREQUESTS_PER_UNBATCHED_TOOL_CALL = 2;
|
|
12
23
|
|
|
13
24
|
export type WorkerMapChunkPlanInput = {
|
|
14
25
|
mapName: string;
|
|
15
26
|
rowCountHint: number | null;
|
|
16
27
|
executionPlan?: ExecutionPlan | null;
|
|
28
|
+
staticPipeline?: PlayStaticPipeline | null;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
type PlanMap = ExecutionPlan['maps'][number];
|
|
32
|
+
type ChunkSizingMap = Pick<
|
|
33
|
+
PlanMap,
|
|
34
|
+
| 'mapName'
|
|
35
|
+
| 'tableNamespace'
|
|
36
|
+
| 'outputFields'
|
|
37
|
+
| 'defaultChunkSize'
|
|
38
|
+
| 'stepsPerChunk'
|
|
39
|
+
> &
|
|
40
|
+
Partial<Pick<PlanMap, 'externalStepFields'>>;
|
|
41
|
+
type ChunkSizingPlan = {
|
|
42
|
+
maps: ChunkSizingMap[];
|
|
43
|
+
toolDeclarations?: ExecutionPlan['toolDeclarations'];
|
|
44
|
+
softWorkflowStepBudget?: number | null;
|
|
17
45
|
};
|
|
18
46
|
|
|
47
|
+
type MapToolStats = [totalToolCount: number, unbatchedToolCount: number];
|
|
48
|
+
|
|
49
|
+
function declarationBelongsToMapOutput(
|
|
50
|
+
declaration: { field?: string | null },
|
|
51
|
+
outputFields: readonly string[],
|
|
52
|
+
externalStepFields: readonly string[] = [],
|
|
53
|
+
): boolean {
|
|
54
|
+
const fieldName = declaration.field;
|
|
55
|
+
if (fieldName == null) return true;
|
|
56
|
+
if (
|
|
57
|
+
outputFields.some(
|
|
58
|
+
(field) => fieldName === field || fieldName.startsWith(`${field}.`),
|
|
59
|
+
)
|
|
60
|
+
) {
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
63
|
+
return externalStepFields.some(
|
|
64
|
+
(field) => fieldName === field || fieldName.endsWith(`.${field}`),
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function isUnbatchedTool(toolId: string): boolean {
|
|
69
|
+
return getPlayRuntimeBatchStrategy(toolId) === null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function countToolSubsteps(
|
|
73
|
+
substeps: readonly PlayStaticSubstep[],
|
|
74
|
+
): MapToolStats {
|
|
75
|
+
let totalToolCount = 0;
|
|
76
|
+
let unbatchedToolCount = 0;
|
|
77
|
+
|
|
78
|
+
for (const substep of substeps) {
|
|
79
|
+
if (substep.type === 'tool') {
|
|
80
|
+
totalToolCount += 1;
|
|
81
|
+
if (isUnbatchedTool(substep.toolId)) {
|
|
82
|
+
unbatchedToolCount += 1;
|
|
83
|
+
}
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (substep.type === 'waterfall') {
|
|
88
|
+
for (const step of substep.steps ?? []) {
|
|
89
|
+
if (!step.toolId) continue;
|
|
90
|
+
totalToolCount += 1;
|
|
91
|
+
if (isUnbatchedTool(step.toolId)) {
|
|
92
|
+
unbatchedToolCount += 1;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
if (
|
|
99
|
+
substep.type === 'step_suite' ||
|
|
100
|
+
substep.type === 'control_flow' ||
|
|
101
|
+
substep.type === 'dataset'
|
|
102
|
+
) {
|
|
103
|
+
const [nestedTotal, nestedUnbatched] = countToolSubsteps(
|
|
104
|
+
substep.steps ?? [],
|
|
105
|
+
);
|
|
106
|
+
totalToolCount += nestedTotal;
|
|
107
|
+
unbatchedToolCount += nestedUnbatched;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return [totalToolCount, unbatchedToolCount];
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function countProducerTools(
|
|
115
|
+
producers: readonly PlayStaticColumnProducer[],
|
|
116
|
+
): MapToolStats {
|
|
117
|
+
let totalToolCount = 0;
|
|
118
|
+
let unbatchedToolCount = 0;
|
|
119
|
+
|
|
120
|
+
for (const producer of producers) {
|
|
121
|
+
if (producer.kind === 'tool' && producer.toolId) {
|
|
122
|
+
totalToolCount += 1;
|
|
123
|
+
if (isUnbatchedTool(producer.toolId)) {
|
|
124
|
+
unbatchedToolCount += 1;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
if (producer.substep.type === 'waterfall') {
|
|
128
|
+
for (const step of producer.substep.steps ?? []) {
|
|
129
|
+
if (!step.toolId) continue;
|
|
130
|
+
totalToolCount += 1;
|
|
131
|
+
if (isUnbatchedTool(step.toolId)) {
|
|
132
|
+
unbatchedToolCount += 1;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
if (producer.steps?.length) {
|
|
137
|
+
const [nestedTotal, nestedUnbatched] = countProducerTools(producer.steps);
|
|
138
|
+
totalToolCount += nestedTotal;
|
|
139
|
+
unbatchedToolCount += nestedUnbatched;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
return [totalToolCount, unbatchedToolCount];
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function datasetBelongsToPlanMap(
|
|
147
|
+
dataset: Extract<PlayStaticSubstep, { type: 'dataset' }>,
|
|
148
|
+
planMap: ChunkSizingMap,
|
|
149
|
+
mapName: string,
|
|
150
|
+
): boolean {
|
|
151
|
+
return [dataset.name, dataset.tableNamespace, dataset.field].some(
|
|
152
|
+
(datasetName) =>
|
|
153
|
+
typeof datasetName === 'string' &&
|
|
154
|
+
datasetName.length > 0 &&
|
|
155
|
+
(datasetName === planMap.mapName ||
|
|
156
|
+
datasetName === planMap.tableNamespace ||
|
|
157
|
+
datasetName === mapName),
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function countMapLocalTools(input: {
|
|
162
|
+
staticPipeline: PlayStaticPipeline | null | undefined;
|
|
163
|
+
planMap: ChunkSizingMap | null | undefined;
|
|
164
|
+
mapName: string;
|
|
165
|
+
}): MapToolStats | null {
|
|
166
|
+
if (!input.staticPipeline || !input.planMap) return null;
|
|
167
|
+
const planMap = input.planMap;
|
|
168
|
+
const dataset = flattenStaticSubsteps(
|
|
169
|
+
getTopLevelPipelineSubsteps(input.staticPipeline),
|
|
170
|
+
).find(
|
|
171
|
+
(substep): substep is Extract<PlayStaticSubstep, { type: 'dataset' }> =>
|
|
172
|
+
substep.type === 'dataset' &&
|
|
173
|
+
datasetBelongsToPlanMap(substep, planMap, input.mapName),
|
|
174
|
+
);
|
|
175
|
+
if (!dataset) return null;
|
|
176
|
+
|
|
177
|
+
if (dataset.steps?.length) {
|
|
178
|
+
return countToolSubsteps(dataset.steps);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (dataset.columns?.length) {
|
|
182
|
+
return countProducerTools(
|
|
183
|
+
dataset.columns.flatMap((column) => column.producers),
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
return [0, 0];
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function countFallbackDeclarationTools(input: {
|
|
191
|
+
declarations: readonly { toolId: string; field?: string | null }[];
|
|
192
|
+
outputFields: readonly string[];
|
|
193
|
+
externalStepFields?: readonly string[];
|
|
194
|
+
}): MapToolStats {
|
|
195
|
+
let totalToolCount = 0;
|
|
196
|
+
let unbatchedToolCount = 0;
|
|
197
|
+
|
|
198
|
+
for (const declaration of input.declarations) {
|
|
199
|
+
if (
|
|
200
|
+
!declarationBelongsToMapOutput(
|
|
201
|
+
declaration,
|
|
202
|
+
input.outputFields,
|
|
203
|
+
input.externalStepFields ?? [],
|
|
204
|
+
)
|
|
205
|
+
) {
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
totalToolCount += 1;
|
|
209
|
+
if (isUnbatchedTool(declaration.toolId)) {
|
|
210
|
+
unbatchedToolCount += 1;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
return [totalToolCount, unbatchedToolCount];
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function countDeclarationTools(
|
|
218
|
+
declarations: readonly { toolId: string; field?: string | null }[],
|
|
219
|
+
): MapToolStats {
|
|
220
|
+
return [
|
|
221
|
+
declarations.length,
|
|
222
|
+
declarations.filter((declaration) => isUnbatchedTool(declaration.toolId))
|
|
223
|
+
.length,
|
|
224
|
+
];
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
function countNestedExecutionSteps(
|
|
228
|
+
substeps: readonly PlayStaticSubstep[] | undefined,
|
|
229
|
+
): number {
|
|
230
|
+
let stepsPerChunk = 1;
|
|
231
|
+
for (const substep of substeps ?? []) {
|
|
232
|
+
if (substep.type === 'waterfall') {
|
|
233
|
+
stepsPerChunk = Math.max(stepsPerChunk, substep.steps?.length ?? 1);
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
if (substep.type === 'step_suite') {
|
|
237
|
+
stepsPerChunk = Math.max(stepsPerChunk, substep.steps.length);
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
if (substep.type === 'control_flow' || substep.type === 'dataset') {
|
|
241
|
+
stepsPerChunk = Math.max(
|
|
242
|
+
stepsPerChunk,
|
|
243
|
+
countNestedExecutionSteps(substep.steps),
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
return stepsPerChunk;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function buildStaticChunkSizingPlan(
|
|
251
|
+
staticPipeline: PlayStaticPipeline | null,
|
|
252
|
+
): ChunkSizingPlan | null {
|
|
253
|
+
if (!staticPipeline) return null;
|
|
254
|
+
const datasets = flattenStaticSubsteps(
|
|
255
|
+
getTopLevelPipelineSubsteps(staticPipeline),
|
|
256
|
+
).filter(
|
|
257
|
+
(substep): substep is Extract<PlayStaticSubstep, { type: 'dataset' }> =>
|
|
258
|
+
substep.type === 'dataset',
|
|
259
|
+
);
|
|
260
|
+
if (datasets.length === 0) return null;
|
|
261
|
+
return {
|
|
262
|
+
maps: datasets.map((dataset) => {
|
|
263
|
+
const stepsPerChunk = countNestedExecutionSteps(dataset.steps);
|
|
264
|
+
return {
|
|
265
|
+
mapName: dataset.name ?? dataset.field,
|
|
266
|
+
tableNamespace: dataset.tableNamespace ?? dataset.field,
|
|
267
|
+
outputFields: dataset.outputFields ?? [],
|
|
268
|
+
defaultChunkSize: stepsPerChunk > 1 ? 1_000 : 5_000,
|
|
269
|
+
stepsPerChunk,
|
|
270
|
+
};
|
|
271
|
+
}),
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
|
|
19
275
|
export function chooseWorkerMapRowsPerChunk(
|
|
20
276
|
input: WorkerMapChunkPlanInput,
|
|
21
277
|
): number {
|
|
22
|
-
const plan = input.executionPlan
|
|
278
|
+
const plan: ChunkSizingPlan | null = input.executionPlan
|
|
279
|
+
? {
|
|
280
|
+
maps: input.executionPlan.maps,
|
|
281
|
+
toolDeclarations: input.executionPlan.toolDeclarations,
|
|
282
|
+
softWorkflowStepBudget:
|
|
283
|
+
input.executionPlan.chunkPlan.softWorkflowStepBudget,
|
|
284
|
+
}
|
|
285
|
+
: buildStaticChunkSizingPlan(input.staticPipeline ?? null);
|
|
23
286
|
const planMap = plan?.maps.find(
|
|
24
287
|
(candidate) =>
|
|
25
288
|
candidate.mapName === input.mapName ||
|
|
26
289
|
candidate.tableNamespace === input.mapName,
|
|
27
290
|
);
|
|
291
|
+
const staticMapToolStats = countMapLocalTools({
|
|
292
|
+
staticPipeline: input.staticPipeline,
|
|
293
|
+
planMap,
|
|
294
|
+
mapName: input.mapName,
|
|
295
|
+
});
|
|
296
|
+
const mapToolStats =
|
|
297
|
+
staticMapToolStats ??
|
|
298
|
+
(plan && planMap
|
|
299
|
+
? countFallbackDeclarationTools({
|
|
300
|
+
declarations: plan.toolDeclarations ?? [],
|
|
301
|
+
outputFields: planMap.outputFields,
|
|
302
|
+
externalStepFields: planMap.externalStepFields ?? [],
|
|
303
|
+
})
|
|
304
|
+
: plan
|
|
305
|
+
? countDeclarationTools(plan.toolDeclarations ?? [])
|
|
306
|
+
: [0, 0]);
|
|
28
307
|
const rowsPerChunk = chooseMapChunkSize({
|
|
29
308
|
totalRows: input.rowCountHint,
|
|
30
309
|
mapCount: Math.max(1, plan?.maps.length ?? 1),
|
|
31
310
|
stepsPerChunk: planMap?.stepsPerChunk ?? 1,
|
|
32
311
|
preferredChunkSize: planMap?.defaultChunkSize,
|
|
33
|
-
softWorkflowStepBudget: plan?.
|
|
312
|
+
softWorkflowStepBudget: plan?.softWorkflowStepBudget,
|
|
34
313
|
});
|
|
35
|
-
const estimatedSubrequestProducingSteps =
|
|
36
|
-
estimateSubrequestProducingStepsForMap(plan, planMap);
|
|
37
|
-
if (estimatedSubrequestProducingSteps > 0) {
|
|
38
|
-
const subrequestSafeRows = Math.max(
|
|
39
|
-
1,
|
|
40
|
-
Math.floor(
|
|
41
|
-
WORKER_SUBREQUEST_SAFE_TOOL_CALLS_PER_CHUNK /
|
|
42
|
-
estimatedSubrequestProducingSteps,
|
|
43
|
-
),
|
|
44
|
-
);
|
|
45
|
-
if (subrequestSafeRows < rowsPerChunk) {
|
|
46
|
-
if (input.rowCountHint === null || !Number.isFinite(input.rowCountHint)) {
|
|
47
|
-
return subrequestSafeRows;
|
|
48
|
-
}
|
|
49
|
-
const totalRows = Math.max(0, Math.floor(input.rowCountHint));
|
|
50
|
-
const mapCount = Math.max(1, plan?.maps.length ?? 1);
|
|
51
|
-
const softBudget =
|
|
52
|
-
plan?.chunkPlan.softWorkflowStepBudget ??
|
|
53
|
-
EXECUTION_PLAN_DEFAULTS.workflowSoftStepBudget;
|
|
54
|
-
const maxChunksPerMap = Math.max(
|
|
55
|
-
1,
|
|
56
|
-
Math.floor(
|
|
57
|
-
Math.max(
|
|
58
|
-
mapCount,
|
|
59
|
-
Math.floor(
|
|
60
|
-
(softBudget -
|
|
61
|
-
EXECUTION_PLAN_DEFAULTS.ingestStepCount -
|
|
62
|
-
EXECUTION_PLAN_DEFAULTS.finalizationStepCount) /
|
|
63
|
-
estimatedSubrequestProducingSteps,
|
|
64
|
-
),
|
|
65
|
-
) / mapCount,
|
|
66
|
-
),
|
|
67
|
-
);
|
|
68
|
-
const requiredChunks = Math.ceil(totalRows / subrequestSafeRows);
|
|
69
|
-
if (requiredChunks <= maxChunksPerMap) return subrequestSafeRows;
|
|
70
|
-
throw new Error(
|
|
71
|
-
`Worker budget exceeded for map "${input.mapName}": ${requiredChunks}/${maxChunksPerMap} chunks.`,
|
|
72
|
-
);
|
|
73
|
-
}
|
|
74
|
-
return rowsPerChunk;
|
|
75
|
-
}
|
|
76
314
|
|
|
77
315
|
const toolFreeSimpleMap =
|
|
78
|
-
!!planMap &&
|
|
79
|
-
planMap.stepsPerChunk === 1 &&
|
|
80
|
-
estimateSubrequestProducingStepsForMap(plan, planMap) === 0;
|
|
316
|
+
!!planMap && planMap.stepsPerChunk === 1 && mapToolStats[0] === 0;
|
|
81
317
|
if (
|
|
82
318
|
toolFreeSimpleMap &&
|
|
83
319
|
(input.rowCountHint === null ||
|
|
@@ -86,75 +322,22 @@ export function chooseWorkerMapRowsPerChunk(
|
|
|
86
322
|
return Math.max(rowsPerChunk, CACHE_ENABLED_SIMPLE_MAP_CHUNK_SIZE);
|
|
87
323
|
}
|
|
88
324
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
explicitExternalSteps > 0
|
|
105
|
-
? 0
|
|
106
|
-
: (planMap.stepsPerChunk ?? 1) > 1
|
|
107
|
-
? (planMap.stepsPerChunk ?? 1)
|
|
108
|
-
: 0;
|
|
109
|
-
if (toolDeclarations.length === 0) {
|
|
110
|
-
return Math.max(mapStepsPerChunk, explicitExternalSteps);
|
|
111
|
-
}
|
|
112
|
-
const outputFields = new Set(
|
|
113
|
-
planMap.outputFields.map((field) => field.trim()).filter(Boolean),
|
|
114
|
-
);
|
|
115
|
-
const explicitExternalStepFields = new Set(
|
|
116
|
-
(planMap.externalStepFields ?? [])
|
|
117
|
-
.map((field) => field.trim())
|
|
118
|
-
.filter(Boolean),
|
|
119
|
-
);
|
|
120
|
-
const hasModernStepMetadata =
|
|
121
|
-
Array.isArray(planMap.stepFields) ||
|
|
122
|
-
Array.isArray(planMap.externalStepFields);
|
|
123
|
-
const mapFields = new Set([
|
|
124
|
-
planMap.mapName,
|
|
125
|
-
planMap.tableNamespace,
|
|
126
|
-
...planMap.outputFields,
|
|
127
|
-
...(planMap.externalStepFields ?? []),
|
|
128
|
-
...(hasModernStepMetadata ? [] : (planMap.stepFields ?? [])),
|
|
129
|
-
...planMap.waterfallStages.flatMap((stage) => [
|
|
130
|
-
stage.waterfallId,
|
|
131
|
-
...stage.stageIds,
|
|
132
|
-
]),
|
|
133
|
-
]);
|
|
134
|
-
let scopedToolDeclarations = 0;
|
|
135
|
-
for (const declaration of toolDeclarations) {
|
|
136
|
-
const field = declaration.field?.trim();
|
|
137
|
-
if (!field) continue;
|
|
138
|
-
if (mapFields.has(field)) {
|
|
139
|
-
scopedToolDeclarations += 1;
|
|
140
|
-
continue;
|
|
141
|
-
}
|
|
142
|
-
if (!field.includes('.')) {
|
|
143
|
-
continue;
|
|
144
|
-
}
|
|
145
|
-
const firstSegment = field.slice(0, field.indexOf('.'));
|
|
146
|
-
const lastSegment = field.slice(field.lastIndexOf('.') + 1);
|
|
147
|
-
if (
|
|
148
|
-
outputFields.has(firstSegment) ||
|
|
149
|
-
explicitExternalStepFields.has(lastSegment) ||
|
|
150
|
-
(!hasModernStepMetadata && outputFields.has(lastSegment))
|
|
151
|
-
) {
|
|
152
|
-
scopedToolDeclarations += 1;
|
|
153
|
-
}
|
|
325
|
+
if (mapToolStats[1] > 0) {
|
|
326
|
+
const unbatchedToolCallsPerRow =
|
|
327
|
+
staticMapToolStats !== null
|
|
328
|
+
? mapToolStats[1]
|
|
329
|
+
: Math.max(planMap?.stepsPerChunk ?? 1, mapToolStats[1]);
|
|
330
|
+
const unbatchedSubrequestsPerRow =
|
|
331
|
+
unbatchedToolCallsPerRow * SUBREQUESTS_PER_UNBATCHED_TOOL_CALL;
|
|
332
|
+
const unbatchedToolRowsPerChunk = Math.max(
|
|
333
|
+
1,
|
|
334
|
+
Math.floor(
|
|
335
|
+
UNBATCHED_TOOL_SUBREQUESTS_PER_CHUNK_BUDGET /
|
|
336
|
+
unbatchedSubrequestsPerRow,
|
|
337
|
+
),
|
|
338
|
+
);
|
|
339
|
+
return Math.min(rowsPerChunk, unbatchedToolRowsPerChunk);
|
|
154
340
|
}
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
explicitExternalSteps,
|
|
158
|
-
scopedToolDeclarations,
|
|
159
|
-
);
|
|
341
|
+
|
|
342
|
+
return rowsPerChunk;
|
|
160
343
|
}
|
|
@@ -104,10 +104,10 @@ export const SDK_RELEASE = {
|
|
|
104
104
|
// 0.1.111 ships dataset-native tool list getters and result row datasets.
|
|
105
105
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
106
106
|
// fields shipped in 0.1.153.
|
|
107
|
-
version: '0.1.
|
|
107
|
+
version: '0.1.164',
|
|
108
108
|
apiContract: '2026-06-dataset-handle-results-hard-cutover',
|
|
109
109
|
supportPolicy: {
|
|
110
|
-
latest: '0.1.
|
|
110
|
+
latest: '0.1.164',
|
|
111
111
|
minimumSupported: '0.1.53',
|
|
112
112
|
deprecatedBelow: '0.1.53',
|
|
113
113
|
commandMinimumSupported: [
|
package/dist/cli/index.js
CHANGED
|
@@ -622,10 +622,10 @@ var SDK_RELEASE = {
|
|
|
622
622
|
// 0.1.111 ships dataset-native tool list getters and result row datasets.
|
|
623
623
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
624
624
|
// fields shipped in 0.1.153.
|
|
625
|
-
version: "0.1.
|
|
625
|
+
version: "0.1.164",
|
|
626
626
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
627
627
|
supportPolicy: {
|
|
628
|
-
latest: "0.1.
|
|
628
|
+
latest: "0.1.164",
|
|
629
629
|
minimumSupported: "0.1.53",
|
|
630
630
|
deprecatedBelow: "0.1.53",
|
|
631
631
|
commandMinimumSupported: [
|
|
@@ -5421,6 +5421,7 @@ async function handleUsage(options) {
|
|
|
5421
5421
|
const params = new URLSearchParams();
|
|
5422
5422
|
if (options.limit) params.set("recent_limit", options.limit);
|
|
5423
5423
|
if (options.offset) params.set("recent_offset", options.offset);
|
|
5424
|
+
if (options.runId) params.set("run_id", options.runId);
|
|
5424
5425
|
const suffix = Array.from(params).length > 0 ? `?${params.toString()}` : "";
|
|
5425
5426
|
const payload = await http.get(
|
|
5426
5427
|
`/api/v2/billing/usage${suffix}`
|
|
@@ -5973,8 +5974,9 @@ Notes:
|
|
|
5973
5974
|
Examples:
|
|
5974
5975
|
deepline billing usage
|
|
5975
5976
|
deepline billing usage --limit 50 --offset 50 --json
|
|
5977
|
+
deepline billing usage --run-id play/example/run/20260630t120000-000-abcdef --json
|
|
5976
5978
|
`
|
|
5977
|
-
).option("--limit <n>", "Recent-call page size").option("--offset <n>", "Recent-call offset").option("--json", "Emit JSON output. Also automatic when stdout is piped").action(handleUsage);
|
|
5979
|
+
).option("--limit <n>", "Recent-call page size").option("--offset <n>", "Recent-call offset").option("--run-id <run_id>", "Show recent usage for one play run").option("--json", "Emit JSON output. Also automatic when stdout is piped").action(handleUsage);
|
|
5978
5980
|
billing.command("limit").description("Show configured monthly limit state.").addHelpText(
|
|
5979
5981
|
"after",
|
|
5980
5982
|
`
|
|
@@ -16830,6 +16832,8 @@ var ENRICH_EXPORT_BACKING_ROWS_WAIT_MS = 6e4;
|
|
|
16830
16832
|
var ENRICH_EXPORT_BACKING_ROWS_POLL_MS = 1e3;
|
|
16831
16833
|
var ENRICH_SOURCE_ROW_INDEX_COLUMN = "__deeplineSourceRowIndex";
|
|
16832
16834
|
var ENRICH_ORIGINAL_SOURCE_ROW_INDEX_COLUMN = "__deeplineOriginalSourceRowIndex";
|
|
16835
|
+
var ENRICH_CELL_META_FIELD = "__deeplineCellMeta";
|
|
16836
|
+
var ENRICH_CELL_META_PATCH_FIELD = "__deeplineCellMetaPatch";
|
|
16833
16837
|
var EXIT_SERVER2 = 5;
|
|
16834
16838
|
var ENRICH_DEBUG_T0 = Date.now();
|
|
16835
16839
|
var GENERATED_ENRICH_ROWS_TABLE_NAMESPACE = "deepline_enrich_rows";
|
|
@@ -17902,6 +17906,18 @@ function exportableSheetRow2(row, sourceRowStart = 0) {
|
|
|
17902
17906
|
const data = record.data;
|
|
17903
17907
|
if (data && typeof data === "object" && !Array.isArray(data)) {
|
|
17904
17908
|
const exported = { ...data };
|
|
17909
|
+
applyFailureCellMeta(exported, exported[ENRICH_CELL_META_FIELD], {
|
|
17910
|
+
rowError: record.error
|
|
17911
|
+
});
|
|
17912
|
+
applyFailureCellMeta(exported, record.cellMeta, { rowError: record.error });
|
|
17913
|
+
applyFailureCellMeta(exported, record.cellMetaPatch, {
|
|
17914
|
+
rowError: record.error
|
|
17915
|
+
});
|
|
17916
|
+
applyFailureCellMeta(exported, record[ENRICH_CELL_META_PATCH_FIELD], {
|
|
17917
|
+
rowError: record.error
|
|
17918
|
+
});
|
|
17919
|
+
delete exported[ENRICH_CELL_META_FIELD];
|
|
17920
|
+
delete exported[ENRICH_CELL_META_PATCH_FIELD];
|
|
17905
17921
|
const inputIndex = typeof record.inputIndex === "number" ? record.inputIndex : typeof record.inputIndex === "string" && record.inputIndex.trim() ? Number(record.inputIndex) : Number.NaN;
|
|
17906
17922
|
if (Number.isInteger(inputIndex) && inputIndex >= 0) {
|
|
17907
17923
|
if (ENRICH_SOURCE_ROW_INDEX_COLUMN in exported) {
|
|
@@ -17912,6 +17928,18 @@ function exportableSheetRow2(row, sourceRowStart = 0) {
|
|
|
17912
17928
|
return exported;
|
|
17913
17929
|
}
|
|
17914
17930
|
const fallback = { ...record };
|
|
17931
|
+
applyFailureCellMeta(fallback, fallback[ENRICH_CELL_META_FIELD], {
|
|
17932
|
+
rowError: record.error
|
|
17933
|
+
});
|
|
17934
|
+
applyFailureCellMeta(fallback, record.cellMeta, {
|
|
17935
|
+
rowError: record.error
|
|
17936
|
+
});
|
|
17937
|
+
applyFailureCellMeta(fallback, record.cellMetaPatch, {
|
|
17938
|
+
rowError: record.error
|
|
17939
|
+
});
|
|
17940
|
+
applyFailureCellMeta(fallback, fallback[ENRICH_CELL_META_PATCH_FIELD], {
|
|
17941
|
+
rowError: record.error
|
|
17942
|
+
});
|
|
17915
17943
|
for (const key of [
|
|
17916
17944
|
"key",
|
|
17917
17945
|
"status",
|
|
@@ -17919,6 +17947,8 @@ function exportableSheetRow2(row, sourceRowStart = 0) {
|
|
|
17919
17947
|
"inputIndex",
|
|
17920
17948
|
"runId",
|
|
17921
17949
|
"error",
|
|
17950
|
+
ENRICH_CELL_META_FIELD,
|
|
17951
|
+
ENRICH_CELL_META_PATCH_FIELD,
|
|
17922
17952
|
"stage",
|
|
17923
17953
|
"provider",
|
|
17924
17954
|
"seq",
|
|
@@ -17929,6 +17959,45 @@ function exportableSheetRow2(row, sourceRowStart = 0) {
|
|
|
17929
17959
|
}
|
|
17930
17960
|
return fallback;
|
|
17931
17961
|
}
|
|
17962
|
+
function failureCellFromMeta(meta, fallback) {
|
|
17963
|
+
if (!isRecord7(meta)) {
|
|
17964
|
+
return null;
|
|
17965
|
+
}
|
|
17966
|
+
const status = typeof meta.status === "string" ? meta.status.trim().toLowerCase() : "";
|
|
17967
|
+
const hasFailureStatus = status === "failed" || status === "error";
|
|
17968
|
+
const explicitError = typeof meta.error === "string" && meta.error.trim() ? meta.error.trim() : typeof meta.last_error === "string" && meta.last_error.trim() ? meta.last_error.trim() : typeof meta.message === "string" && meta.message.trim() ? meta.message.trim() : "";
|
|
17969
|
+
if (!hasFailureStatus && !explicitError) {
|
|
17970
|
+
return null;
|
|
17971
|
+
}
|
|
17972
|
+
const fallbackError = hasFailureStatus && typeof fallback.rowError === "string" && fallback.rowError.trim() ? fallback.rowError.trim() : "";
|
|
17973
|
+
const error = explicitError || fallbackError;
|
|
17974
|
+
const message = error || `Column status: ${status || "failed"}`;
|
|
17975
|
+
return {
|
|
17976
|
+
status: status || "failed",
|
|
17977
|
+
error: message,
|
|
17978
|
+
...typeof meta.operation === "string" && meta.operation.trim() ? { operation: meta.operation.trim() } : {},
|
|
17979
|
+
...typeof meta.provider === "string" && meta.provider.trim() ? { provider: meta.provider.trim() } : {}
|
|
17980
|
+
};
|
|
17981
|
+
}
|
|
17982
|
+
function applyFailureCellMeta(row, cellMeta, fallback) {
|
|
17983
|
+
if (!isRecord7(cellMeta)) {
|
|
17984
|
+
return;
|
|
17985
|
+
}
|
|
17986
|
+
for (const [field, meta] of Object.entries(cellMeta)) {
|
|
17987
|
+
if (!field || field.startsWith("__")) {
|
|
17988
|
+
continue;
|
|
17989
|
+
}
|
|
17990
|
+
const failure = failureCellFromMeta(meta, fallback);
|
|
17991
|
+
if (!failure) {
|
|
17992
|
+
continue;
|
|
17993
|
+
}
|
|
17994
|
+
const existing = row[field];
|
|
17995
|
+
if (isNonEmptyCsvCell(existing) && !cellFailureError(existing)) {
|
|
17996
|
+
continue;
|
|
17997
|
+
}
|
|
17998
|
+
row[field] = failure;
|
|
17999
|
+
}
|
|
18000
|
+
}
|
|
17932
18001
|
function mergeExportedSheetRow(target, next) {
|
|
17933
18002
|
const baseMetadata = target._metadata;
|
|
17934
18003
|
const nextMetadata = next._metadata;
|
|
@@ -17990,7 +18059,7 @@ function stripEnrichSourceRowIndex(row) {
|
|
|
17990
18059
|
}
|
|
17991
18060
|
return stripped;
|
|
17992
18061
|
}
|
|
17993
|
-
function
|
|
18062
|
+
function collectHardFailureAliasSpecs(config) {
|
|
17994
18063
|
const aliases = [];
|
|
17995
18064
|
const seen = /* @__PURE__ */ new Set();
|
|
17996
18065
|
for (const command of config.commands) {
|
|
@@ -18005,10 +18074,25 @@ function collectHardFailureAliases(config) {
|
|
|
18005
18074
|
continue;
|
|
18006
18075
|
}
|
|
18007
18076
|
seen.add(alias);
|
|
18008
|
-
aliases.push(
|
|
18077
|
+
aliases.push({
|
|
18078
|
+
alias,
|
|
18079
|
+
operation: command.operation?.trim() || command.tool.trim() || void 0
|
|
18080
|
+
});
|
|
18009
18081
|
}
|
|
18010
18082
|
return aliases;
|
|
18011
18083
|
}
|
|
18084
|
+
function hardFailureOperationByAlias(config) {
|
|
18085
|
+
const operations = /* @__PURE__ */ new Map();
|
|
18086
|
+
if (!config) {
|
|
18087
|
+
return operations;
|
|
18088
|
+
}
|
|
18089
|
+
for (const spec of collectHardFailureAliasSpecs(config)) {
|
|
18090
|
+
if (spec.operation) {
|
|
18091
|
+
operations.set(normalizeAlias2(spec.alias), spec.operation);
|
|
18092
|
+
}
|
|
18093
|
+
}
|
|
18094
|
+
return operations;
|
|
18095
|
+
}
|
|
18012
18096
|
function cellFailureError(value) {
|
|
18013
18097
|
const parsed = parseMaybeJsonObject(value);
|
|
18014
18098
|
if (!isRecord7(parsed)) {
|
|
@@ -18118,14 +18202,15 @@ function aliasFlattenedCells(row, alias) {
|
|
|
18118
18202
|
return cells;
|
|
18119
18203
|
}
|
|
18120
18204
|
function collectEnrichFailureJobs(input2) {
|
|
18121
|
-
const aliases =
|
|
18205
|
+
const aliases = collectHardFailureAliasSpecs(input2.config);
|
|
18122
18206
|
if (aliases.length === 0 || input2.rows.length === 0) {
|
|
18123
18207
|
return [];
|
|
18124
18208
|
}
|
|
18125
18209
|
const rowOffset = input2.rowStart ?? 0;
|
|
18126
18210
|
const jobs = [];
|
|
18127
18211
|
input2.rows.forEach((row, rowIndex) => {
|
|
18128
|
-
aliases.forEach((
|
|
18212
|
+
aliases.forEach((spec, aliasIndex) => {
|
|
18213
|
+
const { alias } = spec;
|
|
18129
18214
|
const failure = aliasFailureCellCandidates(row, alias).map(cellFailureError).find(
|
|
18130
18215
|
(candidate) => Boolean(candidate)
|
|
18131
18216
|
);
|
|
@@ -18133,15 +18218,20 @@ function collectEnrichFailureJobs(input2) {
|
|
|
18133
18218
|
return;
|
|
18134
18219
|
}
|
|
18135
18220
|
const rowId = sourceRowIndexFromEnrichRow(row, 0, Number.MAX_SAFE_INTEGER) ?? rowOffset + rowIndex;
|
|
18221
|
+
const operation = failure.operation ?? spec.operation;
|
|
18222
|
+
const message = enrichFailureMessageWithOperation(
|
|
18223
|
+
failure.message,
|
|
18224
|
+
operation
|
|
18225
|
+
);
|
|
18136
18226
|
jobs.push({
|
|
18137
18227
|
job_id: `row-${rowId}-${alias}`,
|
|
18138
18228
|
row_id: rowId,
|
|
18139
18229
|
col_index: aliasIndex,
|
|
18140
18230
|
column: alias,
|
|
18141
18231
|
status: "failed",
|
|
18142
|
-
last_error:
|
|
18143
|
-
error:
|
|
18144
|
-
...
|
|
18232
|
+
last_error: message,
|
|
18233
|
+
error: message,
|
|
18234
|
+
...operation ? { operation } : {},
|
|
18145
18235
|
...failure.provider ? { provider: failure.provider } : {}
|
|
18146
18236
|
});
|
|
18147
18237
|
});
|
|
@@ -18149,7 +18239,7 @@ function collectEnrichFailureJobs(input2) {
|
|
|
18149
18239
|
return jobs;
|
|
18150
18240
|
}
|
|
18151
18241
|
function collectStatusFailureJobs(input2) {
|
|
18152
|
-
const aliases =
|
|
18242
|
+
const aliases = collectHardFailureAliasSpecs(input2.config);
|
|
18153
18243
|
if (aliases.length === 0) {
|
|
18154
18244
|
return [];
|
|
18155
18245
|
}
|
|
@@ -18157,7 +18247,8 @@ function collectStatusFailureJobs(input2) {
|
|
|
18157
18247
|
const rowStart = input2.rowRange.rowStart ?? 0;
|
|
18158
18248
|
const selectedRows = input2.rowRange.rowEnd !== null && input2.rowRange.rowEnd >= rowStart ? input2.rowRange.rowEnd - rowStart + 1 : Number.MAX_SAFE_INTEGER;
|
|
18159
18249
|
const jobs = [];
|
|
18160
|
-
aliases.forEach((
|
|
18250
|
+
aliases.forEach((spec, aliasIndex) => {
|
|
18251
|
+
const { alias } = spec;
|
|
18161
18252
|
const stat2 = summaries.map((summary) => summary[alias]).find(isRecord7);
|
|
18162
18253
|
if (!stat2) {
|
|
18163
18254
|
return;
|
|
@@ -18174,7 +18265,12 @@ function collectStatusFailureJobs(input2) {
|
|
|
18174
18265
|
if (!Number.isFinite(failedCount) || failedCount <= 0) {
|
|
18175
18266
|
return;
|
|
18176
18267
|
}
|
|
18177
|
-
const message =
|
|
18268
|
+
const message = statusFailureMessage({
|
|
18269
|
+
stat: stat2,
|
|
18270
|
+
alias,
|
|
18271
|
+
failedCount,
|
|
18272
|
+
operation: spec.operation
|
|
18273
|
+
});
|
|
18178
18274
|
for (let offset = 0; offset < failedCount; offset += 1) {
|
|
18179
18275
|
const rowId = rowStart + offset;
|
|
18180
18276
|
jobs.push({
|
|
@@ -18190,6 +18286,26 @@ function collectStatusFailureJobs(input2) {
|
|
|
18190
18286
|
});
|
|
18191
18287
|
return jobs;
|
|
18192
18288
|
}
|
|
18289
|
+
function statusFailureMessage(input2) {
|
|
18290
|
+
const raw = firstCollectedStringField(input2.stat, "error") ?? firstCollectedStringField(input2.stat, "message");
|
|
18291
|
+
if (!raw) {
|
|
18292
|
+
return `Column ${input2.alias} failed for ${input2.failedCount} row(s).`;
|
|
18293
|
+
}
|
|
18294
|
+
return enrichFailureMessageWithOperation(raw, input2.operation);
|
|
18295
|
+
}
|
|
18296
|
+
function enrichFailureMessageWithOperation(message, operation) {
|
|
18297
|
+
const raw = message.trim();
|
|
18298
|
+
const marker = operation?.trim();
|
|
18299
|
+
if (!marker || raw.includes(marker)) {
|
|
18300
|
+
return raw;
|
|
18301
|
+
}
|
|
18302
|
+
if (/^(?:[A-Za-z0-9_:-]+:\s*)?(?:Error|TypeError|ReferenceError|SyntaxError|RangeError):\s+/.test(
|
|
18303
|
+
raw
|
|
18304
|
+
)) {
|
|
18305
|
+
return `${marker}: ${raw}`;
|
|
18306
|
+
}
|
|
18307
|
+
return `${marker}: Error: ${raw}`;
|
|
18308
|
+
}
|
|
18193
18309
|
function collectCompleteStatusFailureMessages(input2) {
|
|
18194
18310
|
const requiredCount = Math.max(1, Math.trunc(input2.rowCount));
|
|
18195
18311
|
const jobs = collectStatusFailureJobs({
|
|
@@ -18756,6 +18872,7 @@ function materializeAliasSuccessCell(row, alias) {
|
|
|
18756
18872
|
}
|
|
18757
18873
|
function normalizeEnrichRowsForCsvExport(rows, config, options) {
|
|
18758
18874
|
const aliases = config ? collectConfigScalarAliasOrder(config) : [];
|
|
18875
|
+
const failureOperationByAlias = hardFailureOperationByAlias(config);
|
|
18759
18876
|
return rows.map((row) => {
|
|
18760
18877
|
const metadata = legacyMetadataFromRow(row);
|
|
18761
18878
|
const normalized = metadata ? { ...row, _metadata: metadata } : { ...row };
|
|
@@ -18790,7 +18907,10 @@ function normalizeEnrichRowsForCsvExport(rows, config, options) {
|
|
|
18790
18907
|
(candidate) => Boolean(candidate)
|
|
18791
18908
|
);
|
|
18792
18909
|
if (failure) {
|
|
18793
|
-
normalized[alias] =
|
|
18910
|
+
normalized[alias] = enrichFailureMessageWithOperation(
|
|
18911
|
+
failure.message,
|
|
18912
|
+
failure.operation ?? failureOperationByAlias.get(normalizeAlias2(alias))
|
|
18913
|
+
);
|
|
18794
18914
|
}
|
|
18795
18915
|
}
|
|
18796
18916
|
for (const alias of aliases) {
|
|
@@ -18799,21 +18919,24 @@ function normalizeEnrichRowsForCsvExport(rows, config, options) {
|
|
|
18799
18919
|
normalized[alias] = value;
|
|
18800
18920
|
continue;
|
|
18801
18921
|
}
|
|
18802
|
-
if (isNonEmptyCsvCell(normalized[alias])) {
|
|
18803
|
-
continue;
|
|
18804
|
-
}
|
|
18805
18922
|
const failure = aliasFailureCellCandidates(normalized, alias).map(cellFailureError).find(
|
|
18806
18923
|
(candidate) => Boolean(candidate)
|
|
18807
18924
|
);
|
|
18808
18925
|
if (failure) {
|
|
18809
|
-
normalized[alias] =
|
|
18926
|
+
normalized[alias] = enrichFailureMessageWithOperation(
|
|
18927
|
+
failure.message,
|
|
18928
|
+
failure.operation ?? failureOperationByAlias.get(normalizeAlias2(alias))
|
|
18929
|
+
);
|
|
18930
|
+
continue;
|
|
18931
|
+
}
|
|
18932
|
+
if (isNonEmptyCsvCell(normalized[alias])) {
|
|
18810
18933
|
continue;
|
|
18811
18934
|
}
|
|
18812
|
-
const
|
|
18935
|
+
const statusFailureMessage2 = options?.statusFailureMessages?.get(
|
|
18813
18936
|
normalizeAlias2(alias)
|
|
18814
18937
|
);
|
|
18815
|
-
if (
|
|
18816
|
-
normalized[alias] =
|
|
18938
|
+
if (statusFailureMessage2) {
|
|
18939
|
+
normalized[alias] = statusFailureMessage2;
|
|
18817
18940
|
}
|
|
18818
18941
|
}
|
|
18819
18942
|
return normalized;
|
package/dist/cli/index.mjs
CHANGED
|
@@ -607,10 +607,10 @@ var SDK_RELEASE = {
|
|
|
607
607
|
// 0.1.111 ships dataset-native tool list getters and result row datasets.
|
|
608
608
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
609
609
|
// fields shipped in 0.1.153.
|
|
610
|
-
version: "0.1.
|
|
610
|
+
version: "0.1.164",
|
|
611
611
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
612
612
|
supportPolicy: {
|
|
613
|
-
latest: "0.1.
|
|
613
|
+
latest: "0.1.164",
|
|
614
614
|
minimumSupported: "0.1.53",
|
|
615
615
|
deprecatedBelow: "0.1.53",
|
|
616
616
|
commandMinimumSupported: [
|
|
@@ -5418,6 +5418,7 @@ async function handleUsage(options) {
|
|
|
5418
5418
|
const params = new URLSearchParams();
|
|
5419
5419
|
if (options.limit) params.set("recent_limit", options.limit);
|
|
5420
5420
|
if (options.offset) params.set("recent_offset", options.offset);
|
|
5421
|
+
if (options.runId) params.set("run_id", options.runId);
|
|
5421
5422
|
const suffix = Array.from(params).length > 0 ? `?${params.toString()}` : "";
|
|
5422
5423
|
const payload = await http.get(
|
|
5423
5424
|
`/api/v2/billing/usage${suffix}`
|
|
@@ -5970,8 +5971,9 @@ Notes:
|
|
|
5970
5971
|
Examples:
|
|
5971
5972
|
deepline billing usage
|
|
5972
5973
|
deepline billing usage --limit 50 --offset 50 --json
|
|
5974
|
+
deepline billing usage --run-id play/example/run/20260630t120000-000-abcdef --json
|
|
5973
5975
|
`
|
|
5974
|
-
).option("--limit <n>", "Recent-call page size").option("--offset <n>", "Recent-call offset").option("--json", "Emit JSON output. Also automatic when stdout is piped").action(handleUsage);
|
|
5976
|
+
).option("--limit <n>", "Recent-call page size").option("--offset <n>", "Recent-call offset").option("--run-id <run_id>", "Show recent usage for one play run").option("--json", "Emit JSON output. Also automatic when stdout is piped").action(handleUsage);
|
|
5975
5977
|
billing.command("limit").description("Show configured monthly limit state.").addHelpText(
|
|
5976
5978
|
"after",
|
|
5977
5979
|
`
|
|
@@ -16857,6 +16859,8 @@ var ENRICH_EXPORT_BACKING_ROWS_WAIT_MS = 6e4;
|
|
|
16857
16859
|
var ENRICH_EXPORT_BACKING_ROWS_POLL_MS = 1e3;
|
|
16858
16860
|
var ENRICH_SOURCE_ROW_INDEX_COLUMN = "__deeplineSourceRowIndex";
|
|
16859
16861
|
var ENRICH_ORIGINAL_SOURCE_ROW_INDEX_COLUMN = "__deeplineOriginalSourceRowIndex";
|
|
16862
|
+
var ENRICH_CELL_META_FIELD = "__deeplineCellMeta";
|
|
16863
|
+
var ENRICH_CELL_META_PATCH_FIELD = "__deeplineCellMetaPatch";
|
|
16860
16864
|
var EXIT_SERVER2 = 5;
|
|
16861
16865
|
var ENRICH_DEBUG_T0 = Date.now();
|
|
16862
16866
|
var GENERATED_ENRICH_ROWS_TABLE_NAMESPACE = "deepline_enrich_rows";
|
|
@@ -17929,6 +17933,18 @@ function exportableSheetRow2(row, sourceRowStart = 0) {
|
|
|
17929
17933
|
const data = record.data;
|
|
17930
17934
|
if (data && typeof data === "object" && !Array.isArray(data)) {
|
|
17931
17935
|
const exported = { ...data };
|
|
17936
|
+
applyFailureCellMeta(exported, exported[ENRICH_CELL_META_FIELD], {
|
|
17937
|
+
rowError: record.error
|
|
17938
|
+
});
|
|
17939
|
+
applyFailureCellMeta(exported, record.cellMeta, { rowError: record.error });
|
|
17940
|
+
applyFailureCellMeta(exported, record.cellMetaPatch, {
|
|
17941
|
+
rowError: record.error
|
|
17942
|
+
});
|
|
17943
|
+
applyFailureCellMeta(exported, record[ENRICH_CELL_META_PATCH_FIELD], {
|
|
17944
|
+
rowError: record.error
|
|
17945
|
+
});
|
|
17946
|
+
delete exported[ENRICH_CELL_META_FIELD];
|
|
17947
|
+
delete exported[ENRICH_CELL_META_PATCH_FIELD];
|
|
17932
17948
|
const inputIndex = typeof record.inputIndex === "number" ? record.inputIndex : typeof record.inputIndex === "string" && record.inputIndex.trim() ? Number(record.inputIndex) : Number.NaN;
|
|
17933
17949
|
if (Number.isInteger(inputIndex) && inputIndex >= 0) {
|
|
17934
17950
|
if (ENRICH_SOURCE_ROW_INDEX_COLUMN in exported) {
|
|
@@ -17939,6 +17955,18 @@ function exportableSheetRow2(row, sourceRowStart = 0) {
|
|
|
17939
17955
|
return exported;
|
|
17940
17956
|
}
|
|
17941
17957
|
const fallback = { ...record };
|
|
17958
|
+
applyFailureCellMeta(fallback, fallback[ENRICH_CELL_META_FIELD], {
|
|
17959
|
+
rowError: record.error
|
|
17960
|
+
});
|
|
17961
|
+
applyFailureCellMeta(fallback, record.cellMeta, {
|
|
17962
|
+
rowError: record.error
|
|
17963
|
+
});
|
|
17964
|
+
applyFailureCellMeta(fallback, record.cellMetaPatch, {
|
|
17965
|
+
rowError: record.error
|
|
17966
|
+
});
|
|
17967
|
+
applyFailureCellMeta(fallback, fallback[ENRICH_CELL_META_PATCH_FIELD], {
|
|
17968
|
+
rowError: record.error
|
|
17969
|
+
});
|
|
17942
17970
|
for (const key of [
|
|
17943
17971
|
"key",
|
|
17944
17972
|
"status",
|
|
@@ -17946,6 +17974,8 @@ function exportableSheetRow2(row, sourceRowStart = 0) {
|
|
|
17946
17974
|
"inputIndex",
|
|
17947
17975
|
"runId",
|
|
17948
17976
|
"error",
|
|
17977
|
+
ENRICH_CELL_META_FIELD,
|
|
17978
|
+
ENRICH_CELL_META_PATCH_FIELD,
|
|
17949
17979
|
"stage",
|
|
17950
17980
|
"provider",
|
|
17951
17981
|
"seq",
|
|
@@ -17956,6 +17986,45 @@ function exportableSheetRow2(row, sourceRowStart = 0) {
|
|
|
17956
17986
|
}
|
|
17957
17987
|
return fallback;
|
|
17958
17988
|
}
|
|
17989
|
+
function failureCellFromMeta(meta, fallback) {
|
|
17990
|
+
if (!isRecord7(meta)) {
|
|
17991
|
+
return null;
|
|
17992
|
+
}
|
|
17993
|
+
const status = typeof meta.status === "string" ? meta.status.trim().toLowerCase() : "";
|
|
17994
|
+
const hasFailureStatus = status === "failed" || status === "error";
|
|
17995
|
+
const explicitError = typeof meta.error === "string" && meta.error.trim() ? meta.error.trim() : typeof meta.last_error === "string" && meta.last_error.trim() ? meta.last_error.trim() : typeof meta.message === "string" && meta.message.trim() ? meta.message.trim() : "";
|
|
17996
|
+
if (!hasFailureStatus && !explicitError) {
|
|
17997
|
+
return null;
|
|
17998
|
+
}
|
|
17999
|
+
const fallbackError = hasFailureStatus && typeof fallback.rowError === "string" && fallback.rowError.trim() ? fallback.rowError.trim() : "";
|
|
18000
|
+
const error = explicitError || fallbackError;
|
|
18001
|
+
const message = error || `Column status: ${status || "failed"}`;
|
|
18002
|
+
return {
|
|
18003
|
+
status: status || "failed",
|
|
18004
|
+
error: message,
|
|
18005
|
+
...typeof meta.operation === "string" && meta.operation.trim() ? { operation: meta.operation.trim() } : {},
|
|
18006
|
+
...typeof meta.provider === "string" && meta.provider.trim() ? { provider: meta.provider.trim() } : {}
|
|
18007
|
+
};
|
|
18008
|
+
}
|
|
18009
|
+
function applyFailureCellMeta(row, cellMeta, fallback) {
|
|
18010
|
+
if (!isRecord7(cellMeta)) {
|
|
18011
|
+
return;
|
|
18012
|
+
}
|
|
18013
|
+
for (const [field, meta] of Object.entries(cellMeta)) {
|
|
18014
|
+
if (!field || field.startsWith("__")) {
|
|
18015
|
+
continue;
|
|
18016
|
+
}
|
|
18017
|
+
const failure = failureCellFromMeta(meta, fallback);
|
|
18018
|
+
if (!failure) {
|
|
18019
|
+
continue;
|
|
18020
|
+
}
|
|
18021
|
+
const existing = row[field];
|
|
18022
|
+
if (isNonEmptyCsvCell(existing) && !cellFailureError(existing)) {
|
|
18023
|
+
continue;
|
|
18024
|
+
}
|
|
18025
|
+
row[field] = failure;
|
|
18026
|
+
}
|
|
18027
|
+
}
|
|
17959
18028
|
function mergeExportedSheetRow(target, next) {
|
|
17960
18029
|
const baseMetadata = target._metadata;
|
|
17961
18030
|
const nextMetadata = next._metadata;
|
|
@@ -18017,7 +18086,7 @@ function stripEnrichSourceRowIndex(row) {
|
|
|
18017
18086
|
}
|
|
18018
18087
|
return stripped;
|
|
18019
18088
|
}
|
|
18020
|
-
function
|
|
18089
|
+
function collectHardFailureAliasSpecs(config) {
|
|
18021
18090
|
const aliases = [];
|
|
18022
18091
|
const seen = /* @__PURE__ */ new Set();
|
|
18023
18092
|
for (const command of config.commands) {
|
|
@@ -18032,10 +18101,25 @@ function collectHardFailureAliases(config) {
|
|
|
18032
18101
|
continue;
|
|
18033
18102
|
}
|
|
18034
18103
|
seen.add(alias);
|
|
18035
|
-
aliases.push(
|
|
18104
|
+
aliases.push({
|
|
18105
|
+
alias,
|
|
18106
|
+
operation: command.operation?.trim() || command.tool.trim() || void 0
|
|
18107
|
+
});
|
|
18036
18108
|
}
|
|
18037
18109
|
return aliases;
|
|
18038
18110
|
}
|
|
18111
|
+
function hardFailureOperationByAlias(config) {
|
|
18112
|
+
const operations = /* @__PURE__ */ new Map();
|
|
18113
|
+
if (!config) {
|
|
18114
|
+
return operations;
|
|
18115
|
+
}
|
|
18116
|
+
for (const spec of collectHardFailureAliasSpecs(config)) {
|
|
18117
|
+
if (spec.operation) {
|
|
18118
|
+
operations.set(normalizeAlias2(spec.alias), spec.operation);
|
|
18119
|
+
}
|
|
18120
|
+
}
|
|
18121
|
+
return operations;
|
|
18122
|
+
}
|
|
18039
18123
|
function cellFailureError(value) {
|
|
18040
18124
|
const parsed = parseMaybeJsonObject(value);
|
|
18041
18125
|
if (!isRecord7(parsed)) {
|
|
@@ -18145,14 +18229,15 @@ function aliasFlattenedCells(row, alias) {
|
|
|
18145
18229
|
return cells;
|
|
18146
18230
|
}
|
|
18147
18231
|
function collectEnrichFailureJobs(input2) {
|
|
18148
|
-
const aliases =
|
|
18232
|
+
const aliases = collectHardFailureAliasSpecs(input2.config);
|
|
18149
18233
|
if (aliases.length === 0 || input2.rows.length === 0) {
|
|
18150
18234
|
return [];
|
|
18151
18235
|
}
|
|
18152
18236
|
const rowOffset = input2.rowStart ?? 0;
|
|
18153
18237
|
const jobs = [];
|
|
18154
18238
|
input2.rows.forEach((row, rowIndex) => {
|
|
18155
|
-
aliases.forEach((
|
|
18239
|
+
aliases.forEach((spec, aliasIndex) => {
|
|
18240
|
+
const { alias } = spec;
|
|
18156
18241
|
const failure = aliasFailureCellCandidates(row, alias).map(cellFailureError).find(
|
|
18157
18242
|
(candidate) => Boolean(candidate)
|
|
18158
18243
|
);
|
|
@@ -18160,15 +18245,20 @@ function collectEnrichFailureJobs(input2) {
|
|
|
18160
18245
|
return;
|
|
18161
18246
|
}
|
|
18162
18247
|
const rowId = sourceRowIndexFromEnrichRow(row, 0, Number.MAX_SAFE_INTEGER) ?? rowOffset + rowIndex;
|
|
18248
|
+
const operation = failure.operation ?? spec.operation;
|
|
18249
|
+
const message = enrichFailureMessageWithOperation(
|
|
18250
|
+
failure.message,
|
|
18251
|
+
operation
|
|
18252
|
+
);
|
|
18163
18253
|
jobs.push({
|
|
18164
18254
|
job_id: `row-${rowId}-${alias}`,
|
|
18165
18255
|
row_id: rowId,
|
|
18166
18256
|
col_index: aliasIndex,
|
|
18167
18257
|
column: alias,
|
|
18168
18258
|
status: "failed",
|
|
18169
|
-
last_error:
|
|
18170
|
-
error:
|
|
18171
|
-
...
|
|
18259
|
+
last_error: message,
|
|
18260
|
+
error: message,
|
|
18261
|
+
...operation ? { operation } : {},
|
|
18172
18262
|
...failure.provider ? { provider: failure.provider } : {}
|
|
18173
18263
|
});
|
|
18174
18264
|
});
|
|
@@ -18176,7 +18266,7 @@ function collectEnrichFailureJobs(input2) {
|
|
|
18176
18266
|
return jobs;
|
|
18177
18267
|
}
|
|
18178
18268
|
function collectStatusFailureJobs(input2) {
|
|
18179
|
-
const aliases =
|
|
18269
|
+
const aliases = collectHardFailureAliasSpecs(input2.config);
|
|
18180
18270
|
if (aliases.length === 0) {
|
|
18181
18271
|
return [];
|
|
18182
18272
|
}
|
|
@@ -18184,7 +18274,8 @@ function collectStatusFailureJobs(input2) {
|
|
|
18184
18274
|
const rowStart = input2.rowRange.rowStart ?? 0;
|
|
18185
18275
|
const selectedRows = input2.rowRange.rowEnd !== null && input2.rowRange.rowEnd >= rowStart ? input2.rowRange.rowEnd - rowStart + 1 : Number.MAX_SAFE_INTEGER;
|
|
18186
18276
|
const jobs = [];
|
|
18187
|
-
aliases.forEach((
|
|
18277
|
+
aliases.forEach((spec, aliasIndex) => {
|
|
18278
|
+
const { alias } = spec;
|
|
18188
18279
|
const stat2 = summaries.map((summary) => summary[alias]).find(isRecord7);
|
|
18189
18280
|
if (!stat2) {
|
|
18190
18281
|
return;
|
|
@@ -18201,7 +18292,12 @@ function collectStatusFailureJobs(input2) {
|
|
|
18201
18292
|
if (!Number.isFinite(failedCount) || failedCount <= 0) {
|
|
18202
18293
|
return;
|
|
18203
18294
|
}
|
|
18204
|
-
const message =
|
|
18295
|
+
const message = statusFailureMessage({
|
|
18296
|
+
stat: stat2,
|
|
18297
|
+
alias,
|
|
18298
|
+
failedCount,
|
|
18299
|
+
operation: spec.operation
|
|
18300
|
+
});
|
|
18205
18301
|
for (let offset = 0; offset < failedCount; offset += 1) {
|
|
18206
18302
|
const rowId = rowStart + offset;
|
|
18207
18303
|
jobs.push({
|
|
@@ -18217,6 +18313,26 @@ function collectStatusFailureJobs(input2) {
|
|
|
18217
18313
|
});
|
|
18218
18314
|
return jobs;
|
|
18219
18315
|
}
|
|
18316
|
+
function statusFailureMessage(input2) {
|
|
18317
|
+
const raw = firstCollectedStringField(input2.stat, "error") ?? firstCollectedStringField(input2.stat, "message");
|
|
18318
|
+
if (!raw) {
|
|
18319
|
+
return `Column ${input2.alias} failed for ${input2.failedCount} row(s).`;
|
|
18320
|
+
}
|
|
18321
|
+
return enrichFailureMessageWithOperation(raw, input2.operation);
|
|
18322
|
+
}
|
|
18323
|
+
function enrichFailureMessageWithOperation(message, operation) {
|
|
18324
|
+
const raw = message.trim();
|
|
18325
|
+
const marker = operation?.trim();
|
|
18326
|
+
if (!marker || raw.includes(marker)) {
|
|
18327
|
+
return raw;
|
|
18328
|
+
}
|
|
18329
|
+
if (/^(?:[A-Za-z0-9_:-]+:\s*)?(?:Error|TypeError|ReferenceError|SyntaxError|RangeError):\s+/.test(
|
|
18330
|
+
raw
|
|
18331
|
+
)) {
|
|
18332
|
+
return `${marker}: ${raw}`;
|
|
18333
|
+
}
|
|
18334
|
+
return `${marker}: Error: ${raw}`;
|
|
18335
|
+
}
|
|
18220
18336
|
function collectCompleteStatusFailureMessages(input2) {
|
|
18221
18337
|
const requiredCount = Math.max(1, Math.trunc(input2.rowCount));
|
|
18222
18338
|
const jobs = collectStatusFailureJobs({
|
|
@@ -18783,6 +18899,7 @@ function materializeAliasSuccessCell(row, alias) {
|
|
|
18783
18899
|
}
|
|
18784
18900
|
function normalizeEnrichRowsForCsvExport(rows, config, options) {
|
|
18785
18901
|
const aliases = config ? collectConfigScalarAliasOrder(config) : [];
|
|
18902
|
+
const failureOperationByAlias = hardFailureOperationByAlias(config);
|
|
18786
18903
|
return rows.map((row) => {
|
|
18787
18904
|
const metadata = legacyMetadataFromRow(row);
|
|
18788
18905
|
const normalized = metadata ? { ...row, _metadata: metadata } : { ...row };
|
|
@@ -18817,7 +18934,10 @@ function normalizeEnrichRowsForCsvExport(rows, config, options) {
|
|
|
18817
18934
|
(candidate) => Boolean(candidate)
|
|
18818
18935
|
);
|
|
18819
18936
|
if (failure) {
|
|
18820
|
-
normalized[alias] =
|
|
18937
|
+
normalized[alias] = enrichFailureMessageWithOperation(
|
|
18938
|
+
failure.message,
|
|
18939
|
+
failure.operation ?? failureOperationByAlias.get(normalizeAlias2(alias))
|
|
18940
|
+
);
|
|
18821
18941
|
}
|
|
18822
18942
|
}
|
|
18823
18943
|
for (const alias of aliases) {
|
|
@@ -18826,21 +18946,24 @@ function normalizeEnrichRowsForCsvExport(rows, config, options) {
|
|
|
18826
18946
|
normalized[alias] = value;
|
|
18827
18947
|
continue;
|
|
18828
18948
|
}
|
|
18829
|
-
if (isNonEmptyCsvCell(normalized[alias])) {
|
|
18830
|
-
continue;
|
|
18831
|
-
}
|
|
18832
18949
|
const failure = aliasFailureCellCandidates(normalized, alias).map(cellFailureError).find(
|
|
18833
18950
|
(candidate) => Boolean(candidate)
|
|
18834
18951
|
);
|
|
18835
18952
|
if (failure) {
|
|
18836
|
-
normalized[alias] =
|
|
18953
|
+
normalized[alias] = enrichFailureMessageWithOperation(
|
|
18954
|
+
failure.message,
|
|
18955
|
+
failure.operation ?? failureOperationByAlias.get(normalizeAlias2(alias))
|
|
18956
|
+
);
|
|
18957
|
+
continue;
|
|
18958
|
+
}
|
|
18959
|
+
if (isNonEmptyCsvCell(normalized[alias])) {
|
|
18837
18960
|
continue;
|
|
18838
18961
|
}
|
|
18839
|
-
const
|
|
18962
|
+
const statusFailureMessage2 = options?.statusFailureMessages?.get(
|
|
18840
18963
|
normalizeAlias2(alias)
|
|
18841
18964
|
);
|
|
18842
|
-
if (
|
|
18843
|
-
normalized[alias] =
|
|
18965
|
+
if (statusFailureMessage2) {
|
|
18966
|
+
normalized[alias] = statusFailureMessage2;
|
|
18844
18967
|
}
|
|
18845
18968
|
}
|
|
18846
18969
|
return normalized;
|
package/dist/index.js
CHANGED
|
@@ -421,10 +421,10 @@ var SDK_RELEASE = {
|
|
|
421
421
|
// 0.1.111 ships dataset-native tool list getters and result row datasets.
|
|
422
422
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
423
423
|
// fields shipped in 0.1.153.
|
|
424
|
-
version: "0.1.
|
|
424
|
+
version: "0.1.164",
|
|
425
425
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
426
426
|
supportPolicy: {
|
|
427
|
-
latest: "0.1.
|
|
427
|
+
latest: "0.1.164",
|
|
428
428
|
minimumSupported: "0.1.53",
|
|
429
429
|
deprecatedBelow: "0.1.53",
|
|
430
430
|
commandMinimumSupported: [
|
package/dist/index.mjs
CHANGED
|
@@ -351,10 +351,10 @@ var SDK_RELEASE = {
|
|
|
351
351
|
// 0.1.111 ships dataset-native tool list getters and result row datasets.
|
|
352
352
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
353
353
|
// fields shipped in 0.1.153.
|
|
354
|
-
version: "0.1.
|
|
354
|
+
version: "0.1.164",
|
|
355
355
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
356
356
|
supportPolicy: {
|
|
357
|
-
latest: "0.1.
|
|
357
|
+
latest: "0.1.164",
|
|
358
358
|
minimumSupported: "0.1.53",
|
|
359
359
|
deprecatedBelow: "0.1.53",
|
|
360
360
|
commandMinimumSupported: [
|