deepline 0.1.155 → 0.1.157
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +146 -14
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +53 -1
- package/dist/bundling-sources/sdk/src/play.ts +4 -2
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-data-plane/sheet-contract.ts +8 -4
- package/dist/bundling-sources/shared_libs/play-runtime/execution-plan.ts +159 -42
- package/dist/bundling-sources/shared_libs/play-runtime/map-chunk-limits.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +19 -3
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +464 -226
- package/dist/cli/index.js +76 -29
- package/dist/cli/index.mjs +100 -49
- package/dist/index.d.mts +4 -2
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/dist/plays/bundle-play-file.mjs +318 -181
- package/package.json +3 -1
|
@@ -246,6 +246,7 @@ import {
|
|
|
246
246
|
previousCellFromValue,
|
|
247
247
|
type PreviousCell,
|
|
248
248
|
} from '../../../shared_libs/play-runtime/cell-staleness';
|
|
249
|
+
import type { PlayRowUpdate } from '../../../shared_libs/play-runtime/ctx-types';
|
|
249
250
|
|
|
250
251
|
// The play's default export. The bundler injects this — see bundle-play-file.ts.
|
|
251
252
|
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
@@ -1517,6 +1518,13 @@ type RecordedStepProgramOutput = {
|
|
|
1517
1518
|
status?: 'skipped';
|
|
1518
1519
|
};
|
|
1519
1520
|
|
|
1521
|
+
type WorkerStepProgramRecorder = {
|
|
1522
|
+
parentField: string;
|
|
1523
|
+
path: string[];
|
|
1524
|
+
outputs: RecordedStepProgramOutput[];
|
|
1525
|
+
onOutput?: (output: RecordedStepProgramOutput) => void | Promise<void>;
|
|
1526
|
+
};
|
|
1527
|
+
|
|
1520
1528
|
type WorkerStepResolution = {
|
|
1521
1529
|
value: unknown;
|
|
1522
1530
|
status?: 'skipped';
|
|
@@ -2670,11 +2678,7 @@ async function executeWorkerStepResolver(
|
|
|
2670
2678
|
ctx: unknown,
|
|
2671
2679
|
index: number,
|
|
2672
2680
|
previousCell?: PreviousCell,
|
|
2673
|
-
recorder?:
|
|
2674
|
-
parentField: string;
|
|
2675
|
-
path: string[];
|
|
2676
|
-
outputs: RecordedStepProgramOutput[];
|
|
2677
|
-
},
|
|
2681
|
+
recorder?: WorkerStepProgramRecorder,
|
|
2678
2682
|
): Promise<WorkerStepResolution> {
|
|
2679
2683
|
if (isWorkerConditionalStepResolver(resolver)) {
|
|
2680
2684
|
const shouldRun = await resolver.when(row, index);
|
|
@@ -2721,11 +2725,7 @@ async function executeWorkerStepProgram(
|
|
|
2721
2725
|
inputRow: Record<string, unknown>,
|
|
2722
2726
|
ctx: unknown,
|
|
2723
2727
|
index: number,
|
|
2724
|
-
recorder?:
|
|
2725
|
-
parentField: string;
|
|
2726
|
-
path: string[];
|
|
2727
|
-
outputs: RecordedStepProgramOutput[];
|
|
2728
|
-
},
|
|
2728
|
+
recorder?: WorkerStepProgramRecorder,
|
|
2729
2729
|
workflowStep?: WorkflowStep,
|
|
2730
2730
|
): Promise<unknown> {
|
|
2731
2731
|
let currentRow: Record<string, unknown> = cloneCsvAliasedRow(inputRow);
|
|
@@ -2767,13 +2767,15 @@ async function executeWorkerStepProgram(
|
|
|
2767
2767
|
const value = deserializeDurableStepValue(resolution.value);
|
|
2768
2768
|
currentRow = cloneCsvAliasedRow(currentRow, { [step.name]: value });
|
|
2769
2769
|
if (recorder) {
|
|
2770
|
-
|
|
2770
|
+
const output: RecordedStepProgramOutput = {
|
|
2771
2771
|
field: `${recorder.parentField}.${stepId}`,
|
|
2772
2772
|
columnName: stepProgramColumnName(recorder.parentField, stepId),
|
|
2773
2773
|
stepId,
|
|
2774
2774
|
value,
|
|
2775
|
-
status: resolution.status,
|
|
2776
|
-
}
|
|
2775
|
+
...(resolution.status ? { status: resolution.status } : {}),
|
|
2776
|
+
};
|
|
2777
|
+
recorder.outputs.push(output);
|
|
2778
|
+
await recorder.onOutput?.(output);
|
|
2777
2779
|
}
|
|
2778
2780
|
}
|
|
2779
2781
|
if (typeof program.returnResolver === 'function') {
|
|
@@ -3589,7 +3591,7 @@ async function persistCompletedMapRows(input: {
|
|
|
3589
3591
|
];
|
|
3590
3592
|
const sessionScope = runtimeSheetSessionScope(input.req);
|
|
3591
3593
|
const rows = input.rows.map((row) => publicCsvStorageRow(row));
|
|
3592
|
-
await harnessPersistCompletedSheetRows({
|
|
3594
|
+
const result = await harnessPersistCompletedSheetRows({
|
|
3593
3595
|
...sessionScope,
|
|
3594
3596
|
tableNamespace: input.tableNamespace,
|
|
3595
3597
|
sheetContract: augmentSheetContractWithDatasetFields({
|
|
@@ -3600,6 +3602,46 @@ async function persistCompletedMapRows(input: {
|
|
|
3600
3602
|
rows,
|
|
3601
3603
|
outputFields,
|
|
3602
3604
|
});
|
|
3605
|
+
if (result.rowsWritten !== rows.length) {
|
|
3606
|
+
throw new Error(
|
|
3607
|
+
`Runtime sheet persistence mismatch for ctx.dataset("${input.tableNamespace}"): attempted to persist ${rows.length} row(s), but ${result.rowsWritten} row(s) were updated for run ${input.req.runId}.`,
|
|
3608
|
+
);
|
|
3609
|
+
}
|
|
3610
|
+
}
|
|
3611
|
+
|
|
3612
|
+
async function applyLiveMapRowUpdates(input: {
|
|
3613
|
+
req: RunRequest;
|
|
3614
|
+
tableNamespace: string;
|
|
3615
|
+
updates: Array<Omit<PlayRowUpdate, 'rowId'> & { runId?: string }>;
|
|
3616
|
+
outputFields: string[];
|
|
3617
|
+
extraOutputFields?: string[];
|
|
3618
|
+
}): Promise<void> {
|
|
3619
|
+
if (input.updates.length === 0) return;
|
|
3620
|
+
const outputFields = [
|
|
3621
|
+
...input.outputFields,
|
|
3622
|
+
...(input.extraOutputFields ?? []).filter(
|
|
3623
|
+
(field) => !input.outputFields.includes(field),
|
|
3624
|
+
),
|
|
3625
|
+
];
|
|
3626
|
+
const rows = input.updates.map((update) => update.dataPatch ?? {});
|
|
3627
|
+
await postRuntimeApi<{ ok: true }>(
|
|
3628
|
+
input.req.baseUrl,
|
|
3629
|
+
input.req.executorToken,
|
|
3630
|
+
{
|
|
3631
|
+
action: 'apply_row_updates',
|
|
3632
|
+
playName: input.req.playName,
|
|
3633
|
+
tableNamespace: input.tableNamespace,
|
|
3634
|
+
sheetContract: augmentSheetContractWithDatasetFields({
|
|
3635
|
+
contract: requireSheetContract(input.req, input.tableNamespace),
|
|
3636
|
+
rows,
|
|
3637
|
+
outputFields,
|
|
3638
|
+
}),
|
|
3639
|
+
contractSnapshot: input.req.contractSnapshot,
|
|
3640
|
+
runId: input.req.runId,
|
|
3641
|
+
userEmail: input.req.userEmail,
|
|
3642
|
+
updates: input.updates,
|
|
3643
|
+
},
|
|
3644
|
+
);
|
|
3603
3645
|
}
|
|
3604
3646
|
|
|
3605
3647
|
async function prepareMapRows(input: {
|
|
@@ -4551,6 +4593,9 @@ function createMinimalWorkerCtx(
|
|
|
4551
4593
|
false,
|
|
4552
4594
|
);
|
|
4553
4595
|
const generatedOutputFields = new Set<string>();
|
|
4596
|
+
const pendingLiveRowUpdates: Array<
|
|
4597
|
+
Omit<PlayRowUpdate, 'rowId'> & { runId?: string }
|
|
4598
|
+
> = [];
|
|
4554
4599
|
const persistedExecutedIndexes = new Set<number>();
|
|
4555
4600
|
const persistedFailedIndexes = new Set<number>();
|
|
4556
4601
|
let pendingPersistRows = 0;
|
|
@@ -4558,6 +4603,9 @@ function createMinimalWorkerCtx(
|
|
|
4558
4603
|
let scheduledPersistTimer: ReturnType<typeof setTimeout> | null = null;
|
|
4559
4604
|
let persistFlushChain: Promise<void> = Promise.resolve();
|
|
4560
4605
|
let persistFailure: unknown = null;
|
|
4606
|
+
let scheduledLiveUpdateTimer: ReturnType<typeof setTimeout> | null = null;
|
|
4607
|
+
let liveUpdateFlushChain: Promise<void> = Promise.resolve();
|
|
4608
|
+
let liveUpdateFailure: unknown = null;
|
|
4561
4609
|
|
|
4562
4610
|
const clearScheduledPersistTimer = () => {
|
|
4563
4611
|
if (scheduledPersistTimer) {
|
|
@@ -4566,6 +4614,13 @@ function createMinimalWorkerCtx(
|
|
|
4566
4614
|
}
|
|
4567
4615
|
};
|
|
4568
4616
|
|
|
4617
|
+
const clearScheduledLiveUpdateTimer = () => {
|
|
4618
|
+
if (scheduledLiveUpdateTimer) {
|
|
4619
|
+
clearTimeout(scheduledLiveUpdateTimer);
|
|
4620
|
+
scheduledLiveUpdateTimer = null;
|
|
4621
|
+
}
|
|
4622
|
+
};
|
|
4623
|
+
|
|
4569
4624
|
const persistExecutedRows = async () => {
|
|
4570
4625
|
const rowsToPersist = executedRows
|
|
4571
4626
|
.map((row, executedIndex) =>
|
|
@@ -4659,6 +4714,60 @@ function createMinimalWorkerCtx(
|
|
|
4659
4714
|
return task;
|
|
4660
4715
|
};
|
|
4661
4716
|
|
|
4717
|
+
const flushLiveRowUpdates = (): Promise<void> => {
|
|
4718
|
+
clearScheduledLiveUpdateTimer();
|
|
4719
|
+
if (pendingLiveRowUpdates.length === 0) {
|
|
4720
|
+
return liveUpdateFlushChain;
|
|
4721
|
+
}
|
|
4722
|
+
const updates = pendingLiveRowUpdates.splice(0);
|
|
4723
|
+
const extraOutputFields = Array.from(generatedOutputFields);
|
|
4724
|
+
const task = liveUpdateFlushChain.then(async () => {
|
|
4725
|
+
if (liveUpdateFailure) throw liveUpdateFailure;
|
|
4726
|
+
await applyLiveMapRowUpdates({
|
|
4727
|
+
req,
|
|
4728
|
+
tableNamespace: name,
|
|
4729
|
+
outputFields,
|
|
4730
|
+
extraOutputFields,
|
|
4731
|
+
updates,
|
|
4732
|
+
});
|
|
4733
|
+
});
|
|
4734
|
+
liveUpdateFlushChain = task.catch((error) => {
|
|
4735
|
+
liveUpdateFailure ??= error;
|
|
4736
|
+
});
|
|
4737
|
+
return task;
|
|
4738
|
+
};
|
|
4739
|
+
|
|
4740
|
+
const scheduleLiveRowUpdates = () => {
|
|
4741
|
+
if (liveUpdateFailure) return;
|
|
4742
|
+
if (
|
|
4743
|
+
pendingLiveRowUpdates.length >= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS
|
|
4744
|
+
) {
|
|
4745
|
+
void flushLiveRowUpdates().catch(() => undefined);
|
|
4746
|
+
return;
|
|
4747
|
+
}
|
|
4748
|
+
if (scheduledLiveUpdateTimer) return;
|
|
4749
|
+
scheduledLiveUpdateTimer = setTimeout(() => {
|
|
4750
|
+
scheduledLiveUpdateTimer = null;
|
|
4751
|
+
void flushLiveRowUpdates().catch(() => undefined);
|
|
4752
|
+
}, MAP_INCREMENTAL_PERSIST_INTERVAL_MS);
|
|
4753
|
+
};
|
|
4754
|
+
|
|
4755
|
+
const enqueueLiveRowUpdate = (
|
|
4756
|
+
update: Omit<PlayRowUpdate, 'rowId'> & { runId?: string },
|
|
4757
|
+
): Promise<void> => {
|
|
4758
|
+
if (liveUpdateFailure) {
|
|
4759
|
+
return Promise.reject(liveUpdateFailure);
|
|
4760
|
+
}
|
|
4761
|
+
pendingLiveRowUpdates.push(update);
|
|
4762
|
+
if (
|
|
4763
|
+
pendingLiveRowUpdates.length >= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS
|
|
4764
|
+
) {
|
|
4765
|
+
return flushLiveRowUpdates();
|
|
4766
|
+
}
|
|
4767
|
+
scheduleLiveRowUpdates();
|
|
4768
|
+
return Promise.resolve();
|
|
4769
|
+
};
|
|
4770
|
+
|
|
4662
4771
|
const schedulePersistExecutedRows = () => {
|
|
4663
4772
|
if (persistFailure) return;
|
|
4664
4773
|
if (
|
|
@@ -4769,6 +4878,26 @@ function createMinimalWorkerCtx(
|
|
|
4769
4878
|
parentField: key,
|
|
4770
4879
|
path: [],
|
|
4771
4880
|
outputs: stepProgramOutputs,
|
|
4881
|
+
onOutput: async (stepOutput) => {
|
|
4882
|
+
generatedOutputFields.add(stepOutput.columnName);
|
|
4883
|
+
const status = stepOutput.status ?? 'completed';
|
|
4884
|
+
await enqueueLiveRowUpdate({
|
|
4885
|
+
key: entry.rowKey,
|
|
4886
|
+
tableNamespace: name,
|
|
4887
|
+
runId: req.runId,
|
|
4888
|
+
status: 'running',
|
|
4889
|
+
stage: stepOutput.stepId,
|
|
4890
|
+
dataPatch: {
|
|
4891
|
+
[stepOutput.columnName]: stepOutput.value,
|
|
4892
|
+
},
|
|
4893
|
+
cellMetaPatch: {
|
|
4894
|
+
[stepOutput.columnName]: {
|
|
4895
|
+
status,
|
|
4896
|
+
stage: stepOutput.stepId,
|
|
4897
|
+
},
|
|
4898
|
+
},
|
|
4899
|
+
});
|
|
4900
|
+
},
|
|
4772
4901
|
}
|
|
4773
4902
|
: undefined,
|
|
4774
4903
|
);
|
|
@@ -4926,6 +5055,9 @@ function createMinimalWorkerCtx(
|
|
|
4926
5055
|
},
|
|
4927
5056
|
});
|
|
4928
5057
|
try {
|
|
5058
|
+
await flushLiveRowUpdates();
|
|
5059
|
+
await liveUpdateFlushChain;
|
|
5060
|
+
if (liveUpdateFailure) throw liveUpdateFailure;
|
|
4929
5061
|
await enqueuePersistExecutedRows();
|
|
4930
5062
|
await persistFlushChain;
|
|
4931
5063
|
if (persistFailure) throw persistFailure;
|
|
@@ -2,8 +2,10 @@ import {
|
|
|
2
2
|
chooseMapChunkSize,
|
|
3
3
|
type ExecutionPlan,
|
|
4
4
|
} from '../../../../shared_libs/play-runtime/execution-plan';
|
|
5
|
+
import { TOOL_CALLING_MAP_CHUNK_SIZE } from '../../../../shared_libs/play-runtime/map-chunk-limits';
|
|
5
6
|
|
|
6
7
|
export const CACHE_ENABLED_SIMPLE_MAP_CHUNK_SIZE = 10_000;
|
|
8
|
+
export { TOOL_CALLING_MAP_CHUNK_SIZE };
|
|
7
9
|
|
|
8
10
|
export type WorkerMapChunkPlanInput = {
|
|
9
11
|
mapName: string;
|
|
@@ -31,7 +33,7 @@ export function chooseWorkerMapRowsPerChunk(
|
|
|
31
33
|
const toolFreeSimpleMap =
|
|
32
34
|
!!planMap &&
|
|
33
35
|
planMap.stepsPerChunk === 1 &&
|
|
34
|
-
(plan
|
|
36
|
+
!mapDoesExternalWork(planMap, plan);
|
|
35
37
|
if (
|
|
36
38
|
toolFreeSimpleMap &&
|
|
37
39
|
(input.rowCountHint === null ||
|
|
@@ -40,5 +42,55 @@ export function chooseWorkerMapRowsPerChunk(
|
|
|
40
42
|
return Math.max(rowsPerChunk, CACHE_ENABLED_SIMPLE_MAP_CHUNK_SIZE);
|
|
41
43
|
}
|
|
42
44
|
|
|
45
|
+
if (
|
|
46
|
+
mapDoesExternalWork(planMap, plan) &&
|
|
47
|
+
input.rowCountHint !== null &&
|
|
48
|
+
input.rowCountHint <= rowsPerChunk
|
|
49
|
+
) {
|
|
50
|
+
return Math.min(rowsPerChunk, TOOL_CALLING_MAP_CHUNK_SIZE);
|
|
51
|
+
}
|
|
52
|
+
|
|
43
53
|
return rowsPerChunk;
|
|
44
54
|
}
|
|
55
|
+
|
|
56
|
+
function mapDoesExternalWork(
|
|
57
|
+
planMap: ExecutionPlan['maps'][number] | undefined,
|
|
58
|
+
plan: ExecutionPlan | null,
|
|
59
|
+
): boolean {
|
|
60
|
+
if (Array.isArray(planMap?.externalStepFields)) {
|
|
61
|
+
return planMap.externalStepFields.length > 0;
|
|
62
|
+
}
|
|
63
|
+
if (!plan?.toolDeclarations.length) {
|
|
64
|
+
return false;
|
|
65
|
+
}
|
|
66
|
+
if (!planMap) {
|
|
67
|
+
return true;
|
|
68
|
+
}
|
|
69
|
+
const outputFields = new Set(planMap.outputFields);
|
|
70
|
+
const mapFields = new Set([
|
|
71
|
+
planMap.mapName,
|
|
72
|
+
planMap.tableNamespace,
|
|
73
|
+
...planMap.outputFields,
|
|
74
|
+
...(planMap.stepFields ?? []),
|
|
75
|
+
...(planMap.externalStepFields ?? []),
|
|
76
|
+
...planMap.waterfallStages.flatMap((stage) => [
|
|
77
|
+
stage.waterfallId,
|
|
78
|
+
...stage.stageIds,
|
|
79
|
+
]),
|
|
80
|
+
]);
|
|
81
|
+
return plan.toolDeclarations.some((tool) => {
|
|
82
|
+
const field = tool.field?.trim();
|
|
83
|
+
if (!field) {
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
if (mapFields.has(field)) {
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
if (!field.includes('.')) {
|
|
90
|
+
return false;
|
|
91
|
+
}
|
|
92
|
+
const firstSegment = field.slice(0, field.indexOf('.'));
|
|
93
|
+
const lastSegment = field.slice(field.lastIndexOf('.') + 1);
|
|
94
|
+
return outputFields.has(firstSegment) || outputFields.has(lastSegment);
|
|
95
|
+
});
|
|
96
|
+
}
|
|
@@ -281,12 +281,14 @@ export type { PreviousCell } from '../../shared_libs/play-runtime/cell-staleness
|
|
|
281
281
|
* Keyword-style request object for `ctx.tools.execute(...)`.
|
|
282
282
|
*
|
|
283
283
|
* The `tool` value comes from live tool discovery. The `id` is the stable
|
|
284
|
-
* logical call name
|
|
284
|
+
* logical call name used for logs, metadata, and receipt attachment. Provider
|
|
285
|
+
* call reuse is keyed by play, tool, semantic input, auth scope, provider action
|
|
286
|
+
* version, and cache policy.
|
|
285
287
|
*
|
|
286
288
|
* @sdkReference runtime 160
|
|
287
289
|
*/
|
|
288
290
|
export type ToolExecutionRequest = {
|
|
289
|
-
/** Stable logical id for
|
|
291
|
+
/** Stable logical id for logs, metadata, and receipt attachment. */
|
|
290
292
|
id: string;
|
|
291
293
|
/** Current tool id from `deepline tools search` / `deepline tools describe`. */
|
|
292
294
|
tool: string;
|
|
@@ -104,10 +104,10 @@ export const SDK_RELEASE = {
|
|
|
104
104
|
// 0.1.111 ships dataset-native tool list getters and result row datasets.
|
|
105
105
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
106
106
|
// fields shipped in 0.1.153.
|
|
107
|
-
version: '0.1.
|
|
107
|
+
version: '0.1.157',
|
|
108
108
|
apiContract: '2026-06-dataset-handle-results-hard-cutover',
|
|
109
109
|
supportPolicy: {
|
|
110
|
-
latest: '0.1.
|
|
110
|
+
latest: '0.1.157',
|
|
111
111
|
minimumSupported: '0.1.53',
|
|
112
112
|
deprecatedBelow: '0.1.53',
|
|
113
113
|
commandMinimumSupported: [
|
|
@@ -135,7 +135,7 @@ export function augmentSheetContractWithDatasetFields(input: {
|
|
|
135
135
|
}
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
-
const
|
|
138
|
+
const existingDatasetPayloadFields = new Set<string>();
|
|
139
139
|
const existingSqlNames = new Set<string>();
|
|
140
140
|
const inputColumns: PlaySheetContract['columns'] = [];
|
|
141
141
|
const outputColumns: PlaySheetContract['columns'] = [];
|
|
@@ -145,15 +145,19 @@ export function augmentSheetContractWithDatasetFields(input: {
|
|
|
145
145
|
) => {
|
|
146
146
|
const field = typeof column.field === 'string' ? column.field : column.id;
|
|
147
147
|
const sqlName = column.sqlName.trim();
|
|
148
|
+
const isDatasetPayloadColumn =
|
|
149
|
+
column.source === 'input' || column.source === 'datasetColumn';
|
|
148
150
|
if (
|
|
149
151
|
!field ||
|
|
150
152
|
!sqlName ||
|
|
151
|
-
|
|
153
|
+
(isDatasetPayloadColumn && existingDatasetPayloadFields.has(field)) ||
|
|
152
154
|
existingSqlNames.has(sqlName)
|
|
153
155
|
) {
|
|
154
156
|
return;
|
|
155
157
|
}
|
|
156
|
-
|
|
158
|
+
if (isDatasetPayloadColumn) {
|
|
159
|
+
existingDatasetPayloadFields.add(field);
|
|
160
|
+
}
|
|
157
161
|
existingSqlNames.add(sqlName);
|
|
158
162
|
target.push(column);
|
|
159
163
|
};
|
|
@@ -175,7 +179,7 @@ export function augmentSheetContractWithDatasetFields(input: {
|
|
|
175
179
|
}
|
|
176
180
|
|
|
177
181
|
for (const field of candidateFields) {
|
|
178
|
-
if (
|
|
182
|
+
if (existingDatasetPayloadFields.has(field)) continue;
|
|
179
183
|
const sqlName = sqlSafePlayColumnName(field);
|
|
180
184
|
if (existingSqlNames.has(sqlName)) continue;
|
|
181
185
|
appendColumn(outputFields.has(field) ? outputColumns : inputColumns, {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
flattenStaticPipeline,
|
|
3
|
+
flattenStaticSubsteps,
|
|
3
4
|
type PlayStaticPipeline,
|
|
4
5
|
type PlayStaticSubstep,
|
|
5
6
|
} from '../plays/static-pipeline';
|
|
@@ -28,6 +29,8 @@ export type ExecutionPlanMap = {
|
|
|
28
29
|
mapName: string;
|
|
29
30
|
tableNamespace: string;
|
|
30
31
|
outputFields: string[];
|
|
32
|
+
stepFields: string[];
|
|
33
|
+
externalStepFields: string[];
|
|
31
34
|
waterfallStages: Array<{
|
|
32
35
|
waterfallId: string;
|
|
33
36
|
stageIds: string[];
|
|
@@ -198,52 +201,166 @@ function extractPlanMaps(
|
|
|
198
201
|
(substep): substep is Extract<PlayStaticSubstep, { type: 'step_suite' }> =>
|
|
199
202
|
substep.type === 'step_suite',
|
|
200
203
|
);
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
204
|
+
const datasetSubsteps = substeps.filter(
|
|
205
|
+
(substep): substep is Extract<PlayStaticSubstep, { type: 'dataset' }> =>
|
|
206
|
+
substep.type === 'dataset',
|
|
207
|
+
);
|
|
208
|
+
const hasSiblingMaps = datasetSubsteps.length > 1;
|
|
209
|
+
return datasetSubsteps.map((mapSubstep) => {
|
|
210
|
+
const waterfalls = fallbackWaterfalls.filter((waterfall) => {
|
|
211
|
+
if (!mapSubstep.waterfallIds?.length) {
|
|
209
212
|
return (
|
|
210
|
-
|
|
211
|
-
|
|
213
|
+
!hasSiblingMaps ||
|
|
214
|
+
substepFieldBelongsToMap(waterfall.field, mapSubstep)
|
|
212
215
|
);
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
}));
|
|
218
|
-
const stepSuites = fallbackStepSuites.filter((stepSuite) => {
|
|
219
|
-
if (!mapSubstep.waterfallIds?.length) return true;
|
|
220
|
-
return mapSubstep.waterfallIds.includes(stepSuite.field);
|
|
221
|
-
});
|
|
222
|
-
const stepSuiteStepsPerChunk = stepSuites.reduce(
|
|
223
|
-
(max, stepSuite) => Math.max(max, stepSuite.steps.length),
|
|
224
|
-
0,
|
|
225
|
-
);
|
|
226
|
-
const waterfallStepsPerChunk = waterfallStages.reduce(
|
|
227
|
-
(max, waterfall) => Math.max(max, waterfall.stageIds.length),
|
|
228
|
-
0,
|
|
229
|
-
);
|
|
230
|
-
const stepsPerChunk = Math.max(
|
|
231
|
-
1,
|
|
232
|
-
waterfallStepsPerChunk,
|
|
233
|
-
stepSuiteStepsPerChunk,
|
|
216
|
+
}
|
|
217
|
+
return (
|
|
218
|
+
(waterfall.id && mapSubstep.waterfallIds.includes(waterfall.id)) ||
|
|
219
|
+
mapSubstep.waterfallIds.includes(waterfall.field)
|
|
234
220
|
);
|
|
235
|
-
return {
|
|
236
|
-
mapName: mapSubstep.name ?? mapSubstep.field,
|
|
237
|
-
tableNamespace: mapSubstep.tableNamespace ?? mapSubstep.field,
|
|
238
|
-
outputFields: mapSubstep.outputFields ?? [],
|
|
239
|
-
waterfallStages,
|
|
240
|
-
defaultChunkSize:
|
|
241
|
-
stepsPerChunk > 1
|
|
242
|
-
? EXECUTION_PLAN_DEFAULTS.complexMapChunkSize
|
|
243
|
-
: EXECUTION_PLAN_DEFAULTS.largeMapChunkSize,
|
|
244
|
-
stepsPerChunk,
|
|
245
|
-
};
|
|
246
221
|
});
|
|
222
|
+
const waterfallStages = waterfalls.map((waterfall) => ({
|
|
223
|
+
waterfallId: waterfall.id ?? waterfall.field,
|
|
224
|
+
stageIds: waterfall.steps?.map((step) => step.id) ?? [],
|
|
225
|
+
}));
|
|
226
|
+
const stepSuites = fallbackStepSuites.filter((stepSuite) => {
|
|
227
|
+
if (!mapSubstep.waterfallIds?.length) {
|
|
228
|
+
return (
|
|
229
|
+
!hasSiblingMaps ||
|
|
230
|
+
substepFieldBelongsToMap(stepSuite.field, mapSubstep)
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
return mapSubstep.waterfallIds.includes(stepSuite.field);
|
|
234
|
+
});
|
|
235
|
+
const looseMapSubsteps = substeps.filter((substep) => {
|
|
236
|
+
if (!isLooseMapExecutionSubstep(substep)) {
|
|
237
|
+
return false;
|
|
238
|
+
}
|
|
239
|
+
return substepFieldBelongsToMap(substep.field, mapSubstep);
|
|
240
|
+
});
|
|
241
|
+
const stepSuiteStepsPerChunk = stepSuites.reduce(
|
|
242
|
+
(max, stepSuite) => Math.max(max, stepSuite.steps.length),
|
|
243
|
+
0,
|
|
244
|
+
);
|
|
245
|
+
const waterfallStepsPerChunk = waterfallStages.reduce(
|
|
246
|
+
(max, waterfall) => Math.max(max, waterfall.stageIds.length),
|
|
247
|
+
0,
|
|
248
|
+
);
|
|
249
|
+
const directMapStepsPerChunk = maxExecutionSubstepsPerChunk(
|
|
250
|
+
mapSubstep.steps ?? [],
|
|
251
|
+
);
|
|
252
|
+
const stepsPerChunk = Math.max(
|
|
253
|
+
1,
|
|
254
|
+
directMapStepsPerChunk,
|
|
255
|
+
looseMapSubsteps.length,
|
|
256
|
+
waterfallStepsPerChunk,
|
|
257
|
+
stepSuiteStepsPerChunk,
|
|
258
|
+
);
|
|
259
|
+
const mapExecutionSubsteps = [
|
|
260
|
+
...(mapSubstep.steps ?? []),
|
|
261
|
+
...looseMapSubsteps,
|
|
262
|
+
...stepSuites,
|
|
263
|
+
...waterfalls,
|
|
264
|
+
];
|
|
265
|
+
return {
|
|
266
|
+
mapName: mapSubstep.name ?? mapSubstep.field,
|
|
267
|
+
tableNamespace: mapSubstep.tableNamespace ?? mapSubstep.field,
|
|
268
|
+
outputFields: mapSubstep.outputFields ?? [],
|
|
269
|
+
stepFields: collectPlanMapStepFields(mapExecutionSubsteps),
|
|
270
|
+
externalStepFields:
|
|
271
|
+
collectPlanMapExternalStepFields(mapExecutionSubsteps),
|
|
272
|
+
waterfallStages,
|
|
273
|
+
defaultChunkSize:
|
|
274
|
+
stepsPerChunk > 1
|
|
275
|
+
? EXECUTION_PLAN_DEFAULTS.complexMapChunkSize
|
|
276
|
+
: EXECUTION_PLAN_DEFAULTS.largeMapChunkSize,
|
|
277
|
+
stepsPerChunk,
|
|
278
|
+
};
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function maxExecutionSubstepsPerChunk(substeps: PlayStaticSubstep[]): number {
|
|
283
|
+
return substeps.reduce((max, substep) => {
|
|
284
|
+
if (substep.type === 'step_suite') {
|
|
285
|
+
return Math.max(max, substep.steps.length);
|
|
286
|
+
}
|
|
287
|
+
if (substep.type === 'waterfall') {
|
|
288
|
+
return Math.max(max, substep.steps?.length ?? 0);
|
|
289
|
+
}
|
|
290
|
+
if (substep.type === 'control_flow') {
|
|
291
|
+
return Math.max(max, maxExecutionSubstepsPerChunk(substep.steps));
|
|
292
|
+
}
|
|
293
|
+
return Math.max(max, 1);
|
|
294
|
+
}, 0);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function isLooseMapExecutionSubstep(
|
|
298
|
+
substep: PlayStaticSubstep,
|
|
299
|
+
): substep is Extract<
|
|
300
|
+
PlayStaticSubstep,
|
|
301
|
+
{ type: 'tool' | 'play_call' | 'control_flow' | 'code' }
|
|
302
|
+
> {
|
|
303
|
+
return (
|
|
304
|
+
substep.type === 'tool' ||
|
|
305
|
+
substep.type === 'play_call' ||
|
|
306
|
+
substep.type === 'control_flow' ||
|
|
307
|
+
substep.type === 'code'
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function substepFieldBelongsToMap(
|
|
312
|
+
substepField: string,
|
|
313
|
+
mapSubstep: Extract<PlayStaticSubstep, { type: 'dataset' }>,
|
|
314
|
+
): boolean {
|
|
315
|
+
const field = substepField.trim();
|
|
316
|
+
if (!field) {
|
|
317
|
+
return false;
|
|
318
|
+
}
|
|
319
|
+
const mapFields = [
|
|
320
|
+
mapSubstep.field,
|
|
321
|
+
mapSubstep.name,
|
|
322
|
+
mapSubstep.tableNamespace,
|
|
323
|
+
...(mapSubstep.outputFields ?? []),
|
|
324
|
+
].filter((candidate): candidate is string => Boolean(candidate?.trim()));
|
|
325
|
+
return mapFields.some((mapField) => {
|
|
326
|
+
const normalized = mapField.trim();
|
|
327
|
+
return field === normalized || field.startsWith(`${normalized}.`);
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
function collectPlanMapStepFields(substeps: PlayStaticSubstep[]): string[] {
|
|
332
|
+
const fields = new Set<string>();
|
|
333
|
+
for (const substep of flattenStaticSubsteps(substeps)) {
|
|
334
|
+
if ('field' in substep && typeof substep.field === 'string') {
|
|
335
|
+
const field = substep.field.trim();
|
|
336
|
+
if (field) {
|
|
337
|
+
fields.add(field);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
return [...fields];
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
function collectPlanMapExternalStepFields(
|
|
345
|
+
substeps: PlayStaticSubstep[],
|
|
346
|
+
): string[] {
|
|
347
|
+
const fields = new Set<string>();
|
|
348
|
+
for (const substep of flattenStaticSubsteps(substeps)) {
|
|
349
|
+
if (
|
|
350
|
+
substep.type !== 'tool' &&
|
|
351
|
+
substep.type !== 'play_call' &&
|
|
352
|
+
substep.type !== 'waterfall'
|
|
353
|
+
) {
|
|
354
|
+
continue;
|
|
355
|
+
}
|
|
356
|
+
if ('field' in substep && typeof substep.field === 'string') {
|
|
357
|
+
const field = substep.field.trim();
|
|
358
|
+
if (field) {
|
|
359
|
+
fields.add(field);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
return [...fields];
|
|
247
364
|
}
|
|
248
365
|
|
|
249
366
|
function extractToolDeclarations(
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export const TOOL_CALLING_MAP_CHUNK_SIZE = 25;
|
|
@@ -2425,7 +2425,6 @@ async function markRuntimeRowsPendingForRecompute(
|
|
|
2425
2425
|
_version = ${nextRuntimeSheetVersionExpression(session)}
|
|
2426
2426
|
FROM target_rows
|
|
2427
2427
|
WHERE target._key = target_rows._key
|
|
2428
|
-
AND (target._run_id IS NULL OR target._run_id <= $2::text)
|
|
2429
2428
|
AND (
|
|
2430
2429
|
target._status <> 'pending'
|
|
2431
2430
|
OR target._run_id IS DISTINCT FROM $2::text
|