deepline 0.1.77 → 0.1.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -8
- package/dist/cli/index.js +525 -355
- package/dist/cli/index.mjs +538 -368
- package/dist/index.d.mts +31 -2
- package/dist/index.d.ts +31 -2
- package/dist/index.js +3 -2
- package/dist/index.mjs +3 -2
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +273 -83
- package/dist/repo/apps/play-runner-workers/src/dedup-do.ts +18 -3
- package/dist/repo/apps/play-runner-workers/src/workflow-retry-state.ts +203 -0
- package/dist/repo/sdk/src/client.ts +7 -0
- package/dist/repo/sdk/src/play.ts +1 -1
- package/dist/repo/sdk/src/release.ts +2 -2
- package/dist/repo/sdk/src/types.ts +4 -0
- package/dist/repo/shared_libs/plays/dataset.ts +3 -1
- package/dist/repo/shared_libs/plays/static-pipeline.ts +261 -1
- package/package.json +1 -1
|
@@ -113,6 +113,8 @@ type WorkflowRunMapping = {
|
|
|
113
113
|
type WorkflowRunRetryState = {
|
|
114
114
|
runId: string;
|
|
115
115
|
params: unknown;
|
|
116
|
+
paramsRef?: unknown;
|
|
117
|
+
paramsBytes?: number;
|
|
116
118
|
retryAttempts: number;
|
|
117
119
|
updatedAt: number;
|
|
118
120
|
expiresAt: number;
|
|
@@ -1059,8 +1061,10 @@ export class PlayDedup implements DurableObject {
|
|
|
1059
1061
|
ttlMs?: unknown;
|
|
1060
1062
|
} | null;
|
|
1061
1063
|
const runId = typeof body?.runId === 'string' ? body.runId : '';
|
|
1062
|
-
if (!runId || !body || !('params' in body)) {
|
|
1063
|
-
return new Response('runId and params are required', {
|
|
1064
|
+
if (!runId || !body || (!('params' in body) && !('paramsRef' in body))) {
|
|
1065
|
+
return new Response('runId and params or paramsRef are required', {
|
|
1066
|
+
status: 400,
|
|
1067
|
+
});
|
|
1064
1068
|
}
|
|
1065
1069
|
const now = Date.now();
|
|
1066
1070
|
const ttlMs =
|
|
@@ -1075,7 +1079,14 @@ export class PlayDedup implements DurableObject {
|
|
|
1075
1079
|
const existing = await this.state.storage.get<WorkflowRunRetryState>(key);
|
|
1076
1080
|
const retryState = {
|
|
1077
1081
|
runId,
|
|
1078
|
-
params: body.params,
|
|
1082
|
+
params: 'params' in body ? body.params : null,
|
|
1083
|
+
paramsRef: 'paramsRef' in body ? body.paramsRef : null,
|
|
1084
|
+
paramsBytes:
|
|
1085
|
+
typeof (body as { paramsBytes?: unknown }).paramsBytes ===
|
|
1086
|
+
'number' &&
|
|
1087
|
+
Number.isFinite((body as { paramsBytes?: number }).paramsBytes)
|
|
1088
|
+
? (body as { paramsBytes: number }).paramsBytes
|
|
1089
|
+
: undefined,
|
|
1079
1090
|
retryAttempts:
|
|
1080
1091
|
existing?.runId === runId &&
|
|
1081
1092
|
typeof existing.retryAttempts === 'number'
|
|
@@ -1128,6 +1139,8 @@ export class PlayDedup implements DurableObject {
|
|
|
1128
1139
|
claimed: false,
|
|
1129
1140
|
attempts: existing.retryAttempts,
|
|
1130
1141
|
params: existing.params,
|
|
1142
|
+
paramsRef: existing.paramsRef ?? null,
|
|
1143
|
+
paramsBytes: existing.paramsBytes ?? null,
|
|
1131
1144
|
};
|
|
1132
1145
|
return;
|
|
1133
1146
|
}
|
|
@@ -1142,6 +1155,8 @@ export class PlayDedup implements DurableObject {
|
|
|
1142
1155
|
claimed: true,
|
|
1143
1156
|
attempts: nextAttempts,
|
|
1144
1157
|
params: existing.params,
|
|
1158
|
+
paramsRef: existing.paramsRef ?? null,
|
|
1159
|
+
paramsBytes: existing.paramsBytes ?? null,
|
|
1145
1160
|
};
|
|
1146
1161
|
});
|
|
1147
1162
|
return new Response(JSON.stringify(response), {
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
|
|
2
|
+
import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
3
|
+
import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
|
|
4
|
+
import type {
|
|
5
|
+
PlayRuntimeManifest,
|
|
6
|
+
PlayRuntimeManifestMap,
|
|
7
|
+
} from '../../../shared_libs/plays/compiler-manifest';
|
|
8
|
+
|
|
9
|
+
export const WORKFLOW_RETRY_STATE_TARGET_BYTES = 100_000;
|
|
10
|
+
export const WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES =
|
|
11
|
+
WORKFLOW_RETRY_STATE_TARGET_BYTES;
|
|
12
|
+
export const WORKFLOW_RETRY_PARAMS_MAX_BYTES = 1024 * 1024;
|
|
13
|
+
|
|
14
|
+
export type WorkflowRetryParamsRef = {
|
|
15
|
+
storageKind: 'r2';
|
|
16
|
+
storageKey: string;
|
|
17
|
+
bytes: number;
|
|
18
|
+
hash: string;
|
|
19
|
+
expiresAt: number;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export type WorkflowRetryPlayParams = {
|
|
23
|
+
runId: string;
|
|
24
|
+
playId: string;
|
|
25
|
+
playName: string;
|
|
26
|
+
artifactStorageKey: string;
|
|
27
|
+
artifactHash: string;
|
|
28
|
+
graphHash: string;
|
|
29
|
+
input: Record<string, unknown>;
|
|
30
|
+
inputFile?: {
|
|
31
|
+
name?: string;
|
|
32
|
+
r2Key?: string;
|
|
33
|
+
storageKey?: string;
|
|
34
|
+
path?: string;
|
|
35
|
+
fileName?: string;
|
|
36
|
+
logicalPath?: string;
|
|
37
|
+
contentType?: string;
|
|
38
|
+
bytes?: number;
|
|
39
|
+
} | null;
|
|
40
|
+
inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
|
|
41
|
+
packagedFiles?: Array<{
|
|
42
|
+
playPath: string;
|
|
43
|
+
storageKey: string;
|
|
44
|
+
contentType?: string;
|
|
45
|
+
bytes?: number;
|
|
46
|
+
inlineText?: string;
|
|
47
|
+
}> | null;
|
|
48
|
+
contractSnapshot?: unknown;
|
|
49
|
+
executionPlan?: ExecutionPlan | null;
|
|
50
|
+
childPlayManifests?: PlayRuntimeManifestMap | null;
|
|
51
|
+
playCallGovernance?: PlayCallGovernanceSnapshot | null;
|
|
52
|
+
preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
|
|
53
|
+
preloadedDbSessionRef?: {
|
|
54
|
+
runId: string;
|
|
55
|
+
sessionCount: number;
|
|
56
|
+
expiresAt: number;
|
|
57
|
+
} | null;
|
|
58
|
+
dynamicWorkerCode?: string | null;
|
|
59
|
+
executorToken: string;
|
|
60
|
+
baseUrl: string;
|
|
61
|
+
orgId: string;
|
|
62
|
+
userEmail: string;
|
|
63
|
+
userId?: string | null;
|
|
64
|
+
runtimeBackend: string;
|
|
65
|
+
dedupBackend: string;
|
|
66
|
+
totalRows?: number;
|
|
67
|
+
coordinatorUrl?: string | null;
|
|
68
|
+
coordinatorInternalToken?: string | null;
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
export type WorkflowRetryStatePayload<TParams = WorkflowRetryPlayParams> =
|
|
72
|
+
| {
|
|
73
|
+
params: TParams;
|
|
74
|
+
paramsRef?: null;
|
|
75
|
+
paramsBytes: number;
|
|
76
|
+
}
|
|
77
|
+
| {
|
|
78
|
+
params: null;
|
|
79
|
+
paramsRef: WorkflowRetryParamsRef;
|
|
80
|
+
paramsBytes: number;
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
export function buildWorkflowRetryParams<
|
|
84
|
+
TParams extends WorkflowRetryPlayParams,
|
|
85
|
+
>(params: TParams): TParams {
|
|
86
|
+
const retryParams = {
|
|
87
|
+
...params,
|
|
88
|
+
dynamicWorkerCode: null,
|
|
89
|
+
contractSnapshot: stripRetrySourceSnapshot(params.contractSnapshot),
|
|
90
|
+
childPlayManifests: stripRetryChildManifestCode(params.childPlayManifests),
|
|
91
|
+
packagedFiles: stripRetryPackagedFiles(params.packagedFiles),
|
|
92
|
+
} satisfies WorkflowRetryPlayParams as TParams;
|
|
93
|
+
if (jsonByteLength(retryParams) <= WORKFLOW_RETRY_STATE_TARGET_BYTES) {
|
|
94
|
+
return retryParams;
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
...retryParams,
|
|
98
|
+
contractSnapshot: stripRetryContractSnapshotToArtifact(
|
|
99
|
+
retryParams.contractSnapshot,
|
|
100
|
+
params,
|
|
101
|
+
),
|
|
102
|
+
childPlayManifests: stripRetryChildManifestToArtifact(
|
|
103
|
+
retryParams.childPlayManifests,
|
|
104
|
+
),
|
|
105
|
+
} satisfies WorkflowRetryPlayParams as TParams;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export function jsonByteLength(value: unknown): number {
|
|
109
|
+
return new TextEncoder().encode(JSON.stringify(value)).length;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function workflowRetryParamsStorageKey(input: {
|
|
113
|
+
runId: string;
|
|
114
|
+
hash: string;
|
|
115
|
+
}): string {
|
|
116
|
+
const safeRunId = input.runId
|
|
117
|
+
.toLowerCase()
|
|
118
|
+
.replace(/[^a-z0-9_-]+/g, '-')
|
|
119
|
+
.slice(0, 140);
|
|
120
|
+
return `plays/workflow-retry-params/${safeRunId || 'run'}/${input.hash}.json`;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function stripRetryPackagedFiles(
|
|
124
|
+
files: WorkflowRetryPlayParams['packagedFiles'],
|
|
125
|
+
): WorkflowRetryPlayParams['packagedFiles'] {
|
|
126
|
+
return (
|
|
127
|
+
files?.map((file) => ({
|
|
128
|
+
playPath: file.playPath,
|
|
129
|
+
storageKey: file.storageKey,
|
|
130
|
+
contentType: file.contentType,
|
|
131
|
+
bytes: file.bytes,
|
|
132
|
+
})) ?? null
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function stripRetrySourceSnapshot(snapshot: unknown): unknown {
|
|
137
|
+
if (!isRecord(snapshot)) return snapshot;
|
|
138
|
+
const rest = { ...snapshot };
|
|
139
|
+
delete rest.sourceCode;
|
|
140
|
+
delete rest.sourceFiles;
|
|
141
|
+
delete rest.bundledCode;
|
|
142
|
+
return rest;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function stripRetryContractSnapshotToArtifact(
|
|
146
|
+
snapshot: unknown,
|
|
147
|
+
params: WorkflowRetryPlayParams,
|
|
148
|
+
): unknown {
|
|
149
|
+
if (!isRecord(snapshot)) return snapshot;
|
|
150
|
+
return {
|
|
151
|
+
source: snapshot.source ?? 'artifact',
|
|
152
|
+
revisionVersion: snapshot.revisionVersion ?? null,
|
|
153
|
+
staticPipeline: snapshot.staticPipeline ?? null,
|
|
154
|
+
billingLimit: snapshot.billingLimit ?? null,
|
|
155
|
+
artifactMetadata: {
|
|
156
|
+
storageKey: params.artifactStorageKey,
|
|
157
|
+
artifactHash: params.artifactHash,
|
|
158
|
+
graphHash: params.graphHash,
|
|
159
|
+
},
|
|
160
|
+
codeFormat: snapshot.codeFormat ?? null,
|
|
161
|
+
compatibility: snapshot.compatibility ?? null,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function stripRetryChildManifestCode(
|
|
166
|
+
manifests: PlayRuntimeManifestMap | null | undefined,
|
|
167
|
+
): PlayRuntimeManifestMap | null {
|
|
168
|
+
if (!manifests) return null;
|
|
169
|
+
const stripped: PlayRuntimeManifestMap = {};
|
|
170
|
+
for (const [key, manifest] of Object.entries(manifests)) {
|
|
171
|
+
const rest = { ...manifest };
|
|
172
|
+
delete rest.bundledCode;
|
|
173
|
+
delete rest.sourceCode;
|
|
174
|
+
delete (rest as Record<string, unknown>).sourceMap;
|
|
175
|
+
stripped[key] = rest;
|
|
176
|
+
}
|
|
177
|
+
return stripped;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function stripRetryChildManifestToArtifact(
|
|
181
|
+
manifests: PlayRuntimeManifestMap | null | undefined,
|
|
182
|
+
): PlayRuntimeManifestMap | null {
|
|
183
|
+
if (!manifests) return null;
|
|
184
|
+
const stripped: PlayRuntimeManifestMap = {};
|
|
185
|
+
for (const [key, manifest] of Object.entries(manifests)) {
|
|
186
|
+
stripped[key] = {
|
|
187
|
+
playName: manifest.playName,
|
|
188
|
+
graphHash: manifest.graphHash,
|
|
189
|
+
artifactStorageKey: manifest.artifactStorageKey,
|
|
190
|
+
artifactHash: manifest.artifactHash,
|
|
191
|
+
staticPipelineHash: manifest.staticPipelineHash,
|
|
192
|
+
staticPipeline: manifest.staticPipeline,
|
|
193
|
+
compiledAt: manifest.compiledAt,
|
|
194
|
+
compilerVersion: manifest.compilerVersion,
|
|
195
|
+
maxCreditsPerRun: manifest.maxCreditsPerRun,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
return stripped;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
202
|
+
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
203
|
+
}
|
|
@@ -565,6 +565,13 @@ export class DeeplineClient {
|
|
|
565
565
|
outputSchema: options?.compact
|
|
566
566
|
? this.compactSchema(play.outputSchema)
|
|
567
567
|
: (play.outputSchema ?? null),
|
|
568
|
+
staticPipeline: isRecord(play.staticPipeline)
|
|
569
|
+
? play.staticPipeline
|
|
570
|
+
: isRecord(play.currentRevision?.staticPipeline)
|
|
571
|
+
? play.currentRevision.staticPipeline
|
|
572
|
+
: isRecord(play.liveRevision?.staticPipeline)
|
|
573
|
+
? play.liveRevision.staticPipeline
|
|
574
|
+
: null,
|
|
568
575
|
...(csvInput ? { csvInput } : {}),
|
|
569
576
|
...(rowOutputSchema ? { rowOutputSchema } : {}),
|
|
570
577
|
runCommand,
|
|
@@ -540,7 +540,7 @@ export interface DeeplinePlayRuntimeContext {
|
|
|
540
540
|
* @returns Program output.
|
|
541
541
|
*/
|
|
542
542
|
runSteps<TInput extends Record<string, unknown>, TOutput>(
|
|
543
|
-
program: StepProgram<TInput,
|
|
543
|
+
program: StepProgram<TInput, any, TOutput>,
|
|
544
544
|
input: TInput,
|
|
545
545
|
options?: { description?: string },
|
|
546
546
|
): Promise<TOutput>;
|
|
@@ -50,10 +50,10 @@ export type SdkRelease = {
|
|
|
50
50
|
};
|
|
51
51
|
|
|
52
52
|
export const SDK_RELEASE = {
|
|
53
|
-
version: '0.1.
|
|
53
|
+
version: '0.1.79',
|
|
54
54
|
apiContract: '2026-06-dataset-column-cell-stale-hard-cutover',
|
|
55
55
|
supportPolicy: {
|
|
56
|
-
latest: '0.1.
|
|
56
|
+
latest: '0.1.79',
|
|
57
57
|
minimumSupported: '0.1.53',
|
|
58
58
|
deprecatedBelow: '0.1.53',
|
|
59
59
|
},
|
|
@@ -663,6 +663,9 @@ export interface PlayListItem {
|
|
|
663
663
|
isDraftDirty?: boolean;
|
|
664
664
|
inputSchema?: Record<string, unknown> | null;
|
|
665
665
|
outputSchema?: Record<string, unknown> | null;
|
|
666
|
+
staticPipeline?: unknown;
|
|
667
|
+
currentRevision?: PlayRevisionSummary | null;
|
|
668
|
+
liveRevision?: PlayRevisionSummary | null;
|
|
666
669
|
aliases?: string[];
|
|
667
670
|
}
|
|
668
671
|
|
|
@@ -677,6 +680,7 @@ export interface PlayDescription {
|
|
|
677
680
|
aliases: string[];
|
|
678
681
|
inputSchema?: Record<string, unknown> | null;
|
|
679
682
|
outputSchema?: Record<string, unknown> | null;
|
|
683
|
+
staticPipeline?: Record<string, unknown> | null;
|
|
680
684
|
csvInput?: Record<string, unknown> | null;
|
|
681
685
|
rowOutputSchema?: Record<string, unknown> | null;
|
|
682
686
|
runCommand: string;
|
|
@@ -71,7 +71,9 @@ export type PlayDatasetTransformOptions = {
|
|
|
71
71
|
* Deepline keeps row progress, retries, memory use, and table output under
|
|
72
72
|
* runtime control. Use `count()` and `peek()` for bounded inspection. Use
|
|
73
73
|
* `materialize(limit)` or async iteration only when the dataset is intentionally
|
|
74
|
-
* small and bounded.
|
|
74
|
+
* small and bounded. `PlayDataset` intentionally does not expose `.rows`,
|
|
75
|
+
* `.toArray()`, or other array aliases; those hide the runtime cost of loading
|
|
76
|
+
* persisted rows into memory.
|
|
75
77
|
*/
|
|
76
78
|
export interface PlayDataset<T> extends AsyncIterable<T> {
|
|
77
79
|
readonly [PLAY_DATASET_BRAND]: true;
|
|
@@ -3,6 +3,7 @@ import { normalizeTableNamespace } from './row-identity';
|
|
|
3
3
|
export interface PlayStaticPipeline {
|
|
4
4
|
tableNamespace?: string;
|
|
5
5
|
inputFields?: string[];
|
|
6
|
+
rowKeyFields?: string[];
|
|
6
7
|
csvArg?: string;
|
|
7
8
|
hasInlineData?: boolean;
|
|
8
9
|
csvDescription?: string;
|
|
@@ -32,6 +33,7 @@ export interface PlaySheetColumnContract {
|
|
|
32
33
|
outputSqlName?: string;
|
|
33
34
|
stepId?: string;
|
|
34
35
|
toolId?: string;
|
|
36
|
+
isRowKey?: boolean;
|
|
35
37
|
}
|
|
36
38
|
|
|
37
39
|
export interface PlaySheetContract {
|
|
@@ -39,6 +41,40 @@ export interface PlaySheetContract {
|
|
|
39
41
|
columns: PlaySheetColumnContract[];
|
|
40
42
|
}
|
|
41
43
|
|
|
44
|
+
export type PlayStaticColumnProducerKind =
|
|
45
|
+
| 'tool'
|
|
46
|
+
| 'waterfall'
|
|
47
|
+
| 'stepProgram'
|
|
48
|
+
| 'playCall'
|
|
49
|
+
| 'transform';
|
|
50
|
+
|
|
51
|
+
export interface PlayStaticColumnProducer {
|
|
52
|
+
id: string;
|
|
53
|
+
kind: PlayStaticColumnProducerKind;
|
|
54
|
+
field: string;
|
|
55
|
+
toolId?: string;
|
|
56
|
+
playId?: string;
|
|
57
|
+
conditional?: boolean;
|
|
58
|
+
sourceRange?: PlayStaticSourceRange;
|
|
59
|
+
steps?: PlayStaticColumnProducer[];
|
|
60
|
+
substep: PlayStaticSubstep;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface PlayStaticDatasetColumn {
|
|
64
|
+
id: string;
|
|
65
|
+
source: PlaySheetColumnSource;
|
|
66
|
+
sqlName?: string;
|
|
67
|
+
producers: PlayStaticColumnProducer[];
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export interface PlayCompiledStaticGraph {
|
|
71
|
+
topLevel: PlayStaticSubstep[];
|
|
72
|
+
datasets: Array<{
|
|
73
|
+
tableNamespace: string;
|
|
74
|
+
columns: PlayStaticDatasetColumn[];
|
|
75
|
+
}>;
|
|
76
|
+
}
|
|
77
|
+
|
|
42
78
|
export function ensureCompiledSheetContract(
|
|
43
79
|
pipeline: PlayStaticPipeline | null | undefined,
|
|
44
80
|
): PlayStaticPipeline | null | undefined {
|
|
@@ -92,8 +128,27 @@ function truncateStaticSubstepsForStorage(
|
|
|
92
128
|
return {
|
|
93
129
|
...base,
|
|
94
130
|
inputFields: base.inputFields ? [...base.inputFields] : undefined,
|
|
131
|
+
rowKeyFields: base.rowKeyFields ? [...base.rowKeyFields] : undefined,
|
|
95
132
|
outputFields: base.outputFields ? [...base.outputFields] : undefined,
|
|
133
|
+
columns: base.columns
|
|
134
|
+
? base.columns.map((column) => ({
|
|
135
|
+
...column,
|
|
136
|
+
producers: column.producers.map((producer) => ({
|
|
137
|
+
...producer,
|
|
138
|
+
sourceRange: cloneStorageSafeSourceRange(producer.sourceRange),
|
|
139
|
+
steps: producer.steps
|
|
140
|
+
? producer.steps.map((stepProducer) => ({
|
|
141
|
+
...stepProducer,
|
|
142
|
+
sourceRange: cloneStorageSafeSourceRange(
|
|
143
|
+
stepProducer.sourceRange,
|
|
144
|
+
),
|
|
145
|
+
}))
|
|
146
|
+
: undefined,
|
|
147
|
+
})),
|
|
148
|
+
}))
|
|
149
|
+
: undefined,
|
|
96
150
|
waterfallIds: base.waterfallIds ? [...base.waterfallIds] : undefined,
|
|
151
|
+
steps: truncateStaticSubstepsForStorage(base.steps, input),
|
|
97
152
|
sheetContract: cloneStorageSafeSheetContract(base.sheetContract),
|
|
98
153
|
};
|
|
99
154
|
}
|
|
@@ -153,6 +208,9 @@ export function truncateStaticPipelineForStorage(
|
|
|
153
208
|
return {
|
|
154
209
|
...pipeline,
|
|
155
210
|
inputFields: pipeline.inputFields ? [...pipeline.inputFields] : undefined,
|
|
211
|
+
rowKeyFields: pipeline.rowKeyFields
|
|
212
|
+
? [...pipeline.rowKeyFields]
|
|
213
|
+
: undefined,
|
|
156
214
|
fields: [...(pipeline.fields ?? [])],
|
|
157
215
|
stages: truncateStaticSubstepsForStorage(pipeline.stages, {
|
|
158
216
|
embeddedPlayCallPipelineDepth,
|
|
@@ -198,8 +256,11 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
|
|
|
198
256
|
name?: string;
|
|
199
257
|
tableNamespace?: string;
|
|
200
258
|
inputFields?: string[];
|
|
259
|
+
rowKeyFields?: string[];
|
|
201
260
|
outputFields?: string[];
|
|
261
|
+
columns?: PlayStaticDatasetColumn[];
|
|
202
262
|
waterfallIds?: string[];
|
|
263
|
+
steps?: PlayStaticSubstep[];
|
|
203
264
|
sheetContract?: PlaySheetContract | null;
|
|
204
265
|
description?: string;
|
|
205
266
|
sourceRange?: PlayStaticSourceRange;
|
|
@@ -210,6 +271,8 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
|
|
|
210
271
|
type: 'tool';
|
|
211
272
|
toolId: string;
|
|
212
273
|
field: string;
|
|
274
|
+
paramsSource?: string;
|
|
275
|
+
sourceText?: string;
|
|
213
276
|
description?: string;
|
|
214
277
|
inLoop?: boolean;
|
|
215
278
|
isEventWait?: boolean;
|
|
@@ -263,6 +326,7 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
|
|
|
263
326
|
| {
|
|
264
327
|
type: 'run_javascript';
|
|
265
328
|
alias: string;
|
|
329
|
+
sourceText?: string;
|
|
266
330
|
description?: string;
|
|
267
331
|
sourceRange?: PlayStaticSourceRange;
|
|
268
332
|
callDepth?: number;
|
|
@@ -271,6 +335,7 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
|
|
|
271
335
|
| {
|
|
272
336
|
type: 'code';
|
|
273
337
|
field: string;
|
|
338
|
+
sourceText?: string;
|
|
274
339
|
description?: string;
|
|
275
340
|
sourceRange?: PlayStaticSourceRange;
|
|
276
341
|
callDepth?: number;
|
|
@@ -289,6 +354,12 @@ export function getCompiledPipelineSubsteps(
|
|
|
289
354
|
|
|
290
355
|
export function getTopLevelPipelineSubsteps(
|
|
291
356
|
pipeline: PlayStaticPipeline | null | undefined,
|
|
357
|
+
): PlayStaticSubstep[] {
|
|
358
|
+
return compileStaticGraph(pipeline).topLevel;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function getRawTopLevelPipelineSubsteps(
|
|
362
|
+
pipeline: PlayStaticPipeline | null | undefined,
|
|
292
363
|
): PlayStaticSubstep[] {
|
|
293
364
|
if (!pipeline) {
|
|
294
365
|
return [];
|
|
@@ -328,6 +399,14 @@ export function flattenStaticSubsteps(
|
|
|
328
399
|
|
|
329
400
|
for (const substep of substeps) {
|
|
330
401
|
flattened.push(substep);
|
|
402
|
+
if (substep.type === 'dataset' && substep.steps?.length) {
|
|
403
|
+
flattened.push(...flattenStaticSubsteps(substep.steps));
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
if (substep.type === 'step_suite') {
|
|
407
|
+
flattened.push(...flattenStaticSubsteps(substep.steps));
|
|
408
|
+
continue;
|
|
409
|
+
}
|
|
331
410
|
if (substep.type === 'play_call' && substep.pipeline) {
|
|
332
411
|
const nestedSubsteps = getCompiledPipelineSubsteps(substep.pipeline);
|
|
333
412
|
if (nestedSubsteps.length > 0) {
|
|
@@ -345,6 +424,137 @@ export function flattenStaticPipeline(
|
|
|
345
424
|
return flattenStaticSubsteps(getCompiledPipelineSubsteps(pipeline));
|
|
346
425
|
}
|
|
347
426
|
|
|
427
|
+
export function compileStaticGraph(
|
|
428
|
+
pipeline: PlayStaticPipeline | null | undefined,
|
|
429
|
+
): PlayCompiledStaticGraph {
|
|
430
|
+
const rawTopLevel = getRawTopLevelPipelineSubsteps(pipeline);
|
|
431
|
+
const datasets: PlayCompiledStaticGraph['datasets'] = [];
|
|
432
|
+
const topLevel = rawTopLevel.map((substep) => {
|
|
433
|
+
if (substep.type !== 'dataset') {
|
|
434
|
+
return substep;
|
|
435
|
+
}
|
|
436
|
+
const columns = compileDatasetColumns(substep);
|
|
437
|
+
const tableNamespace = (substep.tableNamespace ?? substep.field).trim();
|
|
438
|
+
if (tableNamespace) {
|
|
439
|
+
datasets.push({ tableNamespace, columns });
|
|
440
|
+
}
|
|
441
|
+
return {
|
|
442
|
+
...substep,
|
|
443
|
+
columns,
|
|
444
|
+
} satisfies PlayStaticSubstep;
|
|
445
|
+
});
|
|
446
|
+
return { topLevel, datasets };
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
function compileDatasetColumns(
|
|
450
|
+
dataset: Extract<PlayStaticSubstep, { type: 'dataset' }>,
|
|
451
|
+
): PlayStaticDatasetColumn[] {
|
|
452
|
+
const columnsById = new Map<string, PlayStaticDatasetColumn>();
|
|
453
|
+
const ensureColumn = (
|
|
454
|
+
id: string,
|
|
455
|
+
source: PlaySheetColumnSource,
|
|
456
|
+
sqlName?: string,
|
|
457
|
+
) => {
|
|
458
|
+
const trimmed = id.trim();
|
|
459
|
+
if (!trimmed) return null;
|
|
460
|
+
const existing = columnsById.get(trimmed);
|
|
461
|
+
if (existing) {
|
|
462
|
+
if (!existing.sqlName && sqlName) existing.sqlName = sqlName;
|
|
463
|
+
return existing;
|
|
464
|
+
}
|
|
465
|
+
const column: PlayStaticDatasetColumn = {
|
|
466
|
+
id: trimmed,
|
|
467
|
+
source,
|
|
468
|
+
...(sqlName ? { sqlName } : {}),
|
|
469
|
+
producers: [],
|
|
470
|
+
};
|
|
471
|
+
columnsById.set(trimmed, column);
|
|
472
|
+
return column;
|
|
473
|
+
};
|
|
474
|
+
|
|
475
|
+
for (const column of dataset.sheetContract?.columns ?? []) {
|
|
476
|
+
ensureColumn(column.id, column.source, column.sqlName);
|
|
477
|
+
}
|
|
478
|
+
for (const field of dataset.inputFields ?? []) {
|
|
479
|
+
ensureColumn(field, 'input', sqlSafePlayColumnName(field));
|
|
480
|
+
}
|
|
481
|
+
for (const field of dataset.outputFields ?? []) {
|
|
482
|
+
ensureColumn(field, 'datasetColumn', sqlSafePlayColumnName(field));
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
for (const substep of dataset.steps ?? []) {
|
|
486
|
+
const field = fieldForColumnProducer(substep);
|
|
487
|
+
if (!field) continue;
|
|
488
|
+
const column = ensureColumn(
|
|
489
|
+
field,
|
|
490
|
+
'datasetColumn',
|
|
491
|
+
sqlSafePlayColumnName(field),
|
|
492
|
+
);
|
|
493
|
+
if (!column) continue;
|
|
494
|
+
column.producers.push(columnProducerFromSubstep(substep, field));
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
return [...columnsById.values()];
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
function fieldForColumnProducer(substep: PlayStaticSubstep): string | null {
|
|
501
|
+
if ('field' in substep && typeof substep.field === 'string') {
|
|
502
|
+
return substep.field.trim() || null;
|
|
503
|
+
}
|
|
504
|
+
if (substep.type === 'run_javascript') {
|
|
505
|
+
return substep.alias.trim() || null;
|
|
506
|
+
}
|
|
507
|
+
return null;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
function columnProducerFromSubstep(
|
|
511
|
+
substep: PlayStaticSubstep,
|
|
512
|
+
field: string,
|
|
513
|
+
): PlayStaticColumnProducer {
|
|
514
|
+
const steps =
|
|
515
|
+
substep.type === 'step_suite'
|
|
516
|
+
? substep.steps
|
|
517
|
+
.map((step) => {
|
|
518
|
+
const stepField = fieldForColumnProducer(step) ?? field;
|
|
519
|
+
return columnProducerFromSubstep(step, stepField);
|
|
520
|
+
})
|
|
521
|
+
.filter((producer) => producer.field.trim())
|
|
522
|
+
: undefined;
|
|
523
|
+
const kind: PlayStaticColumnProducerKind =
|
|
524
|
+
substep.type === 'tool'
|
|
525
|
+
? 'tool'
|
|
526
|
+
: substep.type === 'waterfall'
|
|
527
|
+
? 'waterfall'
|
|
528
|
+
: substep.type === 'step_suite'
|
|
529
|
+
? 'stepProgram'
|
|
530
|
+
: substep.type === 'play_call'
|
|
531
|
+
? 'playCall'
|
|
532
|
+
: 'transform';
|
|
533
|
+
return {
|
|
534
|
+
id: producerId(substep, field),
|
|
535
|
+
kind,
|
|
536
|
+
field,
|
|
537
|
+
...(substep.type === 'tool' ? { toolId: substep.toolId } : {}),
|
|
538
|
+
...(substep.type === 'play_call' ? { playId: substep.playId } : {}),
|
|
539
|
+
...(substep.conditional ? { conditional: true } : {}),
|
|
540
|
+
...(substep.sourceRange ? { sourceRange: substep.sourceRange } : {}),
|
|
541
|
+
...(steps && steps.length > 0 ? { steps } : {}),
|
|
542
|
+
substep,
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
function producerId(substep: PlayStaticSubstep, field: string): string {
|
|
547
|
+
if (substep.type === 'tool') return `${field}:tool:${substep.toolId}`;
|
|
548
|
+
if (substep.type === 'waterfall') {
|
|
549
|
+
return `${field}:waterfall:${substep.id ?? substep.tool ?? 'unknown'}`;
|
|
550
|
+
}
|
|
551
|
+
if (substep.type === 'play_call') return `${field}:play:${substep.playId}`;
|
|
552
|
+
if (substep.type === 'step_suite') return `${field}:steps`;
|
|
553
|
+
if (substep.type === 'run_javascript')
|
|
554
|
+
return `${field}:transform:${substep.alias}`;
|
|
555
|
+
return `${field}:transform`;
|
|
556
|
+
}
|
|
557
|
+
|
|
348
558
|
export function resolveSheetContractForTableNamespace(
|
|
349
559
|
pipeline: PlayStaticPipeline | null | undefined,
|
|
350
560
|
tableNamespace: string | null | undefined,
|
|
@@ -400,6 +610,50 @@ export function resolveSheetContractForTableNamespace(
|
|
|
400
610
|
return resolveFromPipeline(pipeline);
|
|
401
611
|
}
|
|
402
612
|
|
|
613
|
+
export function resolveStaticDatasetColumnsForTableNamespace(
|
|
614
|
+
pipeline: PlayStaticPipeline | null | undefined,
|
|
615
|
+
tableNamespace: string | null | undefined,
|
|
616
|
+
): PlayStaticDatasetColumn[] {
|
|
617
|
+
const requestedNamespace = tableNamespace?.trim();
|
|
618
|
+
if (!pipeline || !requestedNamespace) {
|
|
619
|
+
return [];
|
|
620
|
+
}
|
|
621
|
+
const normalizedNamespace = normalizeTableNamespace(requestedNamespace);
|
|
622
|
+
const seen = new Set<PlayStaticPipeline>();
|
|
623
|
+
|
|
624
|
+
const resolveFromPipeline = (
|
|
625
|
+
currentPipeline: PlayStaticPipeline | null | undefined,
|
|
626
|
+
): PlayStaticDatasetColumn[] | null => {
|
|
627
|
+
if (!currentPipeline || seen.has(currentPipeline)) {
|
|
628
|
+
return null;
|
|
629
|
+
}
|
|
630
|
+
seen.add(currentPipeline);
|
|
631
|
+
|
|
632
|
+
const compiled = compileStaticGraph(currentPipeline);
|
|
633
|
+
const matchingDataset = compiled.datasets.find(
|
|
634
|
+
(dataset) =>
|
|
635
|
+
normalizeTableNamespace(dataset.tableNamespace) ===
|
|
636
|
+
normalizedNamespace,
|
|
637
|
+
);
|
|
638
|
+
if (matchingDataset) {
|
|
639
|
+
return matchingDataset.columns;
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
for (const substep of getCompiledPipelineSubsteps(currentPipeline)) {
|
|
643
|
+
if (substep.type === 'play_call') {
|
|
644
|
+
const nestedColumns = resolveFromPipeline(substep.pipeline);
|
|
645
|
+
if (nestedColumns) {
|
|
646
|
+
return nestedColumns;
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
return null;
|
|
652
|
+
};
|
|
653
|
+
|
|
654
|
+
return resolveFromPipeline(pipeline) ?? [];
|
|
655
|
+
}
|
|
656
|
+
|
|
403
657
|
export function sqlSafePlayColumnName(id: string): string {
|
|
404
658
|
const normalized = id
|
|
405
659
|
.trim()
|
|
@@ -441,8 +695,14 @@ export function compileSheetContract(pipeline: PlayStaticPipeline): {
|
|
|
441
695
|
const inputFields = pipeline.inputFields?.length
|
|
442
696
|
? pipeline.inputFields
|
|
443
697
|
: [tableNamespace];
|
|
698
|
+
const rowKeyFieldSet = new Set(pipeline.rowKeyFields ?? []);
|
|
444
699
|
for (const inputField of inputFields) {
|
|
445
|
-
addColumn({
|
|
700
|
+
addColumn({
|
|
701
|
+
id: inputField,
|
|
702
|
+
source: 'input',
|
|
703
|
+
field: inputField,
|
|
704
|
+
...(rowKeyFieldSet.has(inputField) ? { isRowKey: true } : {}),
|
|
705
|
+
});
|
|
446
706
|
}
|
|
447
707
|
|
|
448
708
|
for (const field of pipeline.fields) {
|