deepline 0.1.77 → 0.1.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -113,6 +113,8 @@ type WorkflowRunMapping = {
113
113
  type WorkflowRunRetryState = {
114
114
  runId: string;
115
115
  params: unknown;
116
+ paramsRef?: unknown;
117
+ paramsBytes?: number;
116
118
  retryAttempts: number;
117
119
  updatedAt: number;
118
120
  expiresAt: number;
@@ -1059,8 +1061,10 @@ export class PlayDedup implements DurableObject {
1059
1061
  ttlMs?: unknown;
1060
1062
  } | null;
1061
1063
  const runId = typeof body?.runId === 'string' ? body.runId : '';
1062
- if (!runId || !body || !('params' in body)) {
1063
- return new Response('runId and params are required', { status: 400 });
1064
+ if (!runId || !body || (!('params' in body) && !('paramsRef' in body))) {
1065
+ return new Response('runId and params or paramsRef are required', {
1066
+ status: 400,
1067
+ });
1064
1068
  }
1065
1069
  const now = Date.now();
1066
1070
  const ttlMs =
@@ -1075,7 +1079,14 @@ export class PlayDedup implements DurableObject {
1075
1079
  const existing = await this.state.storage.get<WorkflowRunRetryState>(key);
1076
1080
  const retryState = {
1077
1081
  runId,
1078
- params: body.params,
1082
+ params: 'params' in body ? body.params : null,
1083
+ paramsRef: 'paramsRef' in body ? body.paramsRef : null,
1084
+ paramsBytes:
1085
+ typeof (body as { paramsBytes?: unknown }).paramsBytes ===
1086
+ 'number' &&
1087
+ Number.isFinite((body as { paramsBytes?: number }).paramsBytes)
1088
+ ? (body as { paramsBytes: number }).paramsBytes
1089
+ : undefined,
1079
1090
  retryAttempts:
1080
1091
  existing?.runId === runId &&
1081
1092
  typeof existing.retryAttempts === 'number'
@@ -1128,6 +1139,8 @@ export class PlayDedup implements DurableObject {
1128
1139
  claimed: false,
1129
1140
  attempts: existing.retryAttempts,
1130
1141
  params: existing.params,
1142
+ paramsRef: existing.paramsRef ?? null,
1143
+ paramsBytes: existing.paramsBytes ?? null,
1131
1144
  };
1132
1145
  return;
1133
1146
  }
@@ -1142,6 +1155,8 @@ export class PlayDedup implements DurableObject {
1142
1155
  claimed: true,
1143
1156
  attempts: nextAttempts,
1144
1157
  params: existing.params,
1158
+ paramsRef: existing.paramsRef ?? null,
1159
+ paramsBytes: existing.paramsBytes ?? null,
1145
1160
  };
1146
1161
  });
1147
1162
  return new Response(JSON.stringify(response), {
@@ -0,0 +1,203 @@
1
+ import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
2
+ import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
3
+ import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
4
+ import type {
5
+ PlayRuntimeManifest,
6
+ PlayRuntimeManifestMap,
7
+ } from '../../../shared_libs/plays/compiler-manifest';
8
+
9
+ export const WORKFLOW_RETRY_STATE_TARGET_BYTES = 100_000;
10
+ export const WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES =
11
+ WORKFLOW_RETRY_STATE_TARGET_BYTES;
12
+ export const WORKFLOW_RETRY_PARAMS_MAX_BYTES = 1024 * 1024;
13
+
14
+ export type WorkflowRetryParamsRef = {
15
+ storageKind: 'r2';
16
+ storageKey: string;
17
+ bytes: number;
18
+ hash: string;
19
+ expiresAt: number;
20
+ };
21
+
22
+ export type WorkflowRetryPlayParams = {
23
+ runId: string;
24
+ playId: string;
25
+ playName: string;
26
+ artifactStorageKey: string;
27
+ artifactHash: string;
28
+ graphHash: string;
29
+ input: Record<string, unknown>;
30
+ inputFile?: {
31
+ name?: string;
32
+ r2Key?: string;
33
+ storageKey?: string;
34
+ path?: string;
35
+ fileName?: string;
36
+ logicalPath?: string;
37
+ contentType?: string;
38
+ bytes?: number;
39
+ } | null;
40
+ inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
41
+ packagedFiles?: Array<{
42
+ playPath: string;
43
+ storageKey: string;
44
+ contentType?: string;
45
+ bytes?: number;
46
+ inlineText?: string;
47
+ }> | null;
48
+ contractSnapshot?: unknown;
49
+ executionPlan?: ExecutionPlan | null;
50
+ childPlayManifests?: PlayRuntimeManifestMap | null;
51
+ playCallGovernance?: PlayCallGovernanceSnapshot | null;
52
+ preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
53
+ preloadedDbSessionRef?: {
54
+ runId: string;
55
+ sessionCount: number;
56
+ expiresAt: number;
57
+ } | null;
58
+ dynamicWorkerCode?: string | null;
59
+ executorToken: string;
60
+ baseUrl: string;
61
+ orgId: string;
62
+ userEmail: string;
63
+ userId?: string | null;
64
+ runtimeBackend: string;
65
+ dedupBackend: string;
66
+ totalRows?: number;
67
+ coordinatorUrl?: string | null;
68
+ coordinatorInternalToken?: string | null;
69
+ };
70
+
71
+ export type WorkflowRetryStatePayload<TParams = WorkflowRetryPlayParams> =
72
+ | {
73
+ params: TParams;
74
+ paramsRef?: null;
75
+ paramsBytes: number;
76
+ }
77
+ | {
78
+ params: null;
79
+ paramsRef: WorkflowRetryParamsRef;
80
+ paramsBytes: number;
81
+ };
82
+
83
+ export function buildWorkflowRetryParams<
84
+ TParams extends WorkflowRetryPlayParams,
85
+ >(params: TParams): TParams {
86
+ const retryParams = {
87
+ ...params,
88
+ dynamicWorkerCode: null,
89
+ contractSnapshot: stripRetrySourceSnapshot(params.contractSnapshot),
90
+ childPlayManifests: stripRetryChildManifestCode(params.childPlayManifests),
91
+ packagedFiles: stripRetryPackagedFiles(params.packagedFiles),
92
+ } satisfies WorkflowRetryPlayParams as TParams;
93
+ if (jsonByteLength(retryParams) <= WORKFLOW_RETRY_STATE_TARGET_BYTES) {
94
+ return retryParams;
95
+ }
96
+ return {
97
+ ...retryParams,
98
+ contractSnapshot: stripRetryContractSnapshotToArtifact(
99
+ retryParams.contractSnapshot,
100
+ params,
101
+ ),
102
+ childPlayManifests: stripRetryChildManifestToArtifact(
103
+ retryParams.childPlayManifests,
104
+ ),
105
+ } satisfies WorkflowRetryPlayParams as TParams;
106
+ }
107
+
108
+ export function jsonByteLength(value: unknown): number {
109
+ return new TextEncoder().encode(JSON.stringify(value)).length;
110
+ }
111
+
112
+ export function workflowRetryParamsStorageKey(input: {
113
+ runId: string;
114
+ hash: string;
115
+ }): string {
116
+ const safeRunId = input.runId
117
+ .toLowerCase()
118
+ .replace(/[^a-z0-9_-]+/g, '-')
119
+ .slice(0, 140);
120
+ return `plays/workflow-retry-params/${safeRunId || 'run'}/${input.hash}.json`;
121
+ }
122
+
123
+ function stripRetryPackagedFiles(
124
+ files: WorkflowRetryPlayParams['packagedFiles'],
125
+ ): WorkflowRetryPlayParams['packagedFiles'] {
126
+ return (
127
+ files?.map((file) => ({
128
+ playPath: file.playPath,
129
+ storageKey: file.storageKey,
130
+ contentType: file.contentType,
131
+ bytes: file.bytes,
132
+ })) ?? null
133
+ );
134
+ }
135
+
136
+ function stripRetrySourceSnapshot(snapshot: unknown): unknown {
137
+ if (!isRecord(snapshot)) return snapshot;
138
+ const rest = { ...snapshot };
139
+ delete rest.sourceCode;
140
+ delete rest.sourceFiles;
141
+ delete rest.bundledCode;
142
+ return rest;
143
+ }
144
+
145
+ function stripRetryContractSnapshotToArtifact(
146
+ snapshot: unknown,
147
+ params: WorkflowRetryPlayParams,
148
+ ): unknown {
149
+ if (!isRecord(snapshot)) return snapshot;
150
+ return {
151
+ source: snapshot.source ?? 'artifact',
152
+ revisionVersion: snapshot.revisionVersion ?? null,
153
+ staticPipeline: snapshot.staticPipeline ?? null,
154
+ billingLimit: snapshot.billingLimit ?? null,
155
+ artifactMetadata: {
156
+ storageKey: params.artifactStorageKey,
157
+ artifactHash: params.artifactHash,
158
+ graphHash: params.graphHash,
159
+ },
160
+ codeFormat: snapshot.codeFormat ?? null,
161
+ compatibility: snapshot.compatibility ?? null,
162
+ };
163
+ }
164
+
165
+ function stripRetryChildManifestCode(
166
+ manifests: PlayRuntimeManifestMap | null | undefined,
167
+ ): PlayRuntimeManifestMap | null {
168
+ if (!manifests) return null;
169
+ const stripped: PlayRuntimeManifestMap = {};
170
+ for (const [key, manifest] of Object.entries(manifests)) {
171
+ const rest = { ...manifest };
172
+ delete rest.bundledCode;
173
+ delete rest.sourceCode;
174
+ delete (rest as Record<string, unknown>).sourceMap;
175
+ stripped[key] = rest;
176
+ }
177
+ return stripped;
178
+ }
179
+
180
+ function stripRetryChildManifestToArtifact(
181
+ manifests: PlayRuntimeManifestMap | null | undefined,
182
+ ): PlayRuntimeManifestMap | null {
183
+ if (!manifests) return null;
184
+ const stripped: PlayRuntimeManifestMap = {};
185
+ for (const [key, manifest] of Object.entries(manifests)) {
186
+ stripped[key] = {
187
+ playName: manifest.playName,
188
+ graphHash: manifest.graphHash,
189
+ artifactStorageKey: manifest.artifactStorageKey,
190
+ artifactHash: manifest.artifactHash,
191
+ staticPipelineHash: manifest.staticPipelineHash,
192
+ staticPipeline: manifest.staticPipeline,
193
+ compiledAt: manifest.compiledAt,
194
+ compilerVersion: manifest.compilerVersion,
195
+ maxCreditsPerRun: manifest.maxCreditsPerRun,
196
+ };
197
+ }
198
+ return stripped;
199
+ }
200
+
201
+ function isRecord(value: unknown): value is Record<string, unknown> {
202
+ return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
203
+ }
@@ -565,6 +565,13 @@ export class DeeplineClient {
565
565
  outputSchema: options?.compact
566
566
  ? this.compactSchema(play.outputSchema)
567
567
  : (play.outputSchema ?? null),
568
+ staticPipeline: isRecord(play.staticPipeline)
569
+ ? play.staticPipeline
570
+ : isRecord(play.currentRevision?.staticPipeline)
571
+ ? play.currentRevision.staticPipeline
572
+ : isRecord(play.liveRevision?.staticPipeline)
573
+ ? play.liveRevision.staticPipeline
574
+ : null,
568
575
  ...(csvInput ? { csvInput } : {}),
569
576
  ...(rowOutputSchema ? { rowOutputSchema } : {}),
570
577
  runCommand,
@@ -540,7 +540,7 @@ export interface DeeplinePlayRuntimeContext {
540
540
  * @returns Program output.
541
541
  */
542
542
  runSteps<TInput extends Record<string, unknown>, TOutput>(
543
- program: StepProgram<TInput, unknown, TOutput>,
543
+ program: StepProgram<TInput, any, TOutput>,
544
544
  input: TInput,
545
545
  options?: { description?: string },
546
546
  ): Promise<TOutput>;
@@ -50,10 +50,10 @@ export type SdkRelease = {
50
50
  };
51
51
 
52
52
  export const SDK_RELEASE = {
53
- version: '0.1.77',
53
+ version: '0.1.79',
54
54
  apiContract: '2026-06-dataset-column-cell-stale-hard-cutover',
55
55
  supportPolicy: {
56
- latest: '0.1.77',
56
+ latest: '0.1.79',
57
57
  minimumSupported: '0.1.53',
58
58
  deprecatedBelow: '0.1.53',
59
59
  },
@@ -663,6 +663,9 @@ export interface PlayListItem {
663
663
  isDraftDirty?: boolean;
664
664
  inputSchema?: Record<string, unknown> | null;
665
665
  outputSchema?: Record<string, unknown> | null;
666
+ staticPipeline?: unknown;
667
+ currentRevision?: PlayRevisionSummary | null;
668
+ liveRevision?: PlayRevisionSummary | null;
666
669
  aliases?: string[];
667
670
  }
668
671
 
@@ -677,6 +680,7 @@ export interface PlayDescription {
677
680
  aliases: string[];
678
681
  inputSchema?: Record<string, unknown> | null;
679
682
  outputSchema?: Record<string, unknown> | null;
683
+ staticPipeline?: Record<string, unknown> | null;
680
684
  csvInput?: Record<string, unknown> | null;
681
685
  rowOutputSchema?: Record<string, unknown> | null;
682
686
  runCommand: string;
@@ -71,7 +71,9 @@ export type PlayDatasetTransformOptions = {
71
71
  * Deepline keeps row progress, retries, memory use, and table output under
72
72
  * runtime control. Use `count()` and `peek()` for bounded inspection. Use
73
73
  * `materialize(limit)` or async iteration only when the dataset is intentionally
74
- * small and bounded.
74
+ * small and bounded. `PlayDataset` intentionally does not expose `.rows`,
75
+ * `.toArray()`, or other array aliases; those hide the runtime cost of loading
76
+ * persisted rows into memory.
75
77
  */
76
78
  export interface PlayDataset<T> extends AsyncIterable<T> {
77
79
  readonly [PLAY_DATASET_BRAND]: true;
@@ -3,6 +3,7 @@ import { normalizeTableNamespace } from './row-identity';
3
3
  export interface PlayStaticPipeline {
4
4
  tableNamespace?: string;
5
5
  inputFields?: string[];
6
+ rowKeyFields?: string[];
6
7
  csvArg?: string;
7
8
  hasInlineData?: boolean;
8
9
  csvDescription?: string;
@@ -32,6 +33,7 @@ export interface PlaySheetColumnContract {
32
33
  outputSqlName?: string;
33
34
  stepId?: string;
34
35
  toolId?: string;
36
+ isRowKey?: boolean;
35
37
  }
36
38
 
37
39
  export interface PlaySheetContract {
@@ -39,6 +41,40 @@ export interface PlaySheetContract {
39
41
  columns: PlaySheetColumnContract[];
40
42
  }
41
43
 
44
+ export type PlayStaticColumnProducerKind =
45
+ | 'tool'
46
+ | 'waterfall'
47
+ | 'stepProgram'
48
+ | 'playCall'
49
+ | 'transform';
50
+
51
+ export interface PlayStaticColumnProducer {
52
+ id: string;
53
+ kind: PlayStaticColumnProducerKind;
54
+ field: string;
55
+ toolId?: string;
56
+ playId?: string;
57
+ conditional?: boolean;
58
+ sourceRange?: PlayStaticSourceRange;
59
+ steps?: PlayStaticColumnProducer[];
60
+ substep: PlayStaticSubstep;
61
+ }
62
+
63
+ export interface PlayStaticDatasetColumn {
64
+ id: string;
65
+ source: PlaySheetColumnSource;
66
+ sqlName?: string;
67
+ producers: PlayStaticColumnProducer[];
68
+ }
69
+
70
+ export interface PlayCompiledStaticGraph {
71
+ topLevel: PlayStaticSubstep[];
72
+ datasets: Array<{
73
+ tableNamespace: string;
74
+ columns: PlayStaticDatasetColumn[];
75
+ }>;
76
+ }
77
+
42
78
  export function ensureCompiledSheetContract(
43
79
  pipeline: PlayStaticPipeline | null | undefined,
44
80
  ): PlayStaticPipeline | null | undefined {
@@ -92,8 +128,27 @@ function truncateStaticSubstepsForStorage(
92
128
  return {
93
129
  ...base,
94
130
  inputFields: base.inputFields ? [...base.inputFields] : undefined,
131
+ rowKeyFields: base.rowKeyFields ? [...base.rowKeyFields] : undefined,
95
132
  outputFields: base.outputFields ? [...base.outputFields] : undefined,
133
+ columns: base.columns
134
+ ? base.columns.map((column) => ({
135
+ ...column,
136
+ producers: column.producers.map((producer) => ({
137
+ ...producer,
138
+ sourceRange: cloneStorageSafeSourceRange(producer.sourceRange),
139
+ steps: producer.steps
140
+ ? producer.steps.map((stepProducer) => ({
141
+ ...stepProducer,
142
+ sourceRange: cloneStorageSafeSourceRange(
143
+ stepProducer.sourceRange,
144
+ ),
145
+ }))
146
+ : undefined,
147
+ })),
148
+ }))
149
+ : undefined,
96
150
  waterfallIds: base.waterfallIds ? [...base.waterfallIds] : undefined,
151
+ steps: truncateStaticSubstepsForStorage(base.steps, input),
97
152
  sheetContract: cloneStorageSafeSheetContract(base.sheetContract),
98
153
  };
99
154
  }
@@ -153,6 +208,9 @@ export function truncateStaticPipelineForStorage(
153
208
  return {
154
209
  ...pipeline,
155
210
  inputFields: pipeline.inputFields ? [...pipeline.inputFields] : undefined,
211
+ rowKeyFields: pipeline.rowKeyFields
212
+ ? [...pipeline.rowKeyFields]
213
+ : undefined,
156
214
  fields: [...(pipeline.fields ?? [])],
157
215
  stages: truncateStaticSubstepsForStorage(pipeline.stages, {
158
216
  embeddedPlayCallPipelineDepth,
@@ -198,8 +256,11 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
198
256
  name?: string;
199
257
  tableNamespace?: string;
200
258
  inputFields?: string[];
259
+ rowKeyFields?: string[];
201
260
  outputFields?: string[];
261
+ columns?: PlayStaticDatasetColumn[];
202
262
  waterfallIds?: string[];
263
+ steps?: PlayStaticSubstep[];
203
264
  sheetContract?: PlaySheetContract | null;
204
265
  description?: string;
205
266
  sourceRange?: PlayStaticSourceRange;
@@ -210,6 +271,8 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
210
271
  type: 'tool';
211
272
  toolId: string;
212
273
  field: string;
274
+ paramsSource?: string;
275
+ sourceText?: string;
213
276
  description?: string;
214
277
  inLoop?: boolean;
215
278
  isEventWait?: boolean;
@@ -263,6 +326,7 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
263
326
  | {
264
327
  type: 'run_javascript';
265
328
  alias: string;
329
+ sourceText?: string;
266
330
  description?: string;
267
331
  sourceRange?: PlayStaticSourceRange;
268
332
  callDepth?: number;
@@ -271,6 +335,7 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
271
335
  | {
272
336
  type: 'code';
273
337
  field: string;
338
+ sourceText?: string;
274
339
  description?: string;
275
340
  sourceRange?: PlayStaticSourceRange;
276
341
  callDepth?: number;
@@ -289,6 +354,12 @@ export function getCompiledPipelineSubsteps(
289
354
 
290
355
  export function getTopLevelPipelineSubsteps(
291
356
  pipeline: PlayStaticPipeline | null | undefined,
357
+ ): PlayStaticSubstep[] {
358
+ return compileStaticGraph(pipeline).topLevel;
359
+ }
360
+
361
+ function getRawTopLevelPipelineSubsteps(
362
+ pipeline: PlayStaticPipeline | null | undefined,
292
363
  ): PlayStaticSubstep[] {
293
364
  if (!pipeline) {
294
365
  return [];
@@ -328,6 +399,14 @@ export function flattenStaticSubsteps(
328
399
 
329
400
  for (const substep of substeps) {
330
401
  flattened.push(substep);
402
+ if (substep.type === 'dataset' && substep.steps?.length) {
403
+ flattened.push(...flattenStaticSubsteps(substep.steps));
404
+ continue;
405
+ }
406
+ if (substep.type === 'step_suite') {
407
+ flattened.push(...flattenStaticSubsteps(substep.steps));
408
+ continue;
409
+ }
331
410
  if (substep.type === 'play_call' && substep.pipeline) {
332
411
  const nestedSubsteps = getCompiledPipelineSubsteps(substep.pipeline);
333
412
  if (nestedSubsteps.length > 0) {
@@ -345,6 +424,137 @@ export function flattenStaticPipeline(
345
424
  return flattenStaticSubsteps(getCompiledPipelineSubsteps(pipeline));
346
425
  }
347
426
 
427
+ export function compileStaticGraph(
428
+ pipeline: PlayStaticPipeline | null | undefined,
429
+ ): PlayCompiledStaticGraph {
430
+ const rawTopLevel = getRawTopLevelPipelineSubsteps(pipeline);
431
+ const datasets: PlayCompiledStaticGraph['datasets'] = [];
432
+ const topLevel = rawTopLevel.map((substep) => {
433
+ if (substep.type !== 'dataset') {
434
+ return substep;
435
+ }
436
+ const columns = compileDatasetColumns(substep);
437
+ const tableNamespace = (substep.tableNamespace ?? substep.field).trim();
438
+ if (tableNamespace) {
439
+ datasets.push({ tableNamespace, columns });
440
+ }
441
+ return {
442
+ ...substep,
443
+ columns,
444
+ } satisfies PlayStaticSubstep;
445
+ });
446
+ return { topLevel, datasets };
447
+ }
448
+
449
+ function compileDatasetColumns(
450
+ dataset: Extract<PlayStaticSubstep, { type: 'dataset' }>,
451
+ ): PlayStaticDatasetColumn[] {
452
+ const columnsById = new Map<string, PlayStaticDatasetColumn>();
453
+ const ensureColumn = (
454
+ id: string,
455
+ source: PlaySheetColumnSource,
456
+ sqlName?: string,
457
+ ) => {
458
+ const trimmed = id.trim();
459
+ if (!trimmed) return null;
460
+ const existing = columnsById.get(trimmed);
461
+ if (existing) {
462
+ if (!existing.sqlName && sqlName) existing.sqlName = sqlName;
463
+ return existing;
464
+ }
465
+ const column: PlayStaticDatasetColumn = {
466
+ id: trimmed,
467
+ source,
468
+ ...(sqlName ? { sqlName } : {}),
469
+ producers: [],
470
+ };
471
+ columnsById.set(trimmed, column);
472
+ return column;
473
+ };
474
+
475
+ for (const column of dataset.sheetContract?.columns ?? []) {
476
+ ensureColumn(column.id, column.source, column.sqlName);
477
+ }
478
+ for (const field of dataset.inputFields ?? []) {
479
+ ensureColumn(field, 'input', sqlSafePlayColumnName(field));
480
+ }
481
+ for (const field of dataset.outputFields ?? []) {
482
+ ensureColumn(field, 'datasetColumn', sqlSafePlayColumnName(field));
483
+ }
484
+
485
+ for (const substep of dataset.steps ?? []) {
486
+ const field = fieldForColumnProducer(substep);
487
+ if (!field) continue;
488
+ const column = ensureColumn(
489
+ field,
490
+ 'datasetColumn',
491
+ sqlSafePlayColumnName(field),
492
+ );
493
+ if (!column) continue;
494
+ column.producers.push(columnProducerFromSubstep(substep, field));
495
+ }
496
+
497
+ return [...columnsById.values()];
498
+ }
499
+
500
+ function fieldForColumnProducer(substep: PlayStaticSubstep): string | null {
501
+ if ('field' in substep && typeof substep.field === 'string') {
502
+ return substep.field.trim() || null;
503
+ }
504
+ if (substep.type === 'run_javascript') {
505
+ return substep.alias.trim() || null;
506
+ }
507
+ return null;
508
+ }
509
+
510
+ function columnProducerFromSubstep(
511
+ substep: PlayStaticSubstep,
512
+ field: string,
513
+ ): PlayStaticColumnProducer {
514
+ const steps =
515
+ substep.type === 'step_suite'
516
+ ? substep.steps
517
+ .map((step) => {
518
+ const stepField = fieldForColumnProducer(step) ?? field;
519
+ return columnProducerFromSubstep(step, stepField);
520
+ })
521
+ .filter((producer) => producer.field.trim())
522
+ : undefined;
523
+ const kind: PlayStaticColumnProducerKind =
524
+ substep.type === 'tool'
525
+ ? 'tool'
526
+ : substep.type === 'waterfall'
527
+ ? 'waterfall'
528
+ : substep.type === 'step_suite'
529
+ ? 'stepProgram'
530
+ : substep.type === 'play_call'
531
+ ? 'playCall'
532
+ : 'transform';
533
+ return {
534
+ id: producerId(substep, field),
535
+ kind,
536
+ field,
537
+ ...(substep.type === 'tool' ? { toolId: substep.toolId } : {}),
538
+ ...(substep.type === 'play_call' ? { playId: substep.playId } : {}),
539
+ ...(substep.conditional ? { conditional: true } : {}),
540
+ ...(substep.sourceRange ? { sourceRange: substep.sourceRange } : {}),
541
+ ...(steps && steps.length > 0 ? { steps } : {}),
542
+ substep,
543
+ };
544
+ }
545
+
546
+ function producerId(substep: PlayStaticSubstep, field: string): string {
547
+ if (substep.type === 'tool') return `${field}:tool:${substep.toolId}`;
548
+ if (substep.type === 'waterfall') {
549
+ return `${field}:waterfall:${substep.id ?? substep.tool ?? 'unknown'}`;
550
+ }
551
+ if (substep.type === 'play_call') return `${field}:play:${substep.playId}`;
552
+ if (substep.type === 'step_suite') return `${field}:steps`;
553
+ if (substep.type === 'run_javascript')
554
+ return `${field}:transform:${substep.alias}`;
555
+ return `${field}:transform`;
556
+ }
557
+
348
558
  export function resolveSheetContractForTableNamespace(
349
559
  pipeline: PlayStaticPipeline | null | undefined,
350
560
  tableNamespace: string | null | undefined,
@@ -400,6 +610,50 @@ export function resolveSheetContractForTableNamespace(
400
610
  return resolveFromPipeline(pipeline);
401
611
  }
402
612
 
613
+ export function resolveStaticDatasetColumnsForTableNamespace(
614
+ pipeline: PlayStaticPipeline | null | undefined,
615
+ tableNamespace: string | null | undefined,
616
+ ): PlayStaticDatasetColumn[] {
617
+ const requestedNamespace = tableNamespace?.trim();
618
+ if (!pipeline || !requestedNamespace) {
619
+ return [];
620
+ }
621
+ const normalizedNamespace = normalizeTableNamespace(requestedNamespace);
622
+ const seen = new Set<PlayStaticPipeline>();
623
+
624
+ const resolveFromPipeline = (
625
+ currentPipeline: PlayStaticPipeline | null | undefined,
626
+ ): PlayStaticDatasetColumn[] | null => {
627
+ if (!currentPipeline || seen.has(currentPipeline)) {
628
+ return null;
629
+ }
630
+ seen.add(currentPipeline);
631
+
632
+ const compiled = compileStaticGraph(currentPipeline);
633
+ const matchingDataset = compiled.datasets.find(
634
+ (dataset) =>
635
+ normalizeTableNamespace(dataset.tableNamespace) ===
636
+ normalizedNamespace,
637
+ );
638
+ if (matchingDataset) {
639
+ return matchingDataset.columns;
640
+ }
641
+
642
+ for (const substep of getCompiledPipelineSubsteps(currentPipeline)) {
643
+ if (substep.type === 'play_call') {
644
+ const nestedColumns = resolveFromPipeline(substep.pipeline);
645
+ if (nestedColumns) {
646
+ return nestedColumns;
647
+ }
648
+ }
649
+ }
650
+
651
+ return null;
652
+ };
653
+
654
+ return resolveFromPipeline(pipeline) ?? [];
655
+ }
656
+
403
657
  export function sqlSafePlayColumnName(id: string): string {
404
658
  const normalized = id
405
659
  .trim()
@@ -441,8 +695,14 @@ export function compileSheetContract(pipeline: PlayStaticPipeline): {
441
695
  const inputFields = pipeline.inputFields?.length
442
696
  ? pipeline.inputFields
443
697
  : [tableNamespace];
698
+ const rowKeyFieldSet = new Set(pipeline.rowKeyFields ?? []);
444
699
  for (const inputField of inputFields) {
445
- addColumn({ id: inputField, source: 'input', field: inputField });
700
+ addColumn({
701
+ id: inputField,
702
+ source: 'input',
703
+ field: inputField,
704
+ ...(rowKeyFieldSet.has(inputField) ? { isRowKey: true } : {}),
705
+ });
446
706
  }
447
707
 
448
708
  for (const field of pipeline.fields) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "deepline",
3
- "version": "0.1.77",
3
+ "version": "0.1.79",
4
4
  "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
5
5
  "license": "MIT",
6
6
  "repository": {