deepline 0.1.69 → 0.1.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -7
- package/dist/cli/index.js +95 -30
- package/dist/cli/index.mjs +95 -30
- package/dist/index.d.mts +60 -37
- package/dist/index.d.ts +60 -37
- package/dist/index.js +13 -3
- package/dist/index.mjs +12 -3
- package/dist/repo/apps/play-runner-workers/src/entry.ts +34 -21
- package/dist/repo/sdk/src/client.ts +3 -0
- package/dist/repo/sdk/src/index.ts +3 -2
- package/dist/repo/sdk/src/play.ts +70 -30
- package/dist/repo/sdk/src/release.ts +3 -3
- package/dist/repo/sdk/src/types.ts +1 -1
- package/dist/repo/sdk/src/worker-play-entry.ts +13 -2
- package/dist/repo/shared_libs/play-runtime/db-session-plan.ts +1 -1
- package/dist/repo/shared_libs/play-runtime/execution-plan.ts +2 -2
- package/dist/repo/shared_libs/play-runtime/step-lifecycle-tracker.ts +4 -4
- package/dist/repo/shared_libs/plays/dataset.ts +2 -2
- package/dist/repo/shared_libs/plays/row-identity.ts +3 -3
- package/dist/repo/shared_libs/plays/static-pipeline.ts +12 -9
- package/package.json +1 -1
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* the runtime API endpoint, not direct DB. That keeps the Worker bundle
|
|
23
23
|
* compatible with the V8 isolate runtime.
|
|
24
24
|
*
|
|
25
|
-
* Status: experimental. First cut targets tool-basic (ctx.csv + ctx.
|
|
25
|
+
* Status: experimental. First cut targets tool-basic (ctx.csv + ctx.dataset +
|
|
26
26
|
* ctx.tools.execute). Plays that depend on the full ctx surface (durable sleep,
|
|
27
27
|
* checkpoints, batched waterfalls, etc.) will fall back to "not implemented"
|
|
28
28
|
* rather than producing wrong results — opt-in via DEEPLINE_PLAY_RUNNER_BACKEND.
|
|
@@ -534,7 +534,7 @@ function getStaticSubstepNodeId(
|
|
|
534
534
|
switch (substep.type) {
|
|
535
535
|
case 'csv':
|
|
536
536
|
return `csv:${substep.field || index}`;
|
|
537
|
-
case '
|
|
537
|
+
case 'dataset':
|
|
538
538
|
return `map:${substep.tableNamespace ?? substep.field}`;
|
|
539
539
|
case 'tool':
|
|
540
540
|
return `tool:${substep.field}:${substep.toolId}`;
|
|
@@ -2938,7 +2938,7 @@ function requireSheetContract(
|
|
|
2938
2938
|
const contract = resolveSheetContractFromReq(req, tableNamespace);
|
|
2939
2939
|
if (!contract) {
|
|
2940
2940
|
throw new Error(
|
|
2941
|
-
`ctx.
|
|
2941
|
+
`ctx.dataset("${tableNamespace}"): no sheet contract resolvable from contractSnapshot. ` +
|
|
2942
2942
|
'This usually means the dispatcher did not pass a contractSnapshot for this play, ' +
|
|
2943
2943
|
'which is required for direct-Neon sheet IO from the Workers harness.',
|
|
2944
2944
|
);
|
|
@@ -3000,7 +3000,7 @@ function augmentSheetContractWithDatasetFields(input: {
|
|
|
3000
3000
|
columns.push({
|
|
3001
3001
|
id: `runtime:${input.contract.tableNamespace}:${field}`,
|
|
3002
3002
|
sqlName,
|
|
3003
|
-
source: outputFields.has(field) ? '
|
|
3003
|
+
source: outputFields.has(field) ? 'datasetColumn' : 'input',
|
|
3004
3004
|
field,
|
|
3005
3005
|
});
|
|
3006
3006
|
}
|
|
@@ -3107,7 +3107,7 @@ async function prepareMapRows(input: {
|
|
|
3107
3107
|
* Supported:
|
|
3108
3108
|
* - ctx.log(msg)
|
|
3109
3109
|
* - ctx.csv(filename | inline rows) (calls runtime API for file resolve)
|
|
3110
|
-
* - ctx.
|
|
3110
|
+
* - ctx.dataset(name, rows).withColumn(name, resolver).run(opts)
|
|
3111
3111
|
* - ctx.tools.execute({ id, tool, input, ... })
|
|
3112
3112
|
* - ctx.runPlay(key, playRef, input, opts)
|
|
3113
3113
|
*
|
|
@@ -3150,11 +3150,11 @@ function assertNotAborted(signal: AbortSignal | undefined): void {
|
|
|
3150
3150
|
}
|
|
3151
3151
|
}
|
|
3152
3152
|
|
|
3153
|
-
function
|
|
3153
|
+
function childPipelineUsesCtxDataset(
|
|
3154
3154
|
pipeline: PlayStaticPipeline | null | undefined,
|
|
3155
3155
|
): boolean {
|
|
3156
3156
|
return getCompiledPipelineSubsteps(pipeline).some(
|
|
3157
|
-
(substep) => substep.type === '
|
|
3157
|
+
(substep) => substep.type === 'dataset',
|
|
3158
3158
|
);
|
|
3159
3159
|
}
|
|
3160
3160
|
|
|
@@ -3418,7 +3418,7 @@ function createMinimalWorkerCtx(
|
|
|
3418
3418
|
const parts = Array.isArray(raw) ? raw : [raw];
|
|
3419
3419
|
if (parts.some((part) => part === null || part === undefined)) {
|
|
3420
3420
|
throw new Error(
|
|
3421
|
-
`ctx.
|
|
3421
|
+
`ctx.dataset("${name}") key returned null or undefined for row ${index}. ` +
|
|
3422
3422
|
'Return a non-empty string or number derived from a stable input column.',
|
|
3423
3423
|
);
|
|
3424
3424
|
}
|
|
@@ -3430,7 +3430,7 @@ function createMinimalWorkerCtx(
|
|
|
3430
3430
|
});
|
|
3431
3431
|
if (normalizedParts.some((part) => !part)) {
|
|
3432
3432
|
throw new Error(
|
|
3433
|
-
`ctx.
|
|
3433
|
+
`ctx.dataset("${name}") key returned an empty value for row ${index}. ` +
|
|
3434
3434
|
'Return a non-empty string or finite number derived from a stable input column.',
|
|
3435
3435
|
);
|
|
3436
3436
|
}
|
|
@@ -3462,7 +3462,7 @@ function createMinimalWorkerCtx(
|
|
|
3462
3462
|
const previousIndex = explicitRowKeysSeen?.get(keyValue);
|
|
3463
3463
|
if (previousIndex !== undefined) {
|
|
3464
3464
|
throw new Error(
|
|
3465
|
-
`ctx.
|
|
3465
|
+
`ctx.dataset("${name}") key function produced duplicate value "${keyValue}" for rows ${previousIndex} and ${index}. ` +
|
|
3466
3466
|
'Each row must produce a unique key. Combine columns (e.g. `${row.email}|${row.company}`) or pick a column that is unique per row.',
|
|
3467
3467
|
);
|
|
3468
3468
|
}
|
|
@@ -4016,7 +4016,7 @@ function createMinimalWorkerCtx(
|
|
|
4016
4016
|
return dataset;
|
|
4017
4017
|
};
|
|
4018
4018
|
|
|
4019
|
-
class
|
|
4019
|
+
class WorkerDatasetBuilder<T extends Record<string, unknown>> {
|
|
4020
4020
|
private readonly program: WorkerStepProgram = {
|
|
4021
4021
|
kind: 'steps',
|
|
4022
4022
|
steps: [],
|
|
@@ -4027,14 +4027,22 @@ function createMinimalWorkerCtx(
|
|
|
4027
4027
|
private readonly rows: WorkerDatasetInput<T>,
|
|
4028
4028
|
) {}
|
|
4029
4029
|
|
|
4030
|
-
|
|
4030
|
+
withColumn(name: string, resolver: WorkerStepProgramStep['resolver']): this {
|
|
4031
4031
|
if (!name.trim()) {
|
|
4032
|
-
throw new Error(
|
|
4032
|
+
throw new Error(
|
|
4033
|
+
'ctx.dataset(...).withColumn(name, ...) requires a name.',
|
|
4034
|
+
);
|
|
4033
4035
|
}
|
|
4034
4036
|
this.program.steps = [...this.program.steps, { name, resolver }];
|
|
4035
4037
|
return this;
|
|
4036
4038
|
}
|
|
4037
4039
|
|
|
4040
|
+
step(): never {
|
|
4041
|
+
throw new Error(
|
|
4042
|
+
'ctx.dataset(...).step(...) has been replaced by ctx.dataset(...).withColumn(...).',
|
|
4043
|
+
);
|
|
4044
|
+
}
|
|
4045
|
+
|
|
4038
4046
|
run(opts?: WorkerMapOptions): Promise<unknown> {
|
|
4039
4047
|
const fields = Object.fromEntries(
|
|
4040
4048
|
this.program.steps.map((step) => [step.name, step.resolver]),
|
|
@@ -4201,7 +4209,7 @@ function createMinimalWorkerCtx(
|
|
|
4201
4209
|
});
|
|
4202
4210
|
return dataset;
|
|
4203
4211
|
},
|
|
4204
|
-
|
|
4212
|
+
dataset<T extends Record<string, unknown>>(
|
|
4205
4213
|
name: string,
|
|
4206
4214
|
rows: WorkerDatasetInput<T>,
|
|
4207
4215
|
fieldsDef?:
|
|
@@ -4219,7 +4227,7 @@ function createMinimalWorkerCtx(
|
|
|
4219
4227
|
opts?: WorkerMapOptions,
|
|
4220
4228
|
): unknown {
|
|
4221
4229
|
if (arguments.length <= 2 || fieldsDef === undefined) {
|
|
4222
|
-
return new
|
|
4230
|
+
return new WorkerDatasetBuilder(name, rows);
|
|
4223
4231
|
}
|
|
4224
4232
|
if (isWorkerStepProgram(fieldsDef)) {
|
|
4225
4233
|
const fields = Object.fromEntries(
|
|
@@ -4228,7 +4236,12 @@ function createMinimalWorkerCtx(
|
|
|
4228
4236
|
return runMap(name, rows, fields, opts);
|
|
4229
4237
|
}
|
|
4230
4238
|
throw new Error(
|
|
4231
|
-
'ctx.
|
|
4239
|
+
'ctx.dataset(key, rows, fields, options) is not supported. Use ctx.dataset(key, rows).withColumn(...).run(options).',
|
|
4240
|
+
);
|
|
4241
|
+
},
|
|
4242
|
+
map(): never {
|
|
4243
|
+
throw new Error(
|
|
4244
|
+
'ctx.map(...) has been replaced by ctx.dataset(...). Use ctx.dataset(key, rows).withColumn(...).run(options).',
|
|
4232
4245
|
);
|
|
4233
4246
|
},
|
|
4234
4247
|
tools: {
|
|
@@ -4356,7 +4369,7 @@ function createMinimalWorkerCtx(
|
|
|
4356
4369
|
`ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: missing trusted Cloudflare child manifest from top-level submit.`,
|
|
4357
4370
|
);
|
|
4358
4371
|
}
|
|
4359
|
-
const
|
|
4372
|
+
const childIsDatasetBacked = childPipelineUsesCtxDataset(
|
|
4360
4373
|
childManifest.staticPipeline,
|
|
4361
4374
|
);
|
|
4362
4375
|
const childNeedsWorkflowScheduler = childPipelineNeedsWorkflowScheduler(
|
|
@@ -4364,13 +4377,13 @@ function createMinimalWorkerCtx(
|
|
|
4364
4377
|
);
|
|
4365
4378
|
let childConcurrencyAcquired = false;
|
|
4366
4379
|
let releaseChildPlaySlot: (() => void) | null = null;
|
|
4367
|
-
if (
|
|
4380
|
+
if (childIsDatasetBacked) {
|
|
4368
4381
|
const nextInFlight =
|
|
4369
4382
|
(inFlightChildCallsByPlayName[resolvedName] ?? 0) + 1;
|
|
4370
4383
|
if (nextInFlight > 1) {
|
|
4371
4384
|
throw new Error(
|
|
4372
|
-
`Concurrent
|
|
4373
|
-
'A child play that uses ctx.
|
|
4385
|
+
`Concurrent dataset-backed play call blocked for ${resolvedName}. ` +
|
|
4386
|
+
'A child play that uses ctx.dataset() cannot run more than once at the same time because its dataset tables share durable row identity. ' +
|
|
4374
4387
|
'Run these child play calls sequentially, or give each concurrent branch a different child play/table contract.',
|
|
4375
4388
|
);
|
|
4376
4389
|
}
|
|
@@ -4980,7 +4993,7 @@ async function executeRunRequest(
|
|
|
4980
4993
|
emit(event);
|
|
4981
4994
|
};
|
|
4982
4995
|
|
|
4983
|
-
stepLifecycle?.
|
|
4996
|
+
stepLifecycle?.markPreDatasetStepsStarted(startedAt);
|
|
4984
4997
|
flushLedgerEvents(false);
|
|
4985
4998
|
const ctx = createMinimalWorkerCtx(
|
|
4986
4999
|
req,
|
|
@@ -868,6 +868,9 @@ export class DeeplineClient {
|
|
|
868
868
|
? { packagedFiles: request.packagedFiles }
|
|
869
869
|
: {}),
|
|
870
870
|
...(request.force ? { force: true } : {}),
|
|
871
|
+
...(typeof request.waitForCompletionMs === 'number'
|
|
872
|
+
? { waitForCompletionMs: request.waitForCompletionMs }
|
|
873
|
+
: {}),
|
|
871
874
|
...(request.profile ? { profile: request.profile } : {}),
|
|
872
875
|
};
|
|
873
876
|
for await (const event of this.http.streamSse<PlayLiveEvent>(
|
|
@@ -95,6 +95,7 @@ export {
|
|
|
95
95
|
definePlay,
|
|
96
96
|
defineWorkflow,
|
|
97
97
|
getDefinedPlayMetadata,
|
|
98
|
+
runIf,
|
|
98
99
|
steps,
|
|
99
100
|
when,
|
|
100
101
|
} from './play.js';
|
|
@@ -135,14 +136,14 @@ export type {
|
|
|
135
136
|
DeeplinePlayRuntimeContext,
|
|
136
137
|
DefinedPlay,
|
|
137
138
|
ColumnMap,
|
|
139
|
+
ColumnResolver,
|
|
138
140
|
CsvInput,
|
|
141
|
+
DatasetBuilder,
|
|
139
142
|
PlayBindings,
|
|
140
143
|
FileInput,
|
|
141
144
|
PlayInputContract,
|
|
142
145
|
PlayJob,
|
|
143
146
|
ConditionalStepResolver,
|
|
144
|
-
MapStepBuilder,
|
|
145
|
-
MapStepResolver,
|
|
146
147
|
PlayDataset,
|
|
147
148
|
PlayDatasetInput,
|
|
148
149
|
PlayStepProgramStep,
|
|
@@ -254,37 +254,42 @@ export type PlayStepProgramStep = {
|
|
|
254
254
|
| StepProgramResolver<Record<string, unknown>, unknown>;
|
|
255
255
|
};
|
|
256
256
|
|
|
257
|
-
export type
|
|
257
|
+
export type ColumnResolver<Row, Value> =
|
|
258
258
|
| StepResolver<Row, Value>
|
|
259
259
|
| ConditionalStepResolver<Row, Value>
|
|
260
260
|
| StepProgramResolver<Row, Value>;
|
|
261
261
|
|
|
262
|
-
export type
|
|
262
|
+
export type DatasetBuilder<
|
|
263
263
|
InputRow extends object,
|
|
264
264
|
OutputRow extends object,
|
|
265
265
|
> = {
|
|
266
266
|
/**
|
|
267
|
-
* Define one output column for every row in this
|
|
267
|
+
* Define one output column for every row in this dataset.
|
|
268
268
|
*
|
|
269
269
|
* The `name` becomes a field on each output row. For example,
|
|
270
|
-
* `.
|
|
270
|
+
* `.withColumn('contact', ...)` creates `row.contact` in later column resolvers; it does
|
|
271
271
|
* not spread returned object fields such as `contact.email` into `row.email`.
|
|
272
272
|
* Add a later column resolver when you want a top-level export field:
|
|
273
|
-
* `.
|
|
273
|
+
* `.withColumn('email', row => row.contact?.email ?? null)`.
|
|
274
274
|
*
|
|
275
275
|
* @param name - Output column name.
|
|
276
276
|
* @param resolver - Computes the value for one row.
|
|
277
|
-
* @returns The same
|
|
277
|
+
* @returns The same dataset builder with the new column type.
|
|
278
278
|
*/
|
|
279
|
+
withColumn<Name extends string, Value>(
|
|
280
|
+
name: Name,
|
|
281
|
+
resolver: ColumnResolver<OutputRow, Value>,
|
|
282
|
+
): DatasetBuilder<InputRow, OutputRow & Record<Name, Value>>;
|
|
283
|
+
/** @deprecated Dataset `.step(...)` was replaced by `.withColumn(...)`. */
|
|
279
284
|
step<Name extends string, Value>(
|
|
280
285
|
name: Name,
|
|
281
|
-
resolver:
|
|
282
|
-
):
|
|
286
|
+
resolver: ColumnResolver<OutputRow, Value>,
|
|
287
|
+
): never;
|
|
283
288
|
/**
|
|
284
289
|
* Execute the row-column program and return a durable dataset handle.
|
|
285
290
|
*
|
|
286
291
|
* The returned {@link PlayDataset} preserves one output row per input row,
|
|
287
|
-
* with original fields merged with the columns produced by `.
|
|
292
|
+
* with original fields merged with the columns produced by `.withColumn(...)`.
|
|
288
293
|
*
|
|
289
294
|
* @param options - Run options.
|
|
290
295
|
* @returns Output rows as a dataset handle.
|
|
@@ -302,6 +307,7 @@ export type MapStepBuilder<
|
|
|
302
307
|
}): Promise<PlayDataset<OutputRow>>;
|
|
303
308
|
};
|
|
304
309
|
|
|
310
|
+
|
|
305
311
|
export type CsvRenameMap = Record<string, string | readonly string[]>;
|
|
306
312
|
|
|
307
313
|
/**
|
|
@@ -358,10 +364,10 @@ export type CsvOptions = {
|
|
|
358
364
|
* description: 'Look up company details by domain.',
|
|
359
365
|
* });
|
|
360
366
|
*
|
|
361
|
-
* // Fan-out: process items with named
|
|
367
|
+
* // Fan-out: process items with named columns
|
|
362
368
|
* const enriched = await ctx
|
|
363
|
-
* .
|
|
364
|
-
* .
|
|
369
|
+
* .dataset('companies', [{ domain: 'a.com' }, { domain: 'b.com' }])
|
|
370
|
+
* .withColumn('company', (row, rowCtx) =>
|
|
365
371
|
* rowCtx.tools.execute({
|
|
366
372
|
* id: 'company_search',
|
|
367
373
|
* tool: 'test_company_search',
|
|
@@ -391,7 +397,7 @@ export interface DeeplinePlayRuntimeContext {
|
|
|
391
397
|
* Load a staged CSV file as a durable dataset handle.
|
|
392
398
|
*
|
|
393
399
|
* Use this when a play receives a CSV path from the CLI or API and row work
|
|
394
|
-
* should continue through {@link DeeplinePlayRuntimeContext.
|
|
400
|
+
* should continue through {@link DeeplinePlayRuntimeContext.dataset}. The path is
|
|
395
401
|
* normally an input field such as `input.csv`, populated by
|
|
396
402
|
* `deepline plays run my.play.ts --csv rows.csv`.
|
|
397
403
|
*
|
|
@@ -403,7 +409,7 @@ export interface DeeplinePlayRuntimeContext {
|
|
|
403
409
|
* @param path - Staged CSV path.
|
|
404
410
|
* @param options - CSV load options.
|
|
405
411
|
*
|
|
406
|
-
* @returns A {@link PlayDataset} whose rows should usually flow directly into `ctx.
|
|
412
|
+
* @returns A {@link PlayDataset} whose rows should usually flow directly into `ctx.dataset(...)`.
|
|
407
413
|
*/
|
|
408
414
|
csv<T = Record<string, unknown>>(
|
|
409
415
|
path: string,
|
|
@@ -413,9 +419,9 @@ export interface DeeplinePlayRuntimeContext {
|
|
|
413
419
|
/**
|
|
414
420
|
* Create a persisted row dataset/table from input rows.
|
|
415
421
|
*
|
|
416
|
-
* `ctx.
|
|
422
|
+
* `ctx.dataset` is Deepline's row-work primitive. It records row identity,
|
|
417
423
|
* progress, retries, table output, and idempotency for a collection of rows.
|
|
418
|
-
* Use `.
|
|
424
|
+
* Use `.withColumn(name, resolver)` on the returned builder to define output
|
|
419
425
|
* columns, then `.run(...)` to execute the row program.
|
|
420
426
|
*
|
|
421
427
|
* The `key` identifies the logical dataset/table. Renaming it is a persistence
|
|
@@ -423,10 +429,10 @@ export interface DeeplinePlayRuntimeContext {
|
|
|
423
429
|
* automatically from input row content unless `.run({ key: ... })` overrides
|
|
424
430
|
* it with stable business fields such as `domain`, `email`, or `linkedin_url`.
|
|
425
431
|
*
|
|
426
|
-
* By default, `ctx.
|
|
432
|
+
* By default, `ctx.dataset` is row-preserving: one input row produces one output
|
|
427
433
|
* row, with original fields merged with the columns produced by
|
|
428
|
-
* `.
|
|
429
|
-
* documented expand/flatten recipe instead of assuming `ctx.
|
|
434
|
+
* `.withColumn(...)`. If one input entity must become many output rows, use the
|
|
435
|
+
* documented expand/flatten recipe instead of assuming `ctx.dataset` changes
|
|
430
436
|
* row cardinality.
|
|
431
437
|
*
|
|
432
438
|
* @typeParam T - Row type
|
|
@@ -437,8 +443,8 @@ export interface DeeplinePlayRuntimeContext {
|
|
|
437
443
|
* @example Single tool per row
|
|
438
444
|
* ```typescript
|
|
439
445
|
* const results = await ctx
|
|
440
|
-
* .
|
|
441
|
-
* .
|
|
446
|
+
* .dataset('companies', leads)
|
|
447
|
+
* .withColumn('company', (row, ctx) =>
|
|
442
448
|
* ctx.tools.execute({
|
|
443
449
|
* id: 'company_search',
|
|
444
450
|
* tool: 'test_company_search',
|
|
@@ -452,23 +458,30 @@ export interface DeeplinePlayRuntimeContext {
|
|
|
452
458
|
* @example Multiple columns with pre/post logic
|
|
453
459
|
* ```typescript
|
|
454
460
|
* const results = await ctx
|
|
455
|
-
* .
|
|
456
|
-
* .
|
|
461
|
+
* .dataset('leads', leads)
|
|
462
|
+
* .withColumn('company', (row, ctx) =>
|
|
457
463
|
* ctx.tools.execute({
|
|
458
464
|
* id: 'company_search',
|
|
459
465
|
* tool: 'test_company_search',
|
|
460
466
|
* input: { domain: row.domain },
|
|
461
467
|
* description: 'Look up company details by domain.',
|
|
462
468
|
* }))
|
|
463
|
-
* .
|
|
469
|
+
* .withColumn('score', (row) =>
|
|
464
470
|
* row.company?.employeeCount > 100 ? 'enterprise' : 'smb')
|
|
465
471
|
* .run({ description: 'Enrich leads.' });
|
|
466
472
|
* ```
|
|
467
473
|
*/
|
|
474
|
+
dataset<TItem extends object>(
|
|
475
|
+
key: string,
|
|
476
|
+
items: PlayDatasetInput<TItem>,
|
|
477
|
+
): DatasetBuilder<TItem, TItem>;
|
|
478
|
+
/**
|
|
479
|
+
* @deprecated `ctx.map(...)` was replaced by `ctx.dataset(...)`.
|
|
480
|
+
*/
|
|
468
481
|
map<TItem extends object>(
|
|
469
482
|
key: string,
|
|
470
483
|
items: PlayDatasetInput<TItem>,
|
|
471
|
-
):
|
|
484
|
+
): never;
|
|
472
485
|
|
|
473
486
|
/** Tool execution namespace. */
|
|
474
487
|
tools: {
|
|
@@ -499,8 +512,8 @@ export interface DeeplinePlayRuntimeContext {
|
|
|
499
512
|
*
|
|
500
513
|
* `steps().step(...)` is a composable mini-pipeline. Use `ctx.runSteps(...)`
|
|
501
514
|
* when that mini-pipeline should execute outside a row dataset. Inside a
|
|
502
|
-
* `ctx.
|
|
503
|
-
* `.
|
|
515
|
+
* `ctx.dataset` column resolver, pass the step program directly to
|
|
516
|
+
* `.withColumn(name, program)` instead.
|
|
504
517
|
*
|
|
505
518
|
* @param program - Step program.
|
|
506
519
|
* @param input - Program input.
|
|
@@ -521,7 +534,7 @@ export interface DeeplinePlayRuntimeContext {
|
|
|
521
534
|
* value instead of running `run` again.
|
|
522
535
|
*
|
|
523
536
|
* Plain deterministic assignment does not need `ctx.step`. Use
|
|
524
|
-
* `ctx.
|
|
537
|
+
* `ctx.dataset(...).withColumn(...)`, not `ctx.step`, when the value should become a
|
|
525
538
|
* field on each exported row.
|
|
526
539
|
*
|
|
527
540
|
* @param id - Checkpoint id.
|
|
@@ -561,6 +574,22 @@ export interface DeeplinePlayRuntimeContext {
|
|
|
561
574
|
bodyText: string;
|
|
562
575
|
json: unknown | null;
|
|
563
576
|
}>;
|
|
577
|
+
/**
|
|
578
|
+
* Invoke another registered or file-backed play as a child workflow.
|
|
579
|
+
*
|
|
580
|
+
* Use this for real composition boundaries, especially when a fitting
|
|
581
|
+
* scalar prebuilt play already encodes provider order, fallbacks,
|
|
582
|
+
* normalization, and no-result behavior. Do not invoke plays through
|
|
583
|
+
* `ctx.tools.execute`; tools and plays are separate namespaces.
|
|
584
|
+
*
|
|
585
|
+
* `key` is the stable child-call identity for idempotency and traceability.
|
|
586
|
+
*
|
|
587
|
+
* @param key - Stable child-call key.
|
|
588
|
+
* @param playRef - Registered play name, play handle, or file-backed play reference.
|
|
589
|
+
* @param input - Input object passed to the child play.
|
|
590
|
+
* @param options - Child play options.
|
|
591
|
+
* @returns Child play output.
|
|
592
|
+
*/
|
|
564
593
|
secrets: {
|
|
565
594
|
get(name: string): SecretHandle;
|
|
566
595
|
bearer(secret: SecretHandle): SecretAuth;
|
|
@@ -869,6 +898,17 @@ export function steps<TInput>(): StepProgram<TInput, TInput, TInput> {
|
|
|
869
898
|
export function when<Row, Value>(
|
|
870
899
|
predicate: (row: Row, index: number) => boolean | Promise<boolean>,
|
|
871
900
|
resolver: StepResolver<Row, Value>,
|
|
901
|
+
): never {
|
|
902
|
+
void predicate;
|
|
903
|
+
void resolver;
|
|
904
|
+
throw new Error(
|
|
905
|
+
'when(...) has been replaced by runIf(...). Use runIf(predicate, resolver).',
|
|
906
|
+
);
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
export function runIf<Row, Value>(
|
|
910
|
+
predicate: (row: Row, index: number) => boolean | Promise<boolean>,
|
|
911
|
+
resolver: StepResolver<Row, Value>,
|
|
872
912
|
): ConditionalStepResolver<Row, Value, null> {
|
|
873
913
|
return new DeeplineConditionalStepResolver(predicate, resolver, null);
|
|
874
914
|
}
|
|
@@ -1353,8 +1393,8 @@ export function defineInput<TInput>(
|
|
|
1353
1393
|
* const leads = await ctx.csv(input.csv);
|
|
1354
1394
|
* ctx.log(`Processing ${await leads.count()} rows`);
|
|
1355
1395
|
* const results = await ctx
|
|
1356
|
-
* .
|
|
1357
|
-
* .
|
|
1396
|
+
* .dataset('companies', leads)
|
|
1397
|
+
* .withColumn('company', (row, ctx) =>
|
|
1358
1398
|
* ctx.tools.execute({
|
|
1359
1399
|
* id: 'company_search',
|
|
1360
1400
|
* tool: 'test_company_search',
|
|
@@ -50,10 +50,10 @@ export type SdkRelease = {
|
|
|
50
50
|
};
|
|
51
51
|
|
|
52
52
|
export const SDK_RELEASE = {
|
|
53
|
-
version: '0.1.
|
|
54
|
-
apiContract: '2026-
|
|
53
|
+
version: '0.1.71',
|
|
54
|
+
apiContract: '2026-06-dataset-column-syntax-cutover',
|
|
55
55
|
supportPolicy: {
|
|
56
|
-
latest: '0.1.
|
|
56
|
+
latest: '0.1.71',
|
|
57
57
|
minimumSupported: '0.1.53',
|
|
58
58
|
deprecatedBelow: '0.1.53',
|
|
59
59
|
},
|
|
@@ -718,7 +718,7 @@ export interface PlayDetail {
|
|
|
718
718
|
}
|
|
719
719
|
|
|
720
720
|
export interface ClearPlayHistoryRequest {
|
|
721
|
-
/** Optional explicit ctx.
|
|
721
|
+
/** Optional explicit ctx.dataset keys to clear. Omit to clear all discovered sheets for the play. */
|
|
722
722
|
tableNamespaces?: string[];
|
|
723
723
|
}
|
|
724
724
|
|
|
@@ -31,8 +31,8 @@ export type {
|
|
|
31
31
|
ColumnMap,
|
|
32
32
|
CsvInput,
|
|
33
33
|
FileInput,
|
|
34
|
-
|
|
35
|
-
|
|
34
|
+
ColumnResolver,
|
|
35
|
+
DatasetBuilder,
|
|
36
36
|
PlayBindings,
|
|
37
37
|
PlayDataset,
|
|
38
38
|
PlayDatasetInput,
|
|
@@ -128,6 +128,17 @@ export function steps<TInput>(): StepProgram<TInput, TInput, TInput> {
|
|
|
128
128
|
export function when<Row, Value>(
|
|
129
129
|
predicate: (row: Row, index: number) => boolean | Promise<boolean>,
|
|
130
130
|
resolver: StepResolver<Row, Value>,
|
|
131
|
+
): never {
|
|
132
|
+
void predicate;
|
|
133
|
+
void resolver;
|
|
134
|
+
throw new Error(
|
|
135
|
+
'when(...) has been replaced by runIf(...). Use runIf(predicate, resolver).',
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function runIf<Row, Value>(
|
|
140
|
+
predicate: (row: Row, index: number) => boolean | Promise<boolean>,
|
|
141
|
+
resolver: StepResolver<Row, Value>,
|
|
131
142
|
): ConditionalStepResolver<Row, Value, null> {
|
|
132
143
|
return new WorkerConditionalStepResolver(predicate, resolver, null);
|
|
133
144
|
}
|
|
@@ -29,7 +29,7 @@ export function planRuntimeSheetDbSessionRequirements(
|
|
|
29
29
|
}
|
|
30
30
|
const byNamespace = new Map<string, PlaySheetContract>();
|
|
31
31
|
for (const substep of flattenStaticPipeline(pipeline)) {
|
|
32
|
-
if (substep.type !== '
|
|
32
|
+
if (substep.type !== 'dataset') continue;
|
|
33
33
|
const tableNamespace = (
|
|
34
34
|
substep.tableNamespace ??
|
|
35
35
|
substep.field ??
|
|
@@ -200,8 +200,8 @@ function extractPlanMaps(
|
|
|
200
200
|
);
|
|
201
201
|
return substeps
|
|
202
202
|
.filter(
|
|
203
|
-
(substep): substep is Extract<PlayStaticSubstep, { type: '
|
|
204
|
-
substep.type === '
|
|
203
|
+
(substep): substep is Extract<PlayStaticSubstep, { type: 'dataset' }> =>
|
|
204
|
+
substep.type === 'dataset',
|
|
205
205
|
)
|
|
206
206
|
.map((mapSubstep) => {
|
|
207
207
|
const waterfalls = fallbackWaterfalls.filter((waterfall) => {
|
|
@@ -32,7 +32,7 @@ export class PlayStepLifecycleTracker {
|
|
|
32
32
|
private readonly now: () => number = Date.now,
|
|
33
33
|
) {}
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
markPreDatasetStepsStarted(at = this.now()): void {
|
|
36
36
|
for (const node of this.nodes) {
|
|
37
37
|
if (!isAutoStartedSetupNode(node.type)) break;
|
|
38
38
|
if (!this.getProgress()[node.nodeId]?.startedAt) {
|
|
@@ -106,7 +106,7 @@ export class PlayStepLifecycleTracker {
|
|
|
106
106
|
for (let index = 0; index < mapIndex; index += 1) {
|
|
107
107
|
const node = this.nodes[index]!;
|
|
108
108
|
if (
|
|
109
|
-
node.type !== '
|
|
109
|
+
node.type !== 'dataset' &&
|
|
110
110
|
this.getProgress()[node.nodeId]?.startedAt &&
|
|
111
111
|
!this.getProgress()[node.nodeId]?.completedAt &&
|
|
112
112
|
!this.failedNodeIds.has(node.nodeId)
|
|
@@ -143,7 +143,7 @@ export class PlayStepLifecycleTracker {
|
|
|
143
143
|
}
|
|
144
144
|
for (let index = mapIndex + 1; index < this.nodes.length; index += 1) {
|
|
145
145
|
const node = this.nodes[index]!;
|
|
146
|
-
if (node.type === '
|
|
146
|
+
if (node.type === 'dataset') break;
|
|
147
147
|
if (
|
|
148
148
|
isAutoStartedSetupNode(node.type) &&
|
|
149
149
|
!this.getProgress()[node.nodeId]?.startedAt
|
|
@@ -210,7 +210,7 @@ export class PlayStepLifecycleTracker {
|
|
|
210
210
|
|
|
211
211
|
private completeStartedNonMapNodes(at: number): void {
|
|
212
212
|
for (const node of this.nodes) {
|
|
213
|
-
if (node.type === '
|
|
213
|
+
if (node.type === 'dataset') {
|
|
214
214
|
continue;
|
|
215
215
|
}
|
|
216
216
|
const existing = this.getProgress()[node.nodeId];
|
|
@@ -61,13 +61,13 @@ export type PlayDatasetTransformOptions = {
|
|
|
61
61
|
};
|
|
62
62
|
|
|
63
63
|
/**
|
|
64
|
-
* Durable handle for rows produced by `ctx.csv(...)` or `ctx.
|
|
64
|
+
* Durable handle for rows produced by `ctx.csv(...)` or `ctx.dataset(...).run()`.
|
|
65
65
|
*
|
|
66
66
|
* A `PlayDataset` is not a normal in-memory array. It points at runtime-managed
|
|
67
67
|
* rows, usually backed by persisted sheet storage, and carries metadata such as
|
|
68
68
|
* dataset kind, dataset id, table namespace, count, and preview rows.
|
|
69
69
|
*
|
|
70
|
-
* Pass dataset handles directly into later `ctx.
|
|
70
|
+
* Pass dataset handles directly into later `ctx.dataset(...)` stages by default so
|
|
71
71
|
* Deepline keeps row progress, retries, memory use, and table output under
|
|
72
72
|
* runtime control. Use `count()` and `peek()` for bounded inspection. Use
|
|
73
73
|
* `materialize(limit)` or async iteration only when the dataset is intentionally
|
|
@@ -187,7 +187,7 @@ export function normalizePlayNameForSheet(value: string): string {
|
|
|
187
187
|
export function normalizeTableNamespace(value: string): string {
|
|
188
188
|
return validateIdentifierPart(
|
|
189
189
|
value,
|
|
190
|
-
'ctx.
|
|
190
|
+
'ctx.dataset() key',
|
|
191
191
|
MAP_KEY_NAMESPACE_MAX_LENGTH,
|
|
192
192
|
);
|
|
193
193
|
}
|
|
@@ -208,7 +208,7 @@ export function resolveStaleMapTableNamespace(
|
|
|
208
208
|
staleAfterSeconds <= 0
|
|
209
209
|
) {
|
|
210
210
|
throw new Error(
|
|
211
|
-
'ctx.
|
|
211
|
+
'ctx.dataset() staleAfterSeconds must be a positive whole number of seconds.',
|
|
212
212
|
);
|
|
213
213
|
}
|
|
214
214
|
|
|
@@ -242,7 +242,7 @@ export function validatePlaySheetTableName(
|
|
|
242
242
|
if (resolved.length > POSTGRES_IDENTIFIER_MAX_LENGTH) {
|
|
243
243
|
throw new Error(
|
|
244
244
|
`Play sheet table name is too long after normalization (${resolved.length}/63). ` +
|
|
245
|
-
`Shorten the play name or ctx.
|
|
245
|
+
`Shorten the play name or ctx.dataset() key. ` +
|
|
246
246
|
`Resolved table name: "${resolved}".`,
|
|
247
247
|
);
|
|
248
248
|
}
|