deepline 0.1.73 → 0.1.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/dist/cli/index.js +1297 -350
- package/dist/cli/index.mjs +1299 -352
- package/dist/index.d.mts +52 -1
- package/dist/index.d.ts +52 -1
- package/dist/index.js +16 -5
- package/dist/index.mjs +16 -5
- package/dist/repo/apps/play-runner-workers/src/entry.ts +141 -63
- package/dist/repo/sdk/src/client.ts +8 -0
- package/dist/repo/sdk/src/play.ts +59 -9
- package/dist/repo/sdk/src/plays/harness-stub.ts +1 -0
- package/dist/repo/sdk/src/release.ts +3 -3
- package/dist/repo/sdk/src/worker-play-entry.ts +39 -3
- package/dist/repo/shared_libs/play-runtime/cell-staleness.ts +88 -0
- package/dist/repo/shared_libs/play-runtime/step-program-dataset-builder.ts +130 -0
- package/dist/repo/shared_libs/plays/row-identity.ts +0 -40
- package/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -1051,6 +1051,43 @@ interface PlayStagedFileRef {
|
|
|
1051
1051
|
bytes: number;
|
|
1052
1052
|
}
|
|
1053
1053
|
|
|
1054
|
+
type EnrichStepCommand = {
|
|
1055
|
+
alias: string;
|
|
1056
|
+
tool: string;
|
|
1057
|
+
operation?: string;
|
|
1058
|
+
payload: Record<string, unknown>;
|
|
1059
|
+
extract_js?: string;
|
|
1060
|
+
run_if_js?: string;
|
|
1061
|
+
description?: string;
|
|
1062
|
+
disabled?: boolean;
|
|
1063
|
+
};
|
|
1064
|
+
type EnrichWaterfallCommand = {
|
|
1065
|
+
with_waterfall: string;
|
|
1066
|
+
min_results?: number;
|
|
1067
|
+
commands: EnrichCommand[];
|
|
1068
|
+
description?: string;
|
|
1069
|
+
};
|
|
1070
|
+
type EnrichCommand = EnrichStepCommand | EnrichWaterfallCommand;
|
|
1071
|
+
type EnrichCompiledConfig = {
|
|
1072
|
+
version: 1;
|
|
1073
|
+
commands: EnrichCommand[];
|
|
1074
|
+
cost_cap_usd_per_run?: number;
|
|
1075
|
+
_comments?: Array<{
|
|
1076
|
+
path: string;
|
|
1077
|
+
lines: string[];
|
|
1078
|
+
}>;
|
|
1079
|
+
_expansion_preview?: {
|
|
1080
|
+
plays: Array<{
|
|
1081
|
+
alias: string;
|
|
1082
|
+
tool_id: string;
|
|
1083
|
+
template_group: string;
|
|
1084
|
+
runtime_group: string;
|
|
1085
|
+
estimated_credits_range: string;
|
|
1086
|
+
steps: Array<Record<string, unknown>>;
|
|
1087
|
+
}>;
|
|
1088
|
+
};
|
|
1089
|
+
};
|
|
1090
|
+
|
|
1054
1091
|
type ExecuteToolRawOptions = {
|
|
1055
1092
|
includeToolMetadata?: boolean;
|
|
1056
1093
|
};
|
|
@@ -1350,6 +1387,12 @@ declare class DeeplineClient {
|
|
|
1350
1387
|
sourceFiles?: Record<string, string>;
|
|
1351
1388
|
artifact: Record<string, unknown>;
|
|
1352
1389
|
}): Promise<PlayCheckResult>;
|
|
1390
|
+
compileEnrichPlan(input: {
|
|
1391
|
+
plan_args?: string[];
|
|
1392
|
+
config?: unknown;
|
|
1393
|
+
}): Promise<{
|
|
1394
|
+
config: EnrichCompiledConfig;
|
|
1395
|
+
}>;
|
|
1353
1396
|
startPlayRunFromBundle(input: {
|
|
1354
1397
|
name: string;
|
|
1355
1398
|
sourceCode: string;
|
|
@@ -2143,12 +2186,17 @@ type ConditionalStepResolver<Row, Value, Else = null> = {
|
|
|
2143
2186
|
readonly elseValue: Else;
|
|
2144
2187
|
else<ValueElse>(value: ValueElse): ConditionalStepResolver<Row, Value, ValueElse>;
|
|
2145
2188
|
};
|
|
2189
|
+
type StepOptions<Row> = {
|
|
2190
|
+
readonly runIf?: (row: Row, index: number) => boolean | Promise<boolean>;
|
|
2191
|
+
readonly staleAfterSeconds?: number;
|
|
2192
|
+
};
|
|
2146
2193
|
type StepProgram<Input, Output, Return = Output> = {
|
|
2147
2194
|
readonly kind: 'steps';
|
|
2148
2195
|
readonly steps: readonly PlayStepProgramStep[];
|
|
2149
2196
|
readonly returnResolver?: StepResolver<Output, Return>;
|
|
2150
2197
|
readonly __inputType?: (input: Input) => void;
|
|
2151
2198
|
step<Name extends string, Value>(name: Name, resolver: StepResolver<Output, Value> | ConditionalStepResolver<Output, Value> | StepProgramResolver<Output, Value>): StepProgram<Input, Output & Record<Name, Value>, Return>;
|
|
2199
|
+
step<Name extends string, Value>(name: Name, resolver: StepResolver<Output, Value> | StepProgramResolver<Output, Value>, options: StepOptions<Output>): StepProgram<Input, Output & Record<Name, Value | null>, Return>;
|
|
2152
2200
|
return<Value>(resolver: StepResolver<Output, Value>): StepProgram<Input, Output, Value>;
|
|
2153
2201
|
};
|
|
2154
2202
|
type StepProgramResolver<Input, Return> = {
|
|
@@ -2159,9 +2207,11 @@ type StepProgramResolver<Input, Return> = {
|
|
|
2159
2207
|
};
|
|
2160
2208
|
type PlayStepProgramStep = {
|
|
2161
2209
|
readonly name: string;
|
|
2210
|
+
readonly staleAfterSeconds?: number;
|
|
2162
2211
|
readonly resolver: StepResolver<Record<string, unknown>, unknown> | ConditionalStepResolver<Record<string, unknown>, unknown> | StepProgramResolver<Record<string, unknown>, unknown>;
|
|
2163
2212
|
};
|
|
2164
2213
|
type ColumnResolver<Row, Value> = StepResolver<Row, Value> | ConditionalStepResolver<Row, Value> | StepProgramResolver<Row, Value>;
|
|
2214
|
+
type StepProgramOutput<TProgram> = TProgram extends StepProgram<any, infer Output, any> ? Output : never;
|
|
2165
2215
|
type DatasetBuilder<InputRow extends object, OutputRow extends object> = {
|
|
2166
2216
|
/**
|
|
2167
2217
|
* Define one output column for every row in this dataset.
|
|
@@ -2177,6 +2227,8 @@ type DatasetBuilder<InputRow extends object, OutputRow extends object> = {
|
|
|
2177
2227
|
* @returns The same dataset builder with the new column type.
|
|
2178
2228
|
*/
|
|
2179
2229
|
withColumn<Name extends string, Value>(name: Name, resolver: ColumnResolver<OutputRow, Value>): DatasetBuilder<InputRow, OutputRow & Record<Name, Value>>;
|
|
2230
|
+
withColumn<Name extends string, Value>(name: Name, resolver: StepResolver<OutputRow, Value> | StepProgramResolver<OutputRow, Value>, options: StepOptions<OutputRow>): DatasetBuilder<InputRow, OutputRow & Record<Name, Value | null>>;
|
|
2231
|
+
withColumns<Program extends StepProgram<OutputRow, object, unknown>>(program: Program): DatasetBuilder<InputRow, StepProgramOutput<Program>>;
|
|
2180
2232
|
/** @deprecated Dataset `.step(...)` was replaced by `.withColumn(...)`. */
|
|
2181
2233
|
step<Name extends string, Value>(name: Name, resolver: ColumnResolver<OutputRow, Value>): never;
|
|
2182
2234
|
/**
|
|
@@ -2190,7 +2242,6 @@ type DatasetBuilder<InputRow extends object, OutputRow extends object> = {
|
|
|
2190
2242
|
*/
|
|
2191
2243
|
run(options?: {
|
|
2192
2244
|
description?: string;
|
|
2193
|
-
staleAfterSeconds?: number;
|
|
2194
2245
|
key?: (keyof InputRow & string) | readonly (keyof InputRow & string)[] | ((row: InputRow, index: number) => string | number | readonly unknown[]);
|
|
2195
2246
|
}): Promise<PlayDataset<OutputRow>>;
|
|
2196
2247
|
};
|
package/dist/index.d.ts
CHANGED
|
@@ -1051,6 +1051,43 @@ interface PlayStagedFileRef {
|
|
|
1051
1051
|
bytes: number;
|
|
1052
1052
|
}
|
|
1053
1053
|
|
|
1054
|
+
type EnrichStepCommand = {
|
|
1055
|
+
alias: string;
|
|
1056
|
+
tool: string;
|
|
1057
|
+
operation?: string;
|
|
1058
|
+
payload: Record<string, unknown>;
|
|
1059
|
+
extract_js?: string;
|
|
1060
|
+
run_if_js?: string;
|
|
1061
|
+
description?: string;
|
|
1062
|
+
disabled?: boolean;
|
|
1063
|
+
};
|
|
1064
|
+
type EnrichWaterfallCommand = {
|
|
1065
|
+
with_waterfall: string;
|
|
1066
|
+
min_results?: number;
|
|
1067
|
+
commands: EnrichCommand[];
|
|
1068
|
+
description?: string;
|
|
1069
|
+
};
|
|
1070
|
+
type EnrichCommand = EnrichStepCommand | EnrichWaterfallCommand;
|
|
1071
|
+
type EnrichCompiledConfig = {
|
|
1072
|
+
version: 1;
|
|
1073
|
+
commands: EnrichCommand[];
|
|
1074
|
+
cost_cap_usd_per_run?: number;
|
|
1075
|
+
_comments?: Array<{
|
|
1076
|
+
path: string;
|
|
1077
|
+
lines: string[];
|
|
1078
|
+
}>;
|
|
1079
|
+
_expansion_preview?: {
|
|
1080
|
+
plays: Array<{
|
|
1081
|
+
alias: string;
|
|
1082
|
+
tool_id: string;
|
|
1083
|
+
template_group: string;
|
|
1084
|
+
runtime_group: string;
|
|
1085
|
+
estimated_credits_range: string;
|
|
1086
|
+
steps: Array<Record<string, unknown>>;
|
|
1087
|
+
}>;
|
|
1088
|
+
};
|
|
1089
|
+
};
|
|
1090
|
+
|
|
1054
1091
|
type ExecuteToolRawOptions = {
|
|
1055
1092
|
includeToolMetadata?: boolean;
|
|
1056
1093
|
};
|
|
@@ -1350,6 +1387,12 @@ declare class DeeplineClient {
|
|
|
1350
1387
|
sourceFiles?: Record<string, string>;
|
|
1351
1388
|
artifact: Record<string, unknown>;
|
|
1352
1389
|
}): Promise<PlayCheckResult>;
|
|
1390
|
+
compileEnrichPlan(input: {
|
|
1391
|
+
plan_args?: string[];
|
|
1392
|
+
config?: unknown;
|
|
1393
|
+
}): Promise<{
|
|
1394
|
+
config: EnrichCompiledConfig;
|
|
1395
|
+
}>;
|
|
1353
1396
|
startPlayRunFromBundle(input: {
|
|
1354
1397
|
name: string;
|
|
1355
1398
|
sourceCode: string;
|
|
@@ -2143,12 +2186,17 @@ type ConditionalStepResolver<Row, Value, Else = null> = {
|
|
|
2143
2186
|
readonly elseValue: Else;
|
|
2144
2187
|
else<ValueElse>(value: ValueElse): ConditionalStepResolver<Row, Value, ValueElse>;
|
|
2145
2188
|
};
|
|
2189
|
+
type StepOptions<Row> = {
|
|
2190
|
+
readonly runIf?: (row: Row, index: number) => boolean | Promise<boolean>;
|
|
2191
|
+
readonly staleAfterSeconds?: number;
|
|
2192
|
+
};
|
|
2146
2193
|
type StepProgram<Input, Output, Return = Output> = {
|
|
2147
2194
|
readonly kind: 'steps';
|
|
2148
2195
|
readonly steps: readonly PlayStepProgramStep[];
|
|
2149
2196
|
readonly returnResolver?: StepResolver<Output, Return>;
|
|
2150
2197
|
readonly __inputType?: (input: Input) => void;
|
|
2151
2198
|
step<Name extends string, Value>(name: Name, resolver: StepResolver<Output, Value> | ConditionalStepResolver<Output, Value> | StepProgramResolver<Output, Value>): StepProgram<Input, Output & Record<Name, Value>, Return>;
|
|
2199
|
+
step<Name extends string, Value>(name: Name, resolver: StepResolver<Output, Value> | StepProgramResolver<Output, Value>, options: StepOptions<Output>): StepProgram<Input, Output & Record<Name, Value | null>, Return>;
|
|
2152
2200
|
return<Value>(resolver: StepResolver<Output, Value>): StepProgram<Input, Output, Value>;
|
|
2153
2201
|
};
|
|
2154
2202
|
type StepProgramResolver<Input, Return> = {
|
|
@@ -2159,9 +2207,11 @@ type StepProgramResolver<Input, Return> = {
|
|
|
2159
2207
|
};
|
|
2160
2208
|
type PlayStepProgramStep = {
|
|
2161
2209
|
readonly name: string;
|
|
2210
|
+
readonly staleAfterSeconds?: number;
|
|
2162
2211
|
readonly resolver: StepResolver<Record<string, unknown>, unknown> | ConditionalStepResolver<Record<string, unknown>, unknown> | StepProgramResolver<Record<string, unknown>, unknown>;
|
|
2163
2212
|
};
|
|
2164
2213
|
type ColumnResolver<Row, Value> = StepResolver<Row, Value> | ConditionalStepResolver<Row, Value> | StepProgramResolver<Row, Value>;
|
|
2214
|
+
type StepProgramOutput<TProgram> = TProgram extends StepProgram<any, infer Output, any> ? Output : never;
|
|
2165
2215
|
type DatasetBuilder<InputRow extends object, OutputRow extends object> = {
|
|
2166
2216
|
/**
|
|
2167
2217
|
* Define one output column for every row in this dataset.
|
|
@@ -2177,6 +2227,8 @@ type DatasetBuilder<InputRow extends object, OutputRow extends object> = {
|
|
|
2177
2227
|
* @returns The same dataset builder with the new column type.
|
|
2178
2228
|
*/
|
|
2179
2229
|
withColumn<Name extends string, Value>(name: Name, resolver: ColumnResolver<OutputRow, Value>): DatasetBuilder<InputRow, OutputRow & Record<Name, Value>>;
|
|
2230
|
+
withColumn<Name extends string, Value>(name: Name, resolver: StepResolver<OutputRow, Value> | StepProgramResolver<OutputRow, Value>, options: StepOptions<OutputRow>): DatasetBuilder<InputRow, OutputRow & Record<Name, Value | null>>;
|
|
2231
|
+
withColumns<Program extends StepProgram<OutputRow, object, unknown>>(program: Program): DatasetBuilder<InputRow, StepProgramOutput<Program>>;
|
|
2180
2232
|
/** @deprecated Dataset `.step(...)` was replaced by `.withColumn(...)`. */
|
|
2181
2233
|
step<Name extends string, Value>(name: Name, resolver: ColumnResolver<OutputRow, Value>): never;
|
|
2182
2234
|
/**
|
|
@@ -2190,7 +2242,6 @@ type DatasetBuilder<InputRow extends object, OutputRow extends object> = {
|
|
|
2190
2242
|
*/
|
|
2191
2243
|
run(options?: {
|
|
2192
2244
|
description?: string;
|
|
2193
|
-
staleAfterSeconds?: number;
|
|
2194
2245
|
key?: (keyof InputRow & string) | readonly (keyof InputRow & string)[] | ((row: InputRow, index: number) => string | number | readonly unknown[]);
|
|
2195
2246
|
}): Promise<PlayDataset<OutputRow>>;
|
|
2196
2247
|
};
|
package/dist/index.js
CHANGED
|
@@ -241,10 +241,10 @@ var import_node_path2 = require("path");
|
|
|
241
241
|
|
|
242
242
|
// src/release.ts
|
|
243
243
|
var SDK_RELEASE = {
|
|
244
|
-
version: "0.1.
|
|
245
|
-
apiContract: "2026-06-dataset-column-
|
|
244
|
+
version: "0.1.76",
|
|
245
|
+
apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
|
|
246
246
|
supportPolicy: {
|
|
247
|
-
latest: "0.1.
|
|
247
|
+
latest: "0.1.76",
|
|
248
248
|
minimumSupported: "0.1.53",
|
|
249
249
|
deprecatedBelow: "0.1.53"
|
|
250
250
|
}
|
|
@@ -1127,6 +1127,9 @@ var DeeplineClient = class {
|
|
|
1127
1127
|
async checkPlayArtifact(input) {
|
|
1128
1128
|
return this.http.post("/api/v2/plays/check", input);
|
|
1129
1129
|
}
|
|
1130
|
+
async compileEnrichPlan(input) {
|
|
1131
|
+
return this.http.post("/api/v2/enrich/compile", input);
|
|
1132
|
+
}
|
|
1130
1133
|
async startPlayRunFromBundle(input) {
|
|
1131
1134
|
const compilerManifest = input.compilerManifest ?? await this.compilePlayManifest({
|
|
1132
1135
|
name: input.name,
|
|
@@ -2410,18 +2413,23 @@ var DeeplineStepProgram = class _DeeplineStepProgram {
|
|
|
2410
2413
|
steps;
|
|
2411
2414
|
returnResolver;
|
|
2412
2415
|
kind = "steps";
|
|
2413
|
-
step(name, resolver) {
|
|
2416
|
+
step(name, resolver, options) {
|
|
2414
2417
|
if (!name.trim()) {
|
|
2415
2418
|
throw new Error(
|
|
2416
2419
|
"steps().step(name, ...) requires a non-empty step name."
|
|
2417
2420
|
);
|
|
2418
2421
|
}
|
|
2422
|
+
const stepResolver = options?.runIf && !isConditionalStepResolver(resolver) ? new DeeplineConditionalStepResolver(
|
|
2423
|
+
options.runIf,
|
|
2424
|
+
resolver,
|
|
2425
|
+
null
|
|
2426
|
+
) : resolver;
|
|
2419
2427
|
return new _DeeplineStepProgram(
|
|
2420
2428
|
[
|
|
2421
2429
|
...this.steps,
|
|
2422
2430
|
{
|
|
2423
2431
|
name,
|
|
2424
|
-
resolver
|
|
2432
|
+
resolver: stepResolver
|
|
2425
2433
|
}
|
|
2426
2434
|
],
|
|
2427
2435
|
this.returnResolver
|
|
@@ -2431,6 +2439,9 @@ var DeeplineStepProgram = class _DeeplineStepProgram {
|
|
|
2431
2439
|
return new _DeeplineStepProgram(this.steps, resolver);
|
|
2432
2440
|
}
|
|
2433
2441
|
};
|
|
2442
|
+
function isConditionalStepResolver(value) {
|
|
2443
|
+
return value !== null && typeof value === "object" && value.kind === "conditional";
|
|
2444
|
+
}
|
|
2434
2445
|
function steps() {
|
|
2435
2446
|
return new DeeplineStepProgram([]);
|
|
2436
2447
|
}
|
package/dist/index.mjs
CHANGED
|
@@ -179,10 +179,10 @@ import { join as join2 } from "path";
|
|
|
179
179
|
|
|
180
180
|
// src/release.ts
|
|
181
181
|
var SDK_RELEASE = {
|
|
182
|
-
version: "0.1.
|
|
183
|
-
apiContract: "2026-06-dataset-column-
|
|
182
|
+
version: "0.1.76",
|
|
183
|
+
apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
|
|
184
184
|
supportPolicy: {
|
|
185
|
-
latest: "0.1.
|
|
185
|
+
latest: "0.1.76",
|
|
186
186
|
minimumSupported: "0.1.53",
|
|
187
187
|
deprecatedBelow: "0.1.53"
|
|
188
188
|
}
|
|
@@ -1065,6 +1065,9 @@ var DeeplineClient = class {
|
|
|
1065
1065
|
async checkPlayArtifact(input) {
|
|
1066
1066
|
return this.http.post("/api/v2/plays/check", input);
|
|
1067
1067
|
}
|
|
1068
|
+
async compileEnrichPlan(input) {
|
|
1069
|
+
return this.http.post("/api/v2/enrich/compile", input);
|
|
1070
|
+
}
|
|
1068
1071
|
async startPlayRunFromBundle(input) {
|
|
1069
1072
|
const compilerManifest = input.compilerManifest ?? await this.compilePlayManifest({
|
|
1070
1073
|
name: input.name,
|
|
@@ -2348,18 +2351,23 @@ var DeeplineStepProgram = class _DeeplineStepProgram {
|
|
|
2348
2351
|
steps;
|
|
2349
2352
|
returnResolver;
|
|
2350
2353
|
kind = "steps";
|
|
2351
|
-
step(name, resolver) {
|
|
2354
|
+
step(name, resolver, options) {
|
|
2352
2355
|
if (!name.trim()) {
|
|
2353
2356
|
throw new Error(
|
|
2354
2357
|
"steps().step(name, ...) requires a non-empty step name."
|
|
2355
2358
|
);
|
|
2356
2359
|
}
|
|
2360
|
+
const stepResolver = options?.runIf && !isConditionalStepResolver(resolver) ? new DeeplineConditionalStepResolver(
|
|
2361
|
+
options.runIf,
|
|
2362
|
+
resolver,
|
|
2363
|
+
null
|
|
2364
|
+
) : resolver;
|
|
2357
2365
|
return new _DeeplineStepProgram(
|
|
2358
2366
|
[
|
|
2359
2367
|
...this.steps,
|
|
2360
2368
|
{
|
|
2361
2369
|
name,
|
|
2362
|
-
resolver
|
|
2370
|
+
resolver: stepResolver
|
|
2363
2371
|
}
|
|
2364
2372
|
],
|
|
2365
2373
|
this.returnResolver
|
|
@@ -2369,6 +2377,9 @@ var DeeplineStepProgram = class _DeeplineStepProgram {
|
|
|
2369
2377
|
return new _DeeplineStepProgram(this.steps, resolver);
|
|
2370
2378
|
}
|
|
2371
2379
|
};
|
|
2380
|
+
function isConditionalStepResolver(value) {
|
|
2381
|
+
return value !== null && typeof value === "object" && value.kind === "conditional";
|
|
2382
|
+
}
|
|
2372
2383
|
function steps() {
|
|
2373
2384
|
return new DeeplineStepProgram([]);
|
|
2374
2385
|
}
|
|
@@ -151,6 +151,14 @@ import {
|
|
|
151
151
|
isHardBillingToolHttpError,
|
|
152
152
|
normalizeToolHttpErrorMessage,
|
|
153
153
|
} from './runtime/tool-http-errors';
|
|
154
|
+
import {
|
|
155
|
+
StepProgramDatasetBuilder,
|
|
156
|
+
type StepProgramDatasetOptions,
|
|
157
|
+
} from '../../../shared_libs/play-runtime/step-program-dataset-builder';
|
|
158
|
+
import {
|
|
159
|
+
DEEPLINE_CELL_META_FIELD,
|
|
160
|
+
shouldRecomputeCell,
|
|
161
|
+
} from '../../../shared_libs/play-runtime/cell-staleness';
|
|
154
162
|
|
|
155
163
|
// The play's default export. The bundler injects this — see bundle-play-file.ts.
|
|
156
164
|
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
@@ -2028,6 +2036,7 @@ type WorkerConditionalStepResolver = {
|
|
|
2028
2036
|
|
|
2029
2037
|
type WorkerStepProgramStep = {
|
|
2030
2038
|
name: string;
|
|
2039
|
+
staleAfterSeconds?: number;
|
|
2031
2040
|
resolver:
|
|
2032
2041
|
| WorkerStepResolver
|
|
2033
2042
|
| WorkerConditionalStepResolver
|
|
@@ -3045,6 +3054,7 @@ async function prepareMapRows(input: {
|
|
|
3045
3054
|
tableNamespace: string;
|
|
3046
3055
|
rows: Record<string, unknown>[];
|
|
3047
3056
|
outputFields: string[];
|
|
3057
|
+
cellPolicies?: Record<string, { staleAfterSeconds?: number }>;
|
|
3048
3058
|
}): Promise<{
|
|
3049
3059
|
inserted: number;
|
|
3050
3060
|
skipped: number;
|
|
@@ -3069,6 +3079,7 @@ async function prepareMapRows(input: {
|
|
|
3069
3079
|
rows: input.rows.map((row) => ({ ...row })),
|
|
3070
3080
|
runId: input.req.runId,
|
|
3071
3081
|
userEmail: input.req.userEmail,
|
|
3082
|
+
cellPolicies: input.cellPolicies,
|
|
3072
3083
|
});
|
|
3073
3084
|
for (const timing of result.timings ?? []) {
|
|
3074
3085
|
const phase =
|
|
@@ -3217,12 +3228,16 @@ function createMinimalWorkerCtx(
|
|
|
3217
3228
|
},
|
|
3218
3229
|
);
|
|
3219
3230
|
if (!response.ok) {
|
|
3220
|
-
throw new Error(
|
|
3231
|
+
throw new Error(
|
|
3232
|
+
`Secret ${auth.secret.name} is not available to this run.`,
|
|
3233
|
+
);
|
|
3221
3234
|
}
|
|
3222
3235
|
const payload = (await response.json()) as { value?: unknown };
|
|
3223
3236
|
const value = typeof payload.value === 'string' ? payload.value : null;
|
|
3224
3237
|
if (!value) {
|
|
3225
|
-
throw new Error(
|
|
3238
|
+
throw new Error(
|
|
3239
|
+
`Secret ${auth.secret.name} is not available to this run.`,
|
|
3240
|
+
);
|
|
3226
3241
|
}
|
|
3227
3242
|
secretRedactor.register(value);
|
|
3228
3243
|
return auth.kind === 'bearer'
|
|
@@ -3355,6 +3370,7 @@ function createMinimalWorkerCtx(
|
|
|
3355
3370
|
index: number,
|
|
3356
3371
|
) => Promise<unknown> | unknown)
|
|
3357
3372
|
>,
|
|
3373
|
+
cellPolicies?: Record<string, { staleAfterSeconds?: number }>,
|
|
3358
3374
|
opts?: WorkerMapOptions,
|
|
3359
3375
|
): Promise<unknown> => {
|
|
3360
3376
|
const mapStartedAt = nowMs();
|
|
@@ -3494,6 +3510,7 @@ function createMinimalWorkerCtx(
|
|
|
3494
3510
|
req,
|
|
3495
3511
|
tableNamespace: name,
|
|
3496
3512
|
outputFields,
|
|
3513
|
+
cellPolicies,
|
|
3497
3514
|
rows: chunkEntries.map(({ row, rowKey }) => ({
|
|
3498
3515
|
...row,
|
|
3499
3516
|
__deeplineRowKey: rowKey,
|
|
@@ -3568,10 +3585,11 @@ function createMinimalWorkerCtx(
|
|
|
3568
3585
|
| Record<
|
|
3569
3586
|
string,
|
|
3570
3587
|
{
|
|
3571
|
-
status: 'cached' | 'skipped';
|
|
3588
|
+
status: 'cached' | 'skipped' | 'completed';
|
|
3572
3589
|
stage?: string | null;
|
|
3573
3590
|
reused?: boolean;
|
|
3574
3591
|
runId?: string;
|
|
3592
|
+
completedAt?: number;
|
|
3575
3593
|
}
|
|
3576
3594
|
>
|
|
3577
3595
|
| undefined
|
|
@@ -3600,10 +3618,11 @@ function createMinimalWorkerCtx(
|
|
|
3600
3618
|
const cellMetaPatch: Record<
|
|
3601
3619
|
string,
|
|
3602
3620
|
{
|
|
3603
|
-
status: 'cached' | 'skipped';
|
|
3621
|
+
status: 'cached' | 'skipped' | 'completed';
|
|
3604
3622
|
stage?: string | null;
|
|
3605
3623
|
reused?: boolean;
|
|
3606
3624
|
runId?: string;
|
|
3625
|
+
completedAt?: number;
|
|
3607
3626
|
}
|
|
3608
3627
|
> = {};
|
|
3609
3628
|
const waterfallOutputs: RecordedWaterfallOutput[] = [];
|
|
@@ -3639,7 +3658,28 @@ function createMinimalWorkerCtx(
|
|
|
3639
3658
|
),
|
|
3640
3659
|
};
|
|
3641
3660
|
for (const [key, value] of fieldEntries) {
|
|
3642
|
-
|
|
3661
|
+
const rawCellMeta =
|
|
3662
|
+
enriched[DEEPLINE_CELL_META_FIELD] &&
|
|
3663
|
+
typeof enriched[DEEPLINE_CELL_META_FIELD] === 'object'
|
|
3664
|
+
? (
|
|
3665
|
+
enriched[DEEPLINE_CELL_META_FIELD] as Record<
|
|
3666
|
+
string,
|
|
3667
|
+
unknown
|
|
3668
|
+
>
|
|
3669
|
+
)[key]
|
|
3670
|
+
: null;
|
|
3671
|
+
const reuseDecision = shouldRecomputeCell({
|
|
3672
|
+
hasValue: isCompletedWorkerFieldValue(enriched[key]),
|
|
3673
|
+
meta:
|
|
3674
|
+
rawCellMeta && typeof rawCellMeta === 'object'
|
|
3675
|
+
? (rawCellMeta as {
|
|
3676
|
+
status?: string;
|
|
3677
|
+
completedAt?: number;
|
|
3678
|
+
})
|
|
3679
|
+
: null,
|
|
3680
|
+
policy: cellPolicies?.[key],
|
|
3681
|
+
});
|
|
3682
|
+
if (reuseDecision.action === 'reuse') {
|
|
3643
3683
|
cellMetaPatch[key] = {
|
|
3644
3684
|
status: 'cached',
|
|
3645
3685
|
stage: key,
|
|
@@ -3669,6 +3709,13 @@ function createMinimalWorkerCtx(
|
|
|
3669
3709
|
stage: key,
|
|
3670
3710
|
runId: req.runId,
|
|
3671
3711
|
};
|
|
3712
|
+
} else {
|
|
3713
|
+
cellMetaPatch[key] = {
|
|
3714
|
+
status: 'completed',
|
|
3715
|
+
stage: key,
|
|
3716
|
+
runId: req.runId,
|
|
3717
|
+
completedAt: nowMs(),
|
|
3718
|
+
};
|
|
3672
3719
|
}
|
|
3673
3720
|
}
|
|
3674
3721
|
for (const stepOutput of stepProgramOutputs) {
|
|
@@ -4017,37 +4064,63 @@ function createMinimalWorkerCtx(
|
|
|
4017
4064
|
};
|
|
4018
4065
|
|
|
4019
4066
|
class WorkerDatasetBuilder<T extends Record<string, unknown>> {
|
|
4020
|
-
private readonly
|
|
4021
|
-
|
|
4022
|
-
|
|
4023
|
-
|
|
4067
|
+
private readonly builder: StepProgramDatasetBuilder<
|
|
4068
|
+
WorkerStepProgramStep,
|
|
4069
|
+
WorkerStepProgramStep['resolver'],
|
|
4070
|
+
WorkerMapOptions,
|
|
4071
|
+
Promise<unknown>
|
|
4072
|
+
>;
|
|
4024
4073
|
|
|
4025
4074
|
constructor(
|
|
4026
4075
|
private readonly name: string,
|
|
4027
4076
|
private readonly rows: WorkerDatasetInput<T>,
|
|
4028
|
-
) {
|
|
4077
|
+
) {
|
|
4078
|
+
this.builder = new StepProgramDatasetBuilder(
|
|
4079
|
+
(program, opts) => {
|
|
4080
|
+
const fields = Object.fromEntries(
|
|
4081
|
+
program.steps.map((step) => [step.name, step.resolver]),
|
|
4082
|
+
);
|
|
4083
|
+
const cellPolicies = Object.fromEntries(
|
|
4084
|
+
program.steps.map((step) => [
|
|
4085
|
+
step.name,
|
|
4086
|
+
step.staleAfterSeconds === undefined
|
|
4087
|
+
? {}
|
|
4088
|
+
: { staleAfterSeconds: step.staleAfterSeconds },
|
|
4089
|
+
]),
|
|
4090
|
+
);
|
|
4091
|
+
return runMap(this.name, this.rows, fields, cellPolicies, opts);
|
|
4092
|
+
},
|
|
4093
|
+
{
|
|
4094
|
+
emptyColumnName:
|
|
4095
|
+
'ctx.dataset(...).withColumn(name, ...) requires a name.',
|
|
4096
|
+
invalidColumnsProgram:
|
|
4097
|
+
'ctx.dataset(...).withColumns(...) requires a steps() program.',
|
|
4098
|
+
legacyStep:
|
|
4099
|
+
'ctx.dataset(...).step(...) has been replaced by ctx.dataset(...).withColumn(...).',
|
|
4100
|
+
},
|
|
4101
|
+
);
|
|
4102
|
+
}
|
|
4029
4103
|
|
|
4030
|
-
withColumn(
|
|
4031
|
-
|
|
4032
|
-
|
|
4033
|
-
|
|
4034
|
-
|
|
4035
|
-
|
|
4036
|
-
|
|
4104
|
+
withColumn(
|
|
4105
|
+
name: string,
|
|
4106
|
+
resolver: WorkerStepProgramStep['resolver'],
|
|
4107
|
+
options?: StepProgramDatasetOptions,
|
|
4108
|
+
): this {
|
|
4109
|
+
this.builder.withColumn(name, resolver, options);
|
|
4110
|
+
return this;
|
|
4111
|
+
}
|
|
4112
|
+
|
|
4113
|
+
withColumns(program: WorkerStepProgram): this {
|
|
4114
|
+
this.builder.withColumns(program);
|
|
4037
4115
|
return this;
|
|
4038
4116
|
}
|
|
4039
4117
|
|
|
4040
4118
|
step(): never {
|
|
4041
|
-
|
|
4042
|
-
'ctx.dataset(...).step(...) has been replaced by ctx.dataset(...).withColumn(...).',
|
|
4043
|
-
);
|
|
4119
|
+
return this.builder.step();
|
|
4044
4120
|
}
|
|
4045
4121
|
|
|
4046
4122
|
run(opts?: WorkerMapOptions): Promise<unknown> {
|
|
4047
|
-
|
|
4048
|
-
this.program.steps.map((step) => [step.name, step.resolver]),
|
|
4049
|
-
);
|
|
4050
|
-
return runMap(this.name, this.rows, fields, opts);
|
|
4123
|
+
return this.builder.run(opts);
|
|
4051
4124
|
}
|
|
4052
4125
|
}
|
|
4053
4126
|
|
|
@@ -4233,7 +4306,15 @@ function createMinimalWorkerCtx(
|
|
|
4233
4306
|
const fields = Object.fromEntries(
|
|
4234
4307
|
fieldsDef.steps.map((step) => [step.name, step.resolver]),
|
|
4235
4308
|
);
|
|
4236
|
-
|
|
4309
|
+
const cellPolicies = Object.fromEntries(
|
|
4310
|
+
fieldsDef.steps.map((step) => [
|
|
4311
|
+
step.name,
|
|
4312
|
+
step.staleAfterSeconds === undefined
|
|
4313
|
+
? {}
|
|
4314
|
+
: { staleAfterSeconds: step.staleAfterSeconds },
|
|
4315
|
+
]),
|
|
4316
|
+
);
|
|
4317
|
+
return runMap(name, rows, fields, cellPolicies, opts);
|
|
4237
4318
|
}
|
|
4238
4319
|
throw new Error(
|
|
4239
4320
|
'ctx.dataset(key, rows, fields, options) is not supported. Use ctx.dataset(key, rows).withColumn(...).run(options).',
|
|
@@ -4244,11 +4325,11 @@ function createMinimalWorkerCtx(
|
|
|
4244
4325
|
'ctx.map(...) has been replaced by ctx.dataset(...). Use ctx.dataset(key, rows).withColumn(...).run(options).',
|
|
4245
4326
|
);
|
|
4246
4327
|
},
|
|
4247
|
-
|
|
4248
|
-
|
|
4328
|
+
tools: {
|
|
4329
|
+
async execute(requestArg: unknown): Promise<unknown> {
|
|
4249
4330
|
assertNotAborted(abortSignal);
|
|
4250
4331
|
const request = normalizeToolExecuteArgs(requestArg);
|
|
4251
|
-
|
|
4332
|
+
assertNoSecretTaint(request.input, 'ctx.tools.execute input');
|
|
4252
4333
|
return await executeWithRuntimeReceipt(
|
|
4253
4334
|
`tool:${request.id}:${deriveToolRequestIdentity({
|
|
4254
4335
|
toolId: request.toolId,
|
|
@@ -4318,9 +4399,9 @@ function createMinimalWorkerCtx(
|
|
|
4318
4399
|
timeoutMs?: number;
|
|
4319
4400
|
staleAfterSeconds?: number;
|
|
4320
4401
|
},
|
|
4321
|
-
|
|
4322
|
-
|
|
4323
|
-
|
|
4402
|
+
): Promise<unknown> {
|
|
4403
|
+
const normalizedKey = normalizeContextKey(key, 'runPlay');
|
|
4404
|
+
const resolvedName = resolvePlayRefName(playRef);
|
|
4324
4405
|
assertNoSecretTaint(input, 'ctx.runPlay input');
|
|
4325
4406
|
if (!resolvedName) {
|
|
4326
4407
|
throw new Error('ctx.runPlay(...) requires a resolvable play name.');
|
|
@@ -4560,18 +4641,15 @@ function createMinimalWorkerCtx(
|
|
|
4560
4641
|
}
|
|
4561
4642
|
});
|
|
4562
4643
|
},
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
|
|
4571
|
-
if (
|
|
4572
|
-
valueContainsSecret(input) ||
|
|
4573
|
-
valueContainsSecret(init.body)
|
|
4574
|
-
) {
|
|
4644
|
+
async fetch(
|
|
4645
|
+
key: string,
|
|
4646
|
+
input: string | URL,
|
|
4647
|
+
init: SecretAwareRequestInit = {},
|
|
4648
|
+
options?: { staleAfterSeconds?: number },
|
|
4649
|
+
): Promise<WorkerFetchResponse> {
|
|
4650
|
+
assertNotAborted(abortSignal);
|
|
4651
|
+
const normalizedKey = normalizeContextKey(key, 'fetch');
|
|
4652
|
+
if (valueContainsSecret(input) || valueContainsSecret(init.body)) {
|
|
4575
4653
|
throw new Error(
|
|
4576
4654
|
'ctx.fetch does not allow secrets in the URL or body. Use an approved secret auth helper.',
|
|
4577
4655
|
);
|
|
@@ -4584,10 +4662,10 @@ function createMinimalWorkerCtx(
|
|
|
4584
4662
|
if (init.auth !== undefined && !isSecretAuth(init.auth)) {
|
|
4585
4663
|
throw new Error('ctx.fetch auth must come from ctx.secrets.');
|
|
4586
4664
|
}
|
|
4587
|
-
|
|
4588
|
-
|
|
4665
|
+
const url = input.toString();
|
|
4666
|
+
const method = (init.method ?? 'GET').toUpperCase();
|
|
4589
4667
|
const secretHeaderMarkers = secretAuthHeaderMarkers(init.auth);
|
|
4590
|
-
|
|
4668
|
+
const safeHeaders = {
|
|
4591
4669
|
...normalizeFetchHeaders(init.headers),
|
|
4592
4670
|
...secretHeaderMarkers,
|
|
4593
4671
|
};
|
|
@@ -4606,7 +4684,7 @@ function createMinimalWorkerCtx(
|
|
|
4606
4684
|
safeHeaders,
|
|
4607
4685
|
url,
|
|
4608
4686
|
})}${staleRuntimeSuffix(options?.staleAfterSeconds)}`;
|
|
4609
|
-
|
|
4687
|
+
return await executeWithRuntimeReceipt(receiptKey, async () => {
|
|
4610
4688
|
const secretHeaders = await resolveSecretAuth(init.auth);
|
|
4611
4689
|
const headers = {
|
|
4612
4690
|
...normalizeFetchHeaders(init.headers),
|
|
@@ -4614,23 +4692,23 @@ function createMinimalWorkerCtx(
|
|
|
4614
4692
|
};
|
|
4615
4693
|
const fetchInit = { ...init, headers };
|
|
4616
4694
|
delete fetchInit.auth;
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4695
|
+
const response = await fetch(url, fetchInit);
|
|
4696
|
+
assertNotAborted(abortSignal);
|
|
4697
|
+
const bodyText = await response.text();
|
|
4620
4698
|
const redactedBodyText = secretRedactor.redactString(bodyText);
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4625
|
-
|
|
4626
|
-
|
|
4627
|
-
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
|
|
4699
|
+
return {
|
|
4700
|
+
ok: response.ok,
|
|
4701
|
+
status: response.status,
|
|
4702
|
+
statusText: response.statusText,
|
|
4703
|
+
url: response.url,
|
|
4704
|
+
headers: secretRedactor.redact(
|
|
4705
|
+
Object.fromEntries(response.headers.entries()),
|
|
4706
|
+
) as Record<string, string>,
|
|
4707
|
+
bodyText: redactedBodyText,
|
|
4708
|
+
json: secretRedactor.redact(parseFetchJsonOrNull(bodyText)),
|
|
4709
|
+
};
|
|
4710
|
+
});
|
|
4711
|
+
},
|
|
4634
4712
|
secrets: {
|
|
4635
4713
|
get(name: string): SecretHandle {
|
|
4636
4714
|
if (typeof name !== 'string' || !name.trim()) {
|