deepline 0.1.74 → 0.1.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +1295 -344
- package/dist/cli/index.mjs +1297 -346
- package/dist/index.d.mts +45 -1
- package/dist/index.d.ts +45 -1
- package/dist/index.js +6 -3
- package/dist/index.mjs +6 -3
- package/dist/repo/apps/play-runner-workers/src/entry.ts +101 -45
- package/dist/repo/sdk/src/client.ts +8 -0
- package/dist/repo/sdk/src/play.ts +2 -1
- package/dist/repo/sdk/src/plays/harness-stub.ts +1 -0
- package/dist/repo/sdk/src/release.ts +3 -3
- package/dist/repo/sdk/src/worker-play-entry.ts +3 -0
- package/dist/repo/shared_libs/play-runtime/cell-staleness.ts +88 -0
- package/dist/repo/shared_libs/play-runtime/step-program-dataset-builder.ts +5 -0
- package/dist/repo/shared_libs/plays/row-identity.ts +0 -40
- package/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -1051,6 +1051,43 @@ interface PlayStagedFileRef {
|
|
|
1051
1051
|
bytes: number;
|
|
1052
1052
|
}
|
|
1053
1053
|
|
|
1054
|
+
type EnrichStepCommand = {
|
|
1055
|
+
alias: string;
|
|
1056
|
+
tool: string;
|
|
1057
|
+
operation?: string;
|
|
1058
|
+
payload: Record<string, unknown>;
|
|
1059
|
+
extract_js?: string;
|
|
1060
|
+
run_if_js?: string;
|
|
1061
|
+
description?: string;
|
|
1062
|
+
disabled?: boolean;
|
|
1063
|
+
};
|
|
1064
|
+
type EnrichWaterfallCommand = {
|
|
1065
|
+
with_waterfall: string;
|
|
1066
|
+
min_results?: number;
|
|
1067
|
+
commands: EnrichCommand[];
|
|
1068
|
+
description?: string;
|
|
1069
|
+
};
|
|
1070
|
+
type EnrichCommand = EnrichStepCommand | EnrichWaterfallCommand;
|
|
1071
|
+
type EnrichCompiledConfig = {
|
|
1072
|
+
version: 1;
|
|
1073
|
+
commands: EnrichCommand[];
|
|
1074
|
+
cost_cap_usd_per_run?: number;
|
|
1075
|
+
_comments?: Array<{
|
|
1076
|
+
path: string;
|
|
1077
|
+
lines: string[];
|
|
1078
|
+
}>;
|
|
1079
|
+
_expansion_preview?: {
|
|
1080
|
+
plays: Array<{
|
|
1081
|
+
alias: string;
|
|
1082
|
+
tool_id: string;
|
|
1083
|
+
template_group: string;
|
|
1084
|
+
runtime_group: string;
|
|
1085
|
+
estimated_credits_range: string;
|
|
1086
|
+
steps: Array<Record<string, unknown>>;
|
|
1087
|
+
}>;
|
|
1088
|
+
};
|
|
1089
|
+
};
|
|
1090
|
+
|
|
1054
1091
|
type ExecuteToolRawOptions = {
|
|
1055
1092
|
includeToolMetadata?: boolean;
|
|
1056
1093
|
};
|
|
@@ -1350,6 +1387,12 @@ declare class DeeplineClient {
|
|
|
1350
1387
|
sourceFiles?: Record<string, string>;
|
|
1351
1388
|
artifact: Record<string, unknown>;
|
|
1352
1389
|
}): Promise<PlayCheckResult>;
|
|
1390
|
+
compileEnrichPlan(input: {
|
|
1391
|
+
plan_args?: string[];
|
|
1392
|
+
config?: unknown;
|
|
1393
|
+
}): Promise<{
|
|
1394
|
+
config: EnrichCompiledConfig;
|
|
1395
|
+
}>;
|
|
1353
1396
|
startPlayRunFromBundle(input: {
|
|
1354
1397
|
name: string;
|
|
1355
1398
|
sourceCode: string;
|
|
@@ -2145,6 +2188,7 @@ type ConditionalStepResolver<Row, Value, Else = null> = {
|
|
|
2145
2188
|
};
|
|
2146
2189
|
type StepOptions<Row> = {
|
|
2147
2190
|
readonly runIf?: (row: Row, index: number) => boolean | Promise<boolean>;
|
|
2191
|
+
readonly staleAfterSeconds?: number;
|
|
2148
2192
|
};
|
|
2149
2193
|
type StepProgram<Input, Output, Return = Output> = {
|
|
2150
2194
|
readonly kind: 'steps';
|
|
@@ -2163,6 +2207,7 @@ type StepProgramResolver<Input, Return> = {
|
|
|
2163
2207
|
};
|
|
2164
2208
|
type PlayStepProgramStep = {
|
|
2165
2209
|
readonly name: string;
|
|
2210
|
+
readonly staleAfterSeconds?: number;
|
|
2166
2211
|
readonly resolver: StepResolver<Record<string, unknown>, unknown> | ConditionalStepResolver<Record<string, unknown>, unknown> | StepProgramResolver<Record<string, unknown>, unknown>;
|
|
2167
2212
|
};
|
|
2168
2213
|
type ColumnResolver<Row, Value> = StepResolver<Row, Value> | ConditionalStepResolver<Row, Value> | StepProgramResolver<Row, Value>;
|
|
@@ -2197,7 +2242,6 @@ type DatasetBuilder<InputRow extends object, OutputRow extends object> = {
|
|
|
2197
2242
|
*/
|
|
2198
2243
|
run(options?: {
|
|
2199
2244
|
description?: string;
|
|
2200
|
-
staleAfterSeconds?: number;
|
|
2201
2245
|
key?: (keyof InputRow & string) | readonly (keyof InputRow & string)[] | ((row: InputRow, index: number) => string | number | readonly unknown[]);
|
|
2202
2246
|
}): Promise<PlayDataset<OutputRow>>;
|
|
2203
2247
|
};
|
package/dist/index.d.ts
CHANGED
|
@@ -1051,6 +1051,43 @@ interface PlayStagedFileRef {
|
|
|
1051
1051
|
bytes: number;
|
|
1052
1052
|
}
|
|
1053
1053
|
|
|
1054
|
+
type EnrichStepCommand = {
|
|
1055
|
+
alias: string;
|
|
1056
|
+
tool: string;
|
|
1057
|
+
operation?: string;
|
|
1058
|
+
payload: Record<string, unknown>;
|
|
1059
|
+
extract_js?: string;
|
|
1060
|
+
run_if_js?: string;
|
|
1061
|
+
description?: string;
|
|
1062
|
+
disabled?: boolean;
|
|
1063
|
+
};
|
|
1064
|
+
type EnrichWaterfallCommand = {
|
|
1065
|
+
with_waterfall: string;
|
|
1066
|
+
min_results?: number;
|
|
1067
|
+
commands: EnrichCommand[];
|
|
1068
|
+
description?: string;
|
|
1069
|
+
};
|
|
1070
|
+
type EnrichCommand = EnrichStepCommand | EnrichWaterfallCommand;
|
|
1071
|
+
type EnrichCompiledConfig = {
|
|
1072
|
+
version: 1;
|
|
1073
|
+
commands: EnrichCommand[];
|
|
1074
|
+
cost_cap_usd_per_run?: number;
|
|
1075
|
+
_comments?: Array<{
|
|
1076
|
+
path: string;
|
|
1077
|
+
lines: string[];
|
|
1078
|
+
}>;
|
|
1079
|
+
_expansion_preview?: {
|
|
1080
|
+
plays: Array<{
|
|
1081
|
+
alias: string;
|
|
1082
|
+
tool_id: string;
|
|
1083
|
+
template_group: string;
|
|
1084
|
+
runtime_group: string;
|
|
1085
|
+
estimated_credits_range: string;
|
|
1086
|
+
steps: Array<Record<string, unknown>>;
|
|
1087
|
+
}>;
|
|
1088
|
+
};
|
|
1089
|
+
};
|
|
1090
|
+
|
|
1054
1091
|
type ExecuteToolRawOptions = {
|
|
1055
1092
|
includeToolMetadata?: boolean;
|
|
1056
1093
|
};
|
|
@@ -1350,6 +1387,12 @@ declare class DeeplineClient {
|
|
|
1350
1387
|
sourceFiles?: Record<string, string>;
|
|
1351
1388
|
artifact: Record<string, unknown>;
|
|
1352
1389
|
}): Promise<PlayCheckResult>;
|
|
1390
|
+
compileEnrichPlan(input: {
|
|
1391
|
+
plan_args?: string[];
|
|
1392
|
+
config?: unknown;
|
|
1393
|
+
}): Promise<{
|
|
1394
|
+
config: EnrichCompiledConfig;
|
|
1395
|
+
}>;
|
|
1353
1396
|
startPlayRunFromBundle(input: {
|
|
1354
1397
|
name: string;
|
|
1355
1398
|
sourceCode: string;
|
|
@@ -2145,6 +2188,7 @@ type ConditionalStepResolver<Row, Value, Else = null> = {
|
|
|
2145
2188
|
};
|
|
2146
2189
|
type StepOptions<Row> = {
|
|
2147
2190
|
readonly runIf?: (row: Row, index: number) => boolean | Promise<boolean>;
|
|
2191
|
+
readonly staleAfterSeconds?: number;
|
|
2148
2192
|
};
|
|
2149
2193
|
type StepProgram<Input, Output, Return = Output> = {
|
|
2150
2194
|
readonly kind: 'steps';
|
|
@@ -2163,6 +2207,7 @@ type StepProgramResolver<Input, Return> = {
|
|
|
2163
2207
|
};
|
|
2164
2208
|
type PlayStepProgramStep = {
|
|
2165
2209
|
readonly name: string;
|
|
2210
|
+
readonly staleAfterSeconds?: number;
|
|
2166
2211
|
readonly resolver: StepResolver<Record<string, unknown>, unknown> | ConditionalStepResolver<Record<string, unknown>, unknown> | StepProgramResolver<Record<string, unknown>, unknown>;
|
|
2167
2212
|
};
|
|
2168
2213
|
type ColumnResolver<Row, Value> = StepResolver<Row, Value> | ConditionalStepResolver<Row, Value> | StepProgramResolver<Row, Value>;
|
|
@@ -2197,7 +2242,6 @@ type DatasetBuilder<InputRow extends object, OutputRow extends object> = {
|
|
|
2197
2242
|
*/
|
|
2198
2243
|
run(options?: {
|
|
2199
2244
|
description?: string;
|
|
2200
|
-
staleAfterSeconds?: number;
|
|
2201
2245
|
key?: (keyof InputRow & string) | readonly (keyof InputRow & string)[] | ((row: InputRow, index: number) => string | number | readonly unknown[]);
|
|
2202
2246
|
}): Promise<PlayDataset<OutputRow>>;
|
|
2203
2247
|
};
|
package/dist/index.js
CHANGED
|
@@ -241,10 +241,10 @@ var import_node_path2 = require("path");
|
|
|
241
241
|
|
|
242
242
|
// src/release.ts
|
|
243
243
|
var SDK_RELEASE = {
|
|
244
|
-
version: "0.1.
|
|
245
|
-
apiContract: "2026-06-dataset-column-
|
|
244
|
+
version: "0.1.76",
|
|
245
|
+
apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
|
|
246
246
|
supportPolicy: {
|
|
247
|
-
latest: "0.1.
|
|
247
|
+
latest: "0.1.76",
|
|
248
248
|
minimumSupported: "0.1.53",
|
|
249
249
|
deprecatedBelow: "0.1.53"
|
|
250
250
|
}
|
|
@@ -1127,6 +1127,9 @@ var DeeplineClient = class {
|
|
|
1127
1127
|
async checkPlayArtifact(input) {
|
|
1128
1128
|
return this.http.post("/api/v2/plays/check", input);
|
|
1129
1129
|
}
|
|
1130
|
+
async compileEnrichPlan(input) {
|
|
1131
|
+
return this.http.post("/api/v2/enrich/compile", input);
|
|
1132
|
+
}
|
|
1130
1133
|
async startPlayRunFromBundle(input) {
|
|
1131
1134
|
const compilerManifest = input.compilerManifest ?? await this.compilePlayManifest({
|
|
1132
1135
|
name: input.name,
|
package/dist/index.mjs
CHANGED
|
@@ -179,10 +179,10 @@ import { join as join2 } from "path";
|
|
|
179
179
|
|
|
180
180
|
// src/release.ts
|
|
181
181
|
var SDK_RELEASE = {
|
|
182
|
-
version: "0.1.
|
|
183
|
-
apiContract: "2026-06-dataset-column-
|
|
182
|
+
version: "0.1.76",
|
|
183
|
+
apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
|
|
184
184
|
supportPolicy: {
|
|
185
|
-
latest: "0.1.
|
|
185
|
+
latest: "0.1.76",
|
|
186
186
|
minimumSupported: "0.1.53",
|
|
187
187
|
deprecatedBelow: "0.1.53"
|
|
188
188
|
}
|
|
@@ -1065,6 +1065,9 @@ var DeeplineClient = class {
|
|
|
1065
1065
|
async checkPlayArtifact(input) {
|
|
1066
1066
|
return this.http.post("/api/v2/plays/check", input);
|
|
1067
1067
|
}
|
|
1068
|
+
async compileEnrichPlan(input) {
|
|
1069
|
+
return this.http.post("/api/v2/enrich/compile", input);
|
|
1070
|
+
}
|
|
1068
1071
|
async startPlayRunFromBundle(input) {
|
|
1069
1072
|
const compilerManifest = input.compilerManifest ?? await this.compilePlayManifest({
|
|
1070
1073
|
name: input.name,
|
|
@@ -155,6 +155,10 @@ import {
|
|
|
155
155
|
StepProgramDatasetBuilder,
|
|
156
156
|
type StepProgramDatasetOptions,
|
|
157
157
|
} from '../../../shared_libs/play-runtime/step-program-dataset-builder';
|
|
158
|
+
import {
|
|
159
|
+
DEEPLINE_CELL_META_FIELD,
|
|
160
|
+
shouldRecomputeCell,
|
|
161
|
+
} from '../../../shared_libs/play-runtime/cell-staleness';
|
|
158
162
|
|
|
159
163
|
// The play's default export. The bundler injects this — see bundle-play-file.ts.
|
|
160
164
|
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
@@ -2032,6 +2036,7 @@ type WorkerConditionalStepResolver = {
|
|
|
2032
2036
|
|
|
2033
2037
|
type WorkerStepProgramStep = {
|
|
2034
2038
|
name: string;
|
|
2039
|
+
staleAfterSeconds?: number;
|
|
2035
2040
|
resolver:
|
|
2036
2041
|
| WorkerStepResolver
|
|
2037
2042
|
| WorkerConditionalStepResolver
|
|
@@ -3049,6 +3054,7 @@ async function prepareMapRows(input: {
|
|
|
3049
3054
|
tableNamespace: string;
|
|
3050
3055
|
rows: Record<string, unknown>[];
|
|
3051
3056
|
outputFields: string[];
|
|
3057
|
+
cellPolicies?: Record<string, { staleAfterSeconds?: number }>;
|
|
3052
3058
|
}): Promise<{
|
|
3053
3059
|
inserted: number;
|
|
3054
3060
|
skipped: number;
|
|
@@ -3073,6 +3079,7 @@ async function prepareMapRows(input: {
|
|
|
3073
3079
|
rows: input.rows.map((row) => ({ ...row })),
|
|
3074
3080
|
runId: input.req.runId,
|
|
3075
3081
|
userEmail: input.req.userEmail,
|
|
3082
|
+
cellPolicies: input.cellPolicies,
|
|
3076
3083
|
});
|
|
3077
3084
|
for (const timing of result.timings ?? []) {
|
|
3078
3085
|
const phase =
|
|
@@ -3221,12 +3228,16 @@ function createMinimalWorkerCtx(
|
|
|
3221
3228
|
},
|
|
3222
3229
|
);
|
|
3223
3230
|
if (!response.ok) {
|
|
3224
|
-
throw new Error(
|
|
3231
|
+
throw new Error(
|
|
3232
|
+
`Secret ${auth.secret.name} is not available to this run.`,
|
|
3233
|
+
);
|
|
3225
3234
|
}
|
|
3226
3235
|
const payload = (await response.json()) as { value?: unknown };
|
|
3227
3236
|
const value = typeof payload.value === 'string' ? payload.value : null;
|
|
3228
3237
|
if (!value) {
|
|
3229
|
-
throw new Error(
|
|
3238
|
+
throw new Error(
|
|
3239
|
+
`Secret ${auth.secret.name} is not available to this run.`,
|
|
3240
|
+
);
|
|
3230
3241
|
}
|
|
3231
3242
|
secretRedactor.register(value);
|
|
3232
3243
|
return auth.kind === 'bearer'
|
|
@@ -3359,6 +3370,7 @@ function createMinimalWorkerCtx(
|
|
|
3359
3370
|
index: number,
|
|
3360
3371
|
) => Promise<unknown> | unknown)
|
|
3361
3372
|
>,
|
|
3373
|
+
cellPolicies?: Record<string, { staleAfterSeconds?: number }>,
|
|
3362
3374
|
opts?: WorkerMapOptions,
|
|
3363
3375
|
): Promise<unknown> => {
|
|
3364
3376
|
const mapStartedAt = nowMs();
|
|
@@ -3498,6 +3510,7 @@ function createMinimalWorkerCtx(
|
|
|
3498
3510
|
req,
|
|
3499
3511
|
tableNamespace: name,
|
|
3500
3512
|
outputFields,
|
|
3513
|
+
cellPolicies,
|
|
3501
3514
|
rows: chunkEntries.map(({ row, rowKey }) => ({
|
|
3502
3515
|
...row,
|
|
3503
3516
|
__deeplineRowKey: rowKey,
|
|
@@ -3572,10 +3585,11 @@ function createMinimalWorkerCtx(
|
|
|
3572
3585
|
| Record<
|
|
3573
3586
|
string,
|
|
3574
3587
|
{
|
|
3575
|
-
status: 'cached' | 'skipped';
|
|
3588
|
+
status: 'cached' | 'skipped' | 'completed';
|
|
3576
3589
|
stage?: string | null;
|
|
3577
3590
|
reused?: boolean;
|
|
3578
3591
|
runId?: string;
|
|
3592
|
+
completedAt?: number;
|
|
3579
3593
|
}
|
|
3580
3594
|
>
|
|
3581
3595
|
| undefined
|
|
@@ -3604,10 +3618,11 @@ function createMinimalWorkerCtx(
|
|
|
3604
3618
|
const cellMetaPatch: Record<
|
|
3605
3619
|
string,
|
|
3606
3620
|
{
|
|
3607
|
-
status: 'cached' | 'skipped';
|
|
3621
|
+
status: 'cached' | 'skipped' | 'completed';
|
|
3608
3622
|
stage?: string | null;
|
|
3609
3623
|
reused?: boolean;
|
|
3610
3624
|
runId?: string;
|
|
3625
|
+
completedAt?: number;
|
|
3611
3626
|
}
|
|
3612
3627
|
> = {};
|
|
3613
3628
|
const waterfallOutputs: RecordedWaterfallOutput[] = [];
|
|
@@ -3643,7 +3658,28 @@ function createMinimalWorkerCtx(
|
|
|
3643
3658
|
),
|
|
3644
3659
|
};
|
|
3645
3660
|
for (const [key, value] of fieldEntries) {
|
|
3646
|
-
|
|
3661
|
+
const rawCellMeta =
|
|
3662
|
+
enriched[DEEPLINE_CELL_META_FIELD] &&
|
|
3663
|
+
typeof enriched[DEEPLINE_CELL_META_FIELD] === 'object'
|
|
3664
|
+
? (
|
|
3665
|
+
enriched[DEEPLINE_CELL_META_FIELD] as Record<
|
|
3666
|
+
string,
|
|
3667
|
+
unknown
|
|
3668
|
+
>
|
|
3669
|
+
)[key]
|
|
3670
|
+
: null;
|
|
3671
|
+
const reuseDecision = shouldRecomputeCell({
|
|
3672
|
+
hasValue: isCompletedWorkerFieldValue(enriched[key]),
|
|
3673
|
+
meta:
|
|
3674
|
+
rawCellMeta && typeof rawCellMeta === 'object'
|
|
3675
|
+
? (rawCellMeta as {
|
|
3676
|
+
status?: string;
|
|
3677
|
+
completedAt?: number;
|
|
3678
|
+
})
|
|
3679
|
+
: null,
|
|
3680
|
+
policy: cellPolicies?.[key],
|
|
3681
|
+
});
|
|
3682
|
+
if (reuseDecision.action === 'reuse') {
|
|
3647
3683
|
cellMetaPatch[key] = {
|
|
3648
3684
|
status: 'cached',
|
|
3649
3685
|
stage: key,
|
|
@@ -3673,6 +3709,13 @@ function createMinimalWorkerCtx(
|
|
|
3673
3709
|
stage: key,
|
|
3674
3710
|
runId: req.runId,
|
|
3675
3711
|
};
|
|
3712
|
+
} else {
|
|
3713
|
+
cellMetaPatch[key] = {
|
|
3714
|
+
status: 'completed',
|
|
3715
|
+
stage: key,
|
|
3716
|
+
runId: req.runId,
|
|
3717
|
+
completedAt: nowMs(),
|
|
3718
|
+
};
|
|
3676
3719
|
}
|
|
3677
3720
|
}
|
|
3678
3721
|
for (const stepOutput of stepProgramOutputs) {
|
|
@@ -4037,7 +4080,15 @@ function createMinimalWorkerCtx(
|
|
|
4037
4080
|
const fields = Object.fromEntries(
|
|
4038
4081
|
program.steps.map((step) => [step.name, step.resolver]),
|
|
4039
4082
|
);
|
|
4040
|
-
|
|
4083
|
+
const cellPolicies = Object.fromEntries(
|
|
4084
|
+
program.steps.map((step) => [
|
|
4085
|
+
step.name,
|
|
4086
|
+
step.staleAfterSeconds === undefined
|
|
4087
|
+
? {}
|
|
4088
|
+
: { staleAfterSeconds: step.staleAfterSeconds },
|
|
4089
|
+
]),
|
|
4090
|
+
);
|
|
4091
|
+
return runMap(this.name, this.rows, fields, cellPolicies, opts);
|
|
4041
4092
|
},
|
|
4042
4093
|
{
|
|
4043
4094
|
emptyColumnName:
|
|
@@ -4255,7 +4306,15 @@ function createMinimalWorkerCtx(
|
|
|
4255
4306
|
const fields = Object.fromEntries(
|
|
4256
4307
|
fieldsDef.steps.map((step) => [step.name, step.resolver]),
|
|
4257
4308
|
);
|
|
4258
|
-
|
|
4309
|
+
const cellPolicies = Object.fromEntries(
|
|
4310
|
+
fieldsDef.steps.map((step) => [
|
|
4311
|
+
step.name,
|
|
4312
|
+
step.staleAfterSeconds === undefined
|
|
4313
|
+
? {}
|
|
4314
|
+
: { staleAfterSeconds: step.staleAfterSeconds },
|
|
4315
|
+
]),
|
|
4316
|
+
);
|
|
4317
|
+
return runMap(name, rows, fields, cellPolicies, opts);
|
|
4259
4318
|
}
|
|
4260
4319
|
throw new Error(
|
|
4261
4320
|
'ctx.dataset(key, rows, fields, options) is not supported. Use ctx.dataset(key, rows).withColumn(...).run(options).',
|
|
@@ -4266,11 +4325,11 @@ function createMinimalWorkerCtx(
|
|
|
4266
4325
|
'ctx.map(...) has been replaced by ctx.dataset(...). Use ctx.dataset(key, rows).withColumn(...).run(options).',
|
|
4267
4326
|
);
|
|
4268
4327
|
},
|
|
4269
|
-
|
|
4270
|
-
|
|
4328
|
+
tools: {
|
|
4329
|
+
async execute(requestArg: unknown): Promise<unknown> {
|
|
4271
4330
|
assertNotAborted(abortSignal);
|
|
4272
4331
|
const request = normalizeToolExecuteArgs(requestArg);
|
|
4273
|
-
|
|
4332
|
+
assertNoSecretTaint(request.input, 'ctx.tools.execute input');
|
|
4274
4333
|
return await executeWithRuntimeReceipt(
|
|
4275
4334
|
`tool:${request.id}:${deriveToolRequestIdentity({
|
|
4276
4335
|
toolId: request.toolId,
|
|
@@ -4340,9 +4399,9 @@ function createMinimalWorkerCtx(
|
|
|
4340
4399
|
timeoutMs?: number;
|
|
4341
4400
|
staleAfterSeconds?: number;
|
|
4342
4401
|
},
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
|
|
4402
|
+
): Promise<unknown> {
|
|
4403
|
+
const normalizedKey = normalizeContextKey(key, 'runPlay');
|
|
4404
|
+
const resolvedName = resolvePlayRefName(playRef);
|
|
4346
4405
|
assertNoSecretTaint(input, 'ctx.runPlay input');
|
|
4347
4406
|
if (!resolvedName) {
|
|
4348
4407
|
throw new Error('ctx.runPlay(...) requires a resolvable play name.');
|
|
@@ -4582,18 +4641,15 @@ function createMinimalWorkerCtx(
|
|
|
4582
4641
|
}
|
|
4583
4642
|
});
|
|
4584
4643
|
},
|
|
4585
|
-
|
|
4586
|
-
|
|
4587
|
-
|
|
4588
|
-
|
|
4589
|
-
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
if (
|
|
4594
|
-
valueContainsSecret(input) ||
|
|
4595
|
-
valueContainsSecret(init.body)
|
|
4596
|
-
) {
|
|
4644
|
+
async fetch(
|
|
4645
|
+
key: string,
|
|
4646
|
+
input: string | URL,
|
|
4647
|
+
init: SecretAwareRequestInit = {},
|
|
4648
|
+
options?: { staleAfterSeconds?: number },
|
|
4649
|
+
): Promise<WorkerFetchResponse> {
|
|
4650
|
+
assertNotAborted(abortSignal);
|
|
4651
|
+
const normalizedKey = normalizeContextKey(key, 'fetch');
|
|
4652
|
+
if (valueContainsSecret(input) || valueContainsSecret(init.body)) {
|
|
4597
4653
|
throw new Error(
|
|
4598
4654
|
'ctx.fetch does not allow secrets in the URL or body. Use an approved secret auth helper.',
|
|
4599
4655
|
);
|
|
@@ -4606,10 +4662,10 @@ function createMinimalWorkerCtx(
|
|
|
4606
4662
|
if (init.auth !== undefined && !isSecretAuth(init.auth)) {
|
|
4607
4663
|
throw new Error('ctx.fetch auth must come from ctx.secrets.');
|
|
4608
4664
|
}
|
|
4609
|
-
|
|
4610
|
-
|
|
4665
|
+
const url = input.toString();
|
|
4666
|
+
const method = (init.method ?? 'GET').toUpperCase();
|
|
4611
4667
|
const secretHeaderMarkers = secretAuthHeaderMarkers(init.auth);
|
|
4612
|
-
|
|
4668
|
+
const safeHeaders = {
|
|
4613
4669
|
...normalizeFetchHeaders(init.headers),
|
|
4614
4670
|
...secretHeaderMarkers,
|
|
4615
4671
|
};
|
|
@@ -4628,7 +4684,7 @@ function createMinimalWorkerCtx(
|
|
|
4628
4684
|
safeHeaders,
|
|
4629
4685
|
url,
|
|
4630
4686
|
})}${staleRuntimeSuffix(options?.staleAfterSeconds)}`;
|
|
4631
|
-
|
|
4687
|
+
return await executeWithRuntimeReceipt(receiptKey, async () => {
|
|
4632
4688
|
const secretHeaders = await resolveSecretAuth(init.auth);
|
|
4633
4689
|
const headers = {
|
|
4634
4690
|
...normalizeFetchHeaders(init.headers),
|
|
@@ -4636,23 +4692,23 @@ function createMinimalWorkerCtx(
|
|
|
4636
4692
|
};
|
|
4637
4693
|
const fetchInit = { ...init, headers };
|
|
4638
4694
|
delete fetchInit.auth;
|
|
4639
|
-
|
|
4640
|
-
|
|
4641
|
-
|
|
4695
|
+
const response = await fetch(url, fetchInit);
|
|
4696
|
+
assertNotAborted(abortSignal);
|
|
4697
|
+
const bodyText = await response.text();
|
|
4642
4698
|
const redactedBodyText = secretRedactor.redactString(bodyText);
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
|
|
4652
|
-
|
|
4653
|
-
|
|
4654
|
-
|
|
4655
|
-
|
|
4699
|
+
return {
|
|
4700
|
+
ok: response.ok,
|
|
4701
|
+
status: response.status,
|
|
4702
|
+
statusText: response.statusText,
|
|
4703
|
+
url: response.url,
|
|
4704
|
+
headers: secretRedactor.redact(
|
|
4705
|
+
Object.fromEntries(response.headers.entries()),
|
|
4706
|
+
) as Record<string, string>,
|
|
4707
|
+
bodyText: redactedBodyText,
|
|
4708
|
+
json: secretRedactor.redact(parseFetchJsonOrNull(bodyText)),
|
|
4709
|
+
};
|
|
4710
|
+
});
|
|
4711
|
+
},
|
|
4656
4712
|
secrets: {
|
|
4657
4713
|
get(name: string): SecretHandle {
|
|
4658
4714
|
if (typeof name !== 'string' || !name.trim()) {
|
|
@@ -68,6 +68,7 @@ import type {
|
|
|
68
68
|
} from './types.js';
|
|
69
69
|
import type { PlayStagedFileRef } from './plays/local-file-discovery.js';
|
|
70
70
|
import type { PlayCompilerManifest } from '../../shared_libs/plays/compiler-manifest.js';
|
|
71
|
+
import type { EnrichCompiledConfig } from './cli/enrich-play-compiler.js';
|
|
71
72
|
|
|
72
73
|
const TERMINAL_PLAY_STATUSES = new Set(['completed', 'failed', 'cancelled']);
|
|
73
74
|
const INCLUDE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
|
|
@@ -1019,6 +1020,13 @@ export class DeeplineClient {
|
|
|
1019
1020
|
return this.http.post('/api/v2/plays/check', input);
|
|
1020
1021
|
}
|
|
1021
1022
|
|
|
1023
|
+
async compileEnrichPlan(input: {
|
|
1024
|
+
plan_args?: string[];
|
|
1025
|
+
config?: unknown;
|
|
1026
|
+
}): Promise<{ config: EnrichCompiledConfig }> {
|
|
1027
|
+
return this.http.post('/api/v2/enrich/compile', input);
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1022
1030
|
async startPlayRunFromBundle(input: {
|
|
1023
1031
|
name: string;
|
|
1024
1032
|
sourceCode: string;
|
|
@@ -224,6 +224,7 @@ export type ConditionalStepResolver<Row, Value, Else = null> = {
|
|
|
224
224
|
|
|
225
225
|
export type StepOptions<Row> = {
|
|
226
226
|
readonly runIf?: (row: Row, index: number) => boolean | Promise<boolean>;
|
|
227
|
+
readonly staleAfterSeconds?: number;
|
|
227
228
|
};
|
|
228
229
|
|
|
229
230
|
export type StepProgram<Input, Output, Return = Output> = {
|
|
@@ -257,6 +258,7 @@ export type StepProgramResolver<Input, Return> = {
|
|
|
257
258
|
|
|
258
259
|
export type PlayStepProgramStep = {
|
|
259
260
|
readonly name: string;
|
|
261
|
+
readonly staleAfterSeconds?: number;
|
|
260
262
|
readonly resolver:
|
|
261
263
|
| StepResolver<Record<string, unknown>, unknown>
|
|
262
264
|
| ConditionalStepResolver<Record<string, unknown>, unknown>
|
|
@@ -318,7 +320,6 @@ export type DatasetBuilder<
|
|
|
318
320
|
*/
|
|
319
321
|
run(options?: {
|
|
320
322
|
description?: string;
|
|
321
|
-
staleAfterSeconds?: number;
|
|
322
323
|
key?:
|
|
323
324
|
| (keyof InputRow & string)
|
|
324
325
|
| readonly (keyof InputRow & string)[]
|
|
@@ -191,6 +191,7 @@ export async function harnessStartSheetDataset(input: {
|
|
|
191
191
|
runId: string;
|
|
192
192
|
inputOffset?: number;
|
|
193
193
|
userEmail?: string | null;
|
|
194
|
+
cellPolicies?: Record<string, { staleAfterSeconds?: number }>;
|
|
194
195
|
}): Promise<{
|
|
195
196
|
inserted: number;
|
|
196
197
|
skipped: number;
|
|
@@ -50,10 +50,10 @@ export type SdkRelease = {
|
|
|
50
50
|
};
|
|
51
51
|
|
|
52
52
|
export const SDK_RELEASE = {
|
|
53
|
-
version: '0.1.
|
|
54
|
-
apiContract: '2026-06-dataset-column-
|
|
53
|
+
version: '0.1.76',
|
|
54
|
+
apiContract: '2026-06-dataset-column-cell-stale-hard-cutover',
|
|
55
55
|
supportPolicy: {
|
|
56
|
-
latest: '0.1.
|
|
56
|
+
latest: '0.1.76',
|
|
57
57
|
minimumSupported: '0.1.53',
|
|
58
58
|
deprecatedBelow: '0.1.53',
|
|
59
59
|
},
|
|
@@ -128,6 +128,9 @@ class WorkerStepProgram<Input, Output, ReturnValue> implements StepProgram<
|
|
|
128
128
|
{
|
|
129
129
|
name,
|
|
130
130
|
resolver: stepResolver as PlayStepProgramStep['resolver'],
|
|
131
|
+
...(options?.staleAfterSeconds !== undefined
|
|
132
|
+
? { staleAfterSeconds: options.staleAfterSeconds }
|
|
133
|
+
: {}),
|
|
131
134
|
},
|
|
132
135
|
],
|
|
133
136
|
this.returnResolver as StepResolver<
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
export type CellStalenessPolicy = {
|
|
2
|
+
staleAfterSeconds?: number;
|
|
3
|
+
};
|
|
4
|
+
|
|
5
|
+
export type CellStalenessMeta = {
|
|
6
|
+
status?: string | null;
|
|
7
|
+
completedAt?: number | null;
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
export type CellStalenessDecision =
|
|
11
|
+
| { action: 'recompute'; reason: 'missing' | 'failed' | 'stale' }
|
|
12
|
+
| { action: 'reuse'; reason: 'fresh' | 'no_policy' };
|
|
13
|
+
|
|
14
|
+
export type CellStalenessPolicyByField = Record<string, CellStalenessPolicy>;
|
|
15
|
+
|
|
16
|
+
export const DEEPLINE_CELL_META_FIELD = '__deeplineCellMeta';
|
|
17
|
+
|
|
18
|
+
export function validateStaleAfterSeconds(
|
|
19
|
+
staleAfterSeconds: number | undefined,
|
|
20
|
+
label = 'staleAfterSeconds',
|
|
21
|
+
): void {
|
|
22
|
+
if (staleAfterSeconds === undefined) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
if (
|
|
26
|
+
!Number.isFinite(staleAfterSeconds) ||
|
|
27
|
+
!Number.isInteger(staleAfterSeconds) ||
|
|
28
|
+
staleAfterSeconds <= 0
|
|
29
|
+
) {
|
|
30
|
+
throw new Error(`${label} must be a positive whole number of seconds.`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function normalizeCellStalenessPolicy(
|
|
35
|
+
policy: CellStalenessPolicy | undefined,
|
|
36
|
+
): CellStalenessPolicy {
|
|
37
|
+
validateStaleAfterSeconds(policy?.staleAfterSeconds);
|
|
38
|
+
return policy?.staleAfterSeconds === undefined
|
|
39
|
+
? {}
|
|
40
|
+
: { staleAfterSeconds: policy.staleAfterSeconds };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function shouldRecomputeCell(input: {
|
|
44
|
+
hasValue: boolean;
|
|
45
|
+
meta?: CellStalenessMeta | null;
|
|
46
|
+
policy?: CellStalenessPolicy;
|
|
47
|
+
nowMs?: number;
|
|
48
|
+
}): CellStalenessDecision {
|
|
49
|
+
if (!input.hasValue) {
|
|
50
|
+
return { action: 'recompute', reason: 'missing' };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const status = String(input.meta?.status ?? '').trim();
|
|
54
|
+
if (status === 'failed') {
|
|
55
|
+
return { action: 'recompute', reason: 'failed' };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const staleAfterSeconds = input.policy?.staleAfterSeconds;
|
|
59
|
+
validateStaleAfterSeconds(staleAfterSeconds);
|
|
60
|
+
if (staleAfterSeconds === undefined) {
|
|
61
|
+
return { action: 'reuse', reason: 'no_policy' };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const completedAt =
|
|
65
|
+
typeof input.meta?.completedAt === 'number' &&
|
|
66
|
+
Number.isFinite(input.meta.completedAt)
|
|
67
|
+
? input.meta.completedAt
|
|
68
|
+
: null;
|
|
69
|
+
if (completedAt === null) {
|
|
70
|
+
return { action: 'recompute', reason: 'missing' };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const ageMs = Math.max(0, (input.nowMs ?? Date.now()) - completedAt);
|
|
74
|
+
return ageMs > staleAfterSeconds * 1000
|
|
75
|
+
? { action: 'recompute', reason: 'stale' }
|
|
76
|
+
: { action: 'reuse', reason: 'fresh' };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function cellPolicyFields(
|
|
80
|
+
policies: CellStalenessPolicyByField | undefined,
|
|
81
|
+
): string[] {
|
|
82
|
+
if (!policies) {
|
|
83
|
+
return [];
|
|
84
|
+
}
|
|
85
|
+
return Object.entries(policies)
|
|
86
|
+
.filter(([, policy]) => policy.staleAfterSeconds !== undefined)
|
|
87
|
+
.map(([field]) => field);
|
|
88
|
+
}
|