deepline 0.1.74 → 0.1.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +1305 -345
- package/dist/cli/index.mjs +1307 -347
- package/dist/index.d.mts +77 -1
- package/dist/index.d.ts +77 -1
- package/dist/index.js +201 -3
- package/dist/index.mjs +201 -3
- package/dist/repo/apps/play-runner-workers/src/entry.ts +101 -45
- package/dist/repo/sdk/src/client.ts +8 -0
- package/dist/repo/sdk/src/play.ts +2 -1
- package/dist/repo/sdk/src/plays/harness-stub.ts +1 -0
- package/dist/repo/sdk/src/release.ts +3 -3
- package/dist/repo/sdk/src/worker-play-entry.ts +3 -0
- package/dist/repo/shared_libs/play-runtime/cell-staleness.ts +88 -0
- package/dist/repo/shared_libs/play-runtime/email-status.ts +301 -0
- package/dist/repo/shared_libs/play-runtime/step-program-dataset-builder.ts +5 -0
- package/dist/repo/shared_libs/play-runtime/tool-result.ts +58 -1
- package/dist/repo/shared_libs/plays/row-identity.ts +0 -40
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -179,10 +179,10 @@ import { join as join2 } from "path";
|
|
|
179
179
|
|
|
180
180
|
// src/release.ts
|
|
181
181
|
var SDK_RELEASE = {
|
|
182
|
-
version: "0.1.
|
|
183
|
-
apiContract: "2026-06-dataset-column-
|
|
182
|
+
version: "0.1.77",
|
|
183
|
+
apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
|
|
184
184
|
supportPolicy: {
|
|
185
|
-
latest: "0.1.
|
|
185
|
+
latest: "0.1.77",
|
|
186
186
|
minimumSupported: "0.1.53",
|
|
187
187
|
deprecatedBelow: "0.1.53"
|
|
188
188
|
}
|
|
@@ -1065,6 +1065,9 @@ var DeeplineClient = class {
|
|
|
1065
1065
|
async checkPlayArtifact(input) {
|
|
1066
1066
|
return this.http.post("/api/v2/plays/check", input);
|
|
1067
1067
|
}
|
|
1068
|
+
async compileEnrichPlan(input) {
|
|
1069
|
+
return this.http.post("/api/v2/enrich/compile", input);
|
|
1070
|
+
}
|
|
1068
1071
|
async startPlayRunFromBundle(input) {
|
|
1069
1072
|
const compilerManifest = input.compilerManifest ?? await this.compilePlayManifest({
|
|
1070
1073
|
name: input.name,
|
|
@@ -1841,6 +1844,160 @@ function formatPlayBootstrapFinderKindsForSentence() {
|
|
|
1841
1844
|
return `${allButLast.join(", ")} or ${last}`;
|
|
1842
1845
|
}
|
|
1843
1846
|
|
|
1847
|
+
// ../shared_libs/play-runtime/email-status.ts
|
|
1848
|
+
var DEFAULT_STATUS_MAP = {
|
|
1849
|
+
verified: { status: "valid", verdict: "send", verified: true },
|
|
1850
|
+
valid: { status: "valid", verdict: "send", verified: true },
|
|
1851
|
+
deliverable: { status: "valid", verdict: "send", verified: true },
|
|
1852
|
+
true: { status: "valid", verdict: "send", verified: true },
|
|
1853
|
+
invalid: { status: "invalid", verdict: "drop", verified: false },
|
|
1854
|
+
undeliverable: { status: "invalid", verdict: "drop", verified: false },
|
|
1855
|
+
false: { status: "invalid", verdict: "drop", verified: false },
|
|
1856
|
+
"catch-all": {
|
|
1857
|
+
status: "catch_all",
|
|
1858
|
+
verdict: "verify_next",
|
|
1859
|
+
verified: false
|
|
1860
|
+
},
|
|
1861
|
+
catch_all: {
|
|
1862
|
+
status: "catch_all",
|
|
1863
|
+
verdict: "verify_next",
|
|
1864
|
+
verified: false
|
|
1865
|
+
},
|
|
1866
|
+
valid_catch_all: {
|
|
1867
|
+
status: "valid_catch_all",
|
|
1868
|
+
verdict: "send_with_caution",
|
|
1869
|
+
verified: true
|
|
1870
|
+
},
|
|
1871
|
+
accept_all: {
|
|
1872
|
+
status: "catch_all",
|
|
1873
|
+
verdict: "verify_next",
|
|
1874
|
+
verified: false
|
|
1875
|
+
},
|
|
1876
|
+
unknown: { status: "unknown", verdict: "hold", verified: false },
|
|
1877
|
+
unavailable: { status: "unknown", verdict: "hold", verified: false },
|
|
1878
|
+
do_not_mail: { status: "do_not_mail", verdict: "drop", verified: false },
|
|
1879
|
+
spamtrap: { status: "spamtrap", verdict: "drop", verified: false },
|
|
1880
|
+
abuse: { status: "abuse", verdict: "drop", verified: false },
|
|
1881
|
+
disposable: { status: "disposable", verdict: "drop", verified: false }
|
|
1882
|
+
};
|
|
1883
|
+
function normalizeKey(value) {
|
|
1884
|
+
if (value == null) return null;
|
|
1885
|
+
if (typeof value === "boolean") return String(value);
|
|
1886
|
+
const normalized = String(value).trim().toLowerCase().replace(/\s+/g, "_");
|
|
1887
|
+
return normalized || null;
|
|
1888
|
+
}
|
|
1889
|
+
function boolish(value) {
|
|
1890
|
+
if (typeof value === "boolean") return value;
|
|
1891
|
+
if (typeof value === "number") return value === 1 ? true : value === 0 ? false : null;
|
|
1892
|
+
if (typeof value !== "string") return null;
|
|
1893
|
+
const normalized = value.trim().toLowerCase();
|
|
1894
|
+
if (["true", "yes", "y", "1"].includes(normalized)) return true;
|
|
1895
|
+
if (["false", "no", "n", "0"].includes(normalized)) return false;
|
|
1896
|
+
return null;
|
|
1897
|
+
}
|
|
1898
|
+
function numberish(value) {
|
|
1899
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
1900
|
+
if (typeof value !== "string" || value.trim() === "") return null;
|
|
1901
|
+
const parsed = Number(value);
|
|
1902
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
1903
|
+
}
|
|
1904
|
+
function stringish(value) {
|
|
1905
|
+
return typeof value === "string" && value.trim() ? value.trim() : null;
|
|
1906
|
+
}
|
|
1907
|
+
function deliverability(value) {
|
|
1908
|
+
const normalized = normalizeKey(value);
|
|
1909
|
+
return normalized === "high" || normalized === "medium" || normalized === "low" ? normalized : "unknown";
|
|
1910
|
+
}
|
|
1911
|
+
function mxClass(mxProvider, mxRecord) {
|
|
1912
|
+
const haystack = `${stringish(mxProvider) ?? ""} ${stringish(mxRecord) ?? ""}`.toLowerCase();
|
|
1913
|
+
if (!haystack.trim()) return "unknown";
|
|
1914
|
+
if (/proofpoint|pphosted|mimecast|barracuda|ess\.barracudanetworks|ironport|cisco|iphmx|messagelabs|symantec/.test(
|
|
1915
|
+
haystack
|
|
1916
|
+
)) {
|
|
1917
|
+
return "security_gateway";
|
|
1918
|
+
}
|
|
1919
|
+
if (/aspmx\.l\.google|google|g-suite|google workspace/.test(haystack)) {
|
|
1920
|
+
return "workspace_mailbox";
|
|
1921
|
+
}
|
|
1922
|
+
if (/protection\.outlook|office365|microsoft|outlook|exchange online/.test(haystack)) {
|
|
1923
|
+
return "workspace_mailbox";
|
|
1924
|
+
}
|
|
1925
|
+
if (/gmail|yahoo|icloud|aol|hotmail/.test(haystack)) return "consumer_mailbox";
|
|
1926
|
+
if (/postfix|exim|sendmail|zimbra|plesk|cpanel|mail\./.test(haystack)) return "on_prem";
|
|
1927
|
+
return "unknown";
|
|
1928
|
+
}
|
|
1929
|
+
function entryForStatus(key, map) {
|
|
1930
|
+
if (!key) return null;
|
|
1931
|
+
return map?.[key] ?? DEFAULT_STATUS_MAP[key] ?? null;
|
|
1932
|
+
}
|
|
1933
|
+
function read(values, name) {
|
|
1934
|
+
return values[name];
|
|
1935
|
+
}
|
|
1936
|
+
function matchesRule(rule, values) {
|
|
1937
|
+
return Object.entries(rule.when).every(([key, expected]) => {
|
|
1938
|
+
const actual = read(values, key);
|
|
1939
|
+
if (key.endsWith("Lt")) {
|
|
1940
|
+
const source = numberish(read(values, key.slice(0, -2)));
|
|
1941
|
+
return typeof expected === "number" && source != null && source < expected;
|
|
1942
|
+
}
|
|
1943
|
+
if (typeof expected === "boolean") return boolish(actual) === expected;
|
|
1944
|
+
if (typeof expected === "number") return numberish(actual) === expected;
|
|
1945
|
+
return normalizeKey(actual) === normalizeKey(expected);
|
|
1946
|
+
});
|
|
1947
|
+
}
|
|
1948
|
+
function buildEmailStatus({
|
|
1949
|
+
config,
|
|
1950
|
+
values
|
|
1951
|
+
}) {
|
|
1952
|
+
const rawStatus = read(values, "rawStatus");
|
|
1953
|
+
const rawScore = numberish(read(values, "rawScore"));
|
|
1954
|
+
const valid = boolish(read(values, "valid"));
|
|
1955
|
+
const catchAll = boolish(read(values, "catchAll"));
|
|
1956
|
+
const disposable = boolish(read(values, "disposable"));
|
|
1957
|
+
const abuse = boolish(read(values, "abuse"));
|
|
1958
|
+
const spamtrap = boolish(read(values, "spamtrap"));
|
|
1959
|
+
const suspect = boolish(read(values, "suspect"));
|
|
1960
|
+
const rawKey = normalizeKey(rawStatus);
|
|
1961
|
+
const mapped = config.rules?.find((rule) => matchesRule(rule, values)) ?? entryForStatus(rawKey, config.statusMap) ?? entryForStatus(valid == null ? null : String(valid), config.statusMap);
|
|
1962
|
+
const status = mapped?.status ?? (disposable ? "disposable" : abuse ? "abuse" : spamtrap ? "spamtrap" : catchAll ? "catch_all" : valid === true ? "valid" : valid === false ? "invalid" : "unknown");
|
|
1963
|
+
const defaultVerdict = status === "valid" ? "send" : status === "valid_catch_all" ? "send_with_caution" : status === "catch_all" ? "verify_next" : status === "unknown" ? "hold" : "drop";
|
|
1964
|
+
const verdict = mapped?.verdict ?? defaultVerdict;
|
|
1965
|
+
const verified = mapped?.verified ?? (status === "valid" || status === "valid_catch_all" || verdict === "send");
|
|
1966
|
+
const reasons = [
|
|
1967
|
+
mapped?.reason,
|
|
1968
|
+
catchAll ? "catch_all_domain" : null,
|
|
1969
|
+
mxClass(read(values, "mxProvider"), read(values, "mxRecord")) === "security_gateway" ? "security_gateway_mx" : null,
|
|
1970
|
+
suspect ? "provider_marked_suspect" : null
|
|
1971
|
+
].filter((reason) => typeof reason === "string");
|
|
1972
|
+
return {
|
|
1973
|
+
verdict,
|
|
1974
|
+
status,
|
|
1975
|
+
verified,
|
|
1976
|
+
confidence: rawScore,
|
|
1977
|
+
reasons,
|
|
1978
|
+
signals: {
|
|
1979
|
+
catch_all: catchAll,
|
|
1980
|
+
deliverability: deliverability(read(values, "deliverability")),
|
|
1981
|
+
mx_class: mxClass(read(values, "mxProvider"), read(values, "mxRecord")),
|
|
1982
|
+
mx_provider: stringish(read(values, "mxProvider")),
|
|
1983
|
+
mx_record: stringish(read(values, "mxRecord")),
|
|
1984
|
+
fraud_score: numberish(read(values, "fraudScore")),
|
|
1985
|
+
disposable,
|
|
1986
|
+
role_based: boolish(read(values, "roleBased")),
|
|
1987
|
+
free_email: boolish(read(values, "freeEmail")),
|
|
1988
|
+
abuse,
|
|
1989
|
+
spamtrap,
|
|
1990
|
+
suspect,
|
|
1991
|
+
valid
|
|
1992
|
+
},
|
|
1993
|
+
provider: {
|
|
1994
|
+
name: config.provider,
|
|
1995
|
+
raw_status: typeof rawStatus === "string" || typeof rawStatus === "boolean" || typeof rawStatus === "number" ? rawStatus : null,
|
|
1996
|
+
raw_score: rawScore
|
|
1997
|
+
}
|
|
1998
|
+
};
|
|
1999
|
+
}
|
|
2000
|
+
|
|
1844
2001
|
// ../shared_libs/play-runtime/tool-result.ts
|
|
1845
2002
|
var TARGET_FALLBACK_KEYS = {
|
|
1846
2003
|
email: [/^email$/i, /^address$/i, /email/i],
|
|
@@ -2024,6 +2181,42 @@ function findFirstTargetByPath(result, paths) {
|
|
|
2024
2181
|
}
|
|
2025
2182
|
return null;
|
|
2026
2183
|
}
|
|
2184
|
+
function firstValueForPaths(result, paths) {
|
|
2185
|
+
return findFirstTargetByPath(result, paths);
|
|
2186
|
+
}
|
|
2187
|
+
function buildEmailStatusTarget(result, descriptor) {
|
|
2188
|
+
const config = descriptor.emailStatus;
|
|
2189
|
+
if (!config) return null;
|
|
2190
|
+
const values = {};
|
|
2191
|
+
const pathSets = {
|
|
2192
|
+
rawStatus: config.rawStatus,
|
|
2193
|
+
rawScore: config.rawScore,
|
|
2194
|
+
valid: config.valid,
|
|
2195
|
+
deliverability: config.deliverability,
|
|
2196
|
+
catchAll: config.catchAll,
|
|
2197
|
+
mxProvider: config.mxProvider,
|
|
2198
|
+
mxRecord: config.mxRecord,
|
|
2199
|
+
fraudScore: config.fraudScore,
|
|
2200
|
+
disposable: config.disposable,
|
|
2201
|
+
roleBased: config.roleBased,
|
|
2202
|
+
freeEmail: config.freeEmail,
|
|
2203
|
+
abuse: config.abuse,
|
|
2204
|
+
spamtrap: config.spamtrap,
|
|
2205
|
+
suspect: config.suspect
|
|
2206
|
+
};
|
|
2207
|
+
let firstPath = null;
|
|
2208
|
+
for (const [name, paths] of Object.entries(pathSets)) {
|
|
2209
|
+
const match = firstValueForPaths(result, paths);
|
|
2210
|
+
if (!match) continue;
|
|
2211
|
+
values[name] = match.value;
|
|
2212
|
+
firstPath ??= match.path;
|
|
2213
|
+
}
|
|
2214
|
+
if (!firstPath) return null;
|
|
2215
|
+
return {
|
|
2216
|
+
path: firstPath,
|
|
2217
|
+
value: buildEmailStatus({ config, values })
|
|
2218
|
+
};
|
|
2219
|
+
}
|
|
2027
2220
|
function findFirstTargetByKey(result, target, depth = 0, path = []) {
|
|
2028
2221
|
if (depth > 6) return null;
|
|
2029
2222
|
if (Array.isArray(result)) {
|
|
@@ -2173,6 +2366,11 @@ function deriveListKeys(input) {
|
|
|
2173
2366
|
function buildTargets(result, extractors, targetGetters) {
|
|
2174
2367
|
const targets = {};
|
|
2175
2368
|
for (const [target, descriptor] of Object.entries(extractors ?? {})) {
|
|
2369
|
+
const emailStatusTarget = buildEmailStatusTarget(result, descriptor);
|
|
2370
|
+
if (emailStatusTarget) {
|
|
2371
|
+
targets[target] = emailStatusTarget;
|
|
2372
|
+
continue;
|
|
2373
|
+
}
|
|
2176
2374
|
const fromExtractor = findFirstTargetByPath(result, descriptor.paths);
|
|
2177
2375
|
if (!fromExtractor) continue;
|
|
2178
2376
|
const transformed = coerceToEnum(
|
|
@@ -155,6 +155,10 @@ import {
|
|
|
155
155
|
StepProgramDatasetBuilder,
|
|
156
156
|
type StepProgramDatasetOptions,
|
|
157
157
|
} from '../../../shared_libs/play-runtime/step-program-dataset-builder';
|
|
158
|
+
import {
|
|
159
|
+
DEEPLINE_CELL_META_FIELD,
|
|
160
|
+
shouldRecomputeCell,
|
|
161
|
+
} from '../../../shared_libs/play-runtime/cell-staleness';
|
|
158
162
|
|
|
159
163
|
// The play's default export. The bundler injects this — see bundle-play-file.ts.
|
|
160
164
|
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
@@ -2032,6 +2036,7 @@ type WorkerConditionalStepResolver = {
|
|
|
2032
2036
|
|
|
2033
2037
|
type WorkerStepProgramStep = {
|
|
2034
2038
|
name: string;
|
|
2039
|
+
staleAfterSeconds?: number;
|
|
2035
2040
|
resolver:
|
|
2036
2041
|
| WorkerStepResolver
|
|
2037
2042
|
| WorkerConditionalStepResolver
|
|
@@ -3049,6 +3054,7 @@ async function prepareMapRows(input: {
|
|
|
3049
3054
|
tableNamespace: string;
|
|
3050
3055
|
rows: Record<string, unknown>[];
|
|
3051
3056
|
outputFields: string[];
|
|
3057
|
+
cellPolicies?: Record<string, { staleAfterSeconds?: number }>;
|
|
3052
3058
|
}): Promise<{
|
|
3053
3059
|
inserted: number;
|
|
3054
3060
|
skipped: number;
|
|
@@ -3073,6 +3079,7 @@ async function prepareMapRows(input: {
|
|
|
3073
3079
|
rows: input.rows.map((row) => ({ ...row })),
|
|
3074
3080
|
runId: input.req.runId,
|
|
3075
3081
|
userEmail: input.req.userEmail,
|
|
3082
|
+
cellPolicies: input.cellPolicies,
|
|
3076
3083
|
});
|
|
3077
3084
|
for (const timing of result.timings ?? []) {
|
|
3078
3085
|
const phase =
|
|
@@ -3221,12 +3228,16 @@ function createMinimalWorkerCtx(
|
|
|
3221
3228
|
},
|
|
3222
3229
|
);
|
|
3223
3230
|
if (!response.ok) {
|
|
3224
|
-
throw new Error(
|
|
3231
|
+
throw new Error(
|
|
3232
|
+
`Secret ${auth.secret.name} is not available to this run.`,
|
|
3233
|
+
);
|
|
3225
3234
|
}
|
|
3226
3235
|
const payload = (await response.json()) as { value?: unknown };
|
|
3227
3236
|
const value = typeof payload.value === 'string' ? payload.value : null;
|
|
3228
3237
|
if (!value) {
|
|
3229
|
-
throw new Error(
|
|
3238
|
+
throw new Error(
|
|
3239
|
+
`Secret ${auth.secret.name} is not available to this run.`,
|
|
3240
|
+
);
|
|
3230
3241
|
}
|
|
3231
3242
|
secretRedactor.register(value);
|
|
3232
3243
|
return auth.kind === 'bearer'
|
|
@@ -3359,6 +3370,7 @@ function createMinimalWorkerCtx(
|
|
|
3359
3370
|
index: number,
|
|
3360
3371
|
) => Promise<unknown> | unknown)
|
|
3361
3372
|
>,
|
|
3373
|
+
cellPolicies?: Record<string, { staleAfterSeconds?: number }>,
|
|
3362
3374
|
opts?: WorkerMapOptions,
|
|
3363
3375
|
): Promise<unknown> => {
|
|
3364
3376
|
const mapStartedAt = nowMs();
|
|
@@ -3498,6 +3510,7 @@ function createMinimalWorkerCtx(
|
|
|
3498
3510
|
req,
|
|
3499
3511
|
tableNamespace: name,
|
|
3500
3512
|
outputFields,
|
|
3513
|
+
cellPolicies,
|
|
3501
3514
|
rows: chunkEntries.map(({ row, rowKey }) => ({
|
|
3502
3515
|
...row,
|
|
3503
3516
|
__deeplineRowKey: rowKey,
|
|
@@ -3572,10 +3585,11 @@ function createMinimalWorkerCtx(
|
|
|
3572
3585
|
| Record<
|
|
3573
3586
|
string,
|
|
3574
3587
|
{
|
|
3575
|
-
status: 'cached' | 'skipped';
|
|
3588
|
+
status: 'cached' | 'skipped' | 'completed';
|
|
3576
3589
|
stage?: string | null;
|
|
3577
3590
|
reused?: boolean;
|
|
3578
3591
|
runId?: string;
|
|
3592
|
+
completedAt?: number;
|
|
3579
3593
|
}
|
|
3580
3594
|
>
|
|
3581
3595
|
| undefined
|
|
@@ -3604,10 +3618,11 @@ function createMinimalWorkerCtx(
|
|
|
3604
3618
|
const cellMetaPatch: Record<
|
|
3605
3619
|
string,
|
|
3606
3620
|
{
|
|
3607
|
-
status: 'cached' | 'skipped';
|
|
3621
|
+
status: 'cached' | 'skipped' | 'completed';
|
|
3608
3622
|
stage?: string | null;
|
|
3609
3623
|
reused?: boolean;
|
|
3610
3624
|
runId?: string;
|
|
3625
|
+
completedAt?: number;
|
|
3611
3626
|
}
|
|
3612
3627
|
> = {};
|
|
3613
3628
|
const waterfallOutputs: RecordedWaterfallOutput[] = [];
|
|
@@ -3643,7 +3658,28 @@ function createMinimalWorkerCtx(
|
|
|
3643
3658
|
),
|
|
3644
3659
|
};
|
|
3645
3660
|
for (const [key, value] of fieldEntries) {
|
|
3646
|
-
|
|
3661
|
+
const rawCellMeta =
|
|
3662
|
+
enriched[DEEPLINE_CELL_META_FIELD] &&
|
|
3663
|
+
typeof enriched[DEEPLINE_CELL_META_FIELD] === 'object'
|
|
3664
|
+
? (
|
|
3665
|
+
enriched[DEEPLINE_CELL_META_FIELD] as Record<
|
|
3666
|
+
string,
|
|
3667
|
+
unknown
|
|
3668
|
+
>
|
|
3669
|
+
)[key]
|
|
3670
|
+
: null;
|
|
3671
|
+
const reuseDecision = shouldRecomputeCell({
|
|
3672
|
+
hasValue: isCompletedWorkerFieldValue(enriched[key]),
|
|
3673
|
+
meta:
|
|
3674
|
+
rawCellMeta && typeof rawCellMeta === 'object'
|
|
3675
|
+
? (rawCellMeta as {
|
|
3676
|
+
status?: string;
|
|
3677
|
+
completedAt?: number;
|
|
3678
|
+
})
|
|
3679
|
+
: null,
|
|
3680
|
+
policy: cellPolicies?.[key],
|
|
3681
|
+
});
|
|
3682
|
+
if (reuseDecision.action === 'reuse') {
|
|
3647
3683
|
cellMetaPatch[key] = {
|
|
3648
3684
|
status: 'cached',
|
|
3649
3685
|
stage: key,
|
|
@@ -3673,6 +3709,13 @@ function createMinimalWorkerCtx(
|
|
|
3673
3709
|
stage: key,
|
|
3674
3710
|
runId: req.runId,
|
|
3675
3711
|
};
|
|
3712
|
+
} else {
|
|
3713
|
+
cellMetaPatch[key] = {
|
|
3714
|
+
status: 'completed',
|
|
3715
|
+
stage: key,
|
|
3716
|
+
runId: req.runId,
|
|
3717
|
+
completedAt: nowMs(),
|
|
3718
|
+
};
|
|
3676
3719
|
}
|
|
3677
3720
|
}
|
|
3678
3721
|
for (const stepOutput of stepProgramOutputs) {
|
|
@@ -4037,7 +4080,15 @@ function createMinimalWorkerCtx(
|
|
|
4037
4080
|
const fields = Object.fromEntries(
|
|
4038
4081
|
program.steps.map((step) => [step.name, step.resolver]),
|
|
4039
4082
|
);
|
|
4040
|
-
|
|
4083
|
+
const cellPolicies = Object.fromEntries(
|
|
4084
|
+
program.steps.map((step) => [
|
|
4085
|
+
step.name,
|
|
4086
|
+
step.staleAfterSeconds === undefined
|
|
4087
|
+
? {}
|
|
4088
|
+
: { staleAfterSeconds: step.staleAfterSeconds },
|
|
4089
|
+
]),
|
|
4090
|
+
);
|
|
4091
|
+
return runMap(this.name, this.rows, fields, cellPolicies, opts);
|
|
4041
4092
|
},
|
|
4042
4093
|
{
|
|
4043
4094
|
emptyColumnName:
|
|
@@ -4255,7 +4306,15 @@ function createMinimalWorkerCtx(
|
|
|
4255
4306
|
const fields = Object.fromEntries(
|
|
4256
4307
|
fieldsDef.steps.map((step) => [step.name, step.resolver]),
|
|
4257
4308
|
);
|
|
4258
|
-
|
|
4309
|
+
const cellPolicies = Object.fromEntries(
|
|
4310
|
+
fieldsDef.steps.map((step) => [
|
|
4311
|
+
step.name,
|
|
4312
|
+
step.staleAfterSeconds === undefined
|
|
4313
|
+
? {}
|
|
4314
|
+
: { staleAfterSeconds: step.staleAfterSeconds },
|
|
4315
|
+
]),
|
|
4316
|
+
);
|
|
4317
|
+
return runMap(name, rows, fields, cellPolicies, opts);
|
|
4259
4318
|
}
|
|
4260
4319
|
throw new Error(
|
|
4261
4320
|
'ctx.dataset(key, rows, fields, options) is not supported. Use ctx.dataset(key, rows).withColumn(...).run(options).',
|
|
@@ -4266,11 +4325,11 @@ function createMinimalWorkerCtx(
|
|
|
4266
4325
|
'ctx.map(...) has been replaced by ctx.dataset(...). Use ctx.dataset(key, rows).withColumn(...).run(options).',
|
|
4267
4326
|
);
|
|
4268
4327
|
},
|
|
4269
|
-
|
|
4270
|
-
|
|
4328
|
+
tools: {
|
|
4329
|
+
async execute(requestArg: unknown): Promise<unknown> {
|
|
4271
4330
|
assertNotAborted(abortSignal);
|
|
4272
4331
|
const request = normalizeToolExecuteArgs(requestArg);
|
|
4273
|
-
|
|
4332
|
+
assertNoSecretTaint(request.input, 'ctx.tools.execute input');
|
|
4274
4333
|
return await executeWithRuntimeReceipt(
|
|
4275
4334
|
`tool:${request.id}:${deriveToolRequestIdentity({
|
|
4276
4335
|
toolId: request.toolId,
|
|
@@ -4340,9 +4399,9 @@ function createMinimalWorkerCtx(
|
|
|
4340
4399
|
timeoutMs?: number;
|
|
4341
4400
|
staleAfterSeconds?: number;
|
|
4342
4401
|
},
|
|
4343
|
-
|
|
4344
|
-
|
|
4345
|
-
|
|
4402
|
+
): Promise<unknown> {
|
|
4403
|
+
const normalizedKey = normalizeContextKey(key, 'runPlay');
|
|
4404
|
+
const resolvedName = resolvePlayRefName(playRef);
|
|
4346
4405
|
assertNoSecretTaint(input, 'ctx.runPlay input');
|
|
4347
4406
|
if (!resolvedName) {
|
|
4348
4407
|
throw new Error('ctx.runPlay(...) requires a resolvable play name.');
|
|
@@ -4582,18 +4641,15 @@ function createMinimalWorkerCtx(
|
|
|
4582
4641
|
}
|
|
4583
4642
|
});
|
|
4584
4643
|
},
|
|
4585
|
-
|
|
4586
|
-
|
|
4587
|
-
|
|
4588
|
-
|
|
4589
|
-
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
if (
|
|
4594
|
-
valueContainsSecret(input) ||
|
|
4595
|
-
valueContainsSecret(init.body)
|
|
4596
|
-
) {
|
|
4644
|
+
async fetch(
|
|
4645
|
+
key: string,
|
|
4646
|
+
input: string | URL,
|
|
4647
|
+
init: SecretAwareRequestInit = {},
|
|
4648
|
+
options?: { staleAfterSeconds?: number },
|
|
4649
|
+
): Promise<WorkerFetchResponse> {
|
|
4650
|
+
assertNotAborted(abortSignal);
|
|
4651
|
+
const normalizedKey = normalizeContextKey(key, 'fetch');
|
|
4652
|
+
if (valueContainsSecret(input) || valueContainsSecret(init.body)) {
|
|
4597
4653
|
throw new Error(
|
|
4598
4654
|
'ctx.fetch does not allow secrets in the URL or body. Use an approved secret auth helper.',
|
|
4599
4655
|
);
|
|
@@ -4606,10 +4662,10 @@ function createMinimalWorkerCtx(
|
|
|
4606
4662
|
if (init.auth !== undefined && !isSecretAuth(init.auth)) {
|
|
4607
4663
|
throw new Error('ctx.fetch auth must come from ctx.secrets.');
|
|
4608
4664
|
}
|
|
4609
|
-
|
|
4610
|
-
|
|
4665
|
+
const url = input.toString();
|
|
4666
|
+
const method = (init.method ?? 'GET').toUpperCase();
|
|
4611
4667
|
const secretHeaderMarkers = secretAuthHeaderMarkers(init.auth);
|
|
4612
|
-
|
|
4668
|
+
const safeHeaders = {
|
|
4613
4669
|
...normalizeFetchHeaders(init.headers),
|
|
4614
4670
|
...secretHeaderMarkers,
|
|
4615
4671
|
};
|
|
@@ -4628,7 +4684,7 @@ function createMinimalWorkerCtx(
|
|
|
4628
4684
|
safeHeaders,
|
|
4629
4685
|
url,
|
|
4630
4686
|
})}${staleRuntimeSuffix(options?.staleAfterSeconds)}`;
|
|
4631
|
-
|
|
4687
|
+
return await executeWithRuntimeReceipt(receiptKey, async () => {
|
|
4632
4688
|
const secretHeaders = await resolveSecretAuth(init.auth);
|
|
4633
4689
|
const headers = {
|
|
4634
4690
|
...normalizeFetchHeaders(init.headers),
|
|
@@ -4636,23 +4692,23 @@ function createMinimalWorkerCtx(
|
|
|
4636
4692
|
};
|
|
4637
4693
|
const fetchInit = { ...init, headers };
|
|
4638
4694
|
delete fetchInit.auth;
|
|
4639
|
-
|
|
4640
|
-
|
|
4641
|
-
|
|
4695
|
+
const response = await fetch(url, fetchInit);
|
|
4696
|
+
assertNotAborted(abortSignal);
|
|
4697
|
+
const bodyText = await response.text();
|
|
4642
4698
|
const redactedBodyText = secretRedactor.redactString(bodyText);
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
|
|
4652
|
-
|
|
4653
|
-
|
|
4654
|
-
|
|
4655
|
-
|
|
4699
|
+
return {
|
|
4700
|
+
ok: response.ok,
|
|
4701
|
+
status: response.status,
|
|
4702
|
+
statusText: response.statusText,
|
|
4703
|
+
url: response.url,
|
|
4704
|
+
headers: secretRedactor.redact(
|
|
4705
|
+
Object.fromEntries(response.headers.entries()),
|
|
4706
|
+
) as Record<string, string>,
|
|
4707
|
+
bodyText: redactedBodyText,
|
|
4708
|
+
json: secretRedactor.redact(parseFetchJsonOrNull(bodyText)),
|
|
4709
|
+
};
|
|
4710
|
+
});
|
|
4711
|
+
},
|
|
4656
4712
|
secrets: {
|
|
4657
4713
|
get(name: string): SecretHandle {
|
|
4658
4714
|
if (typeof name !== 'string' || !name.trim()) {
|
|
@@ -68,6 +68,7 @@ import type {
|
|
|
68
68
|
} from './types.js';
|
|
69
69
|
import type { PlayStagedFileRef } from './plays/local-file-discovery.js';
|
|
70
70
|
import type { PlayCompilerManifest } from '../../shared_libs/plays/compiler-manifest.js';
|
|
71
|
+
import type { EnrichCompiledConfig } from './cli/enrich-play-compiler.js';
|
|
71
72
|
|
|
72
73
|
const TERMINAL_PLAY_STATUSES = new Set(['completed', 'failed', 'cancelled']);
|
|
73
74
|
const INCLUDE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
|
|
@@ -1019,6 +1020,13 @@ export class DeeplineClient {
|
|
|
1019
1020
|
return this.http.post('/api/v2/plays/check', input);
|
|
1020
1021
|
}
|
|
1021
1022
|
|
|
1023
|
+
async compileEnrichPlan(input: {
|
|
1024
|
+
plan_args?: string[];
|
|
1025
|
+
config?: unknown;
|
|
1026
|
+
}): Promise<{ config: EnrichCompiledConfig }> {
|
|
1027
|
+
return this.http.post('/api/v2/enrich/compile', input);
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1022
1030
|
async startPlayRunFromBundle(input: {
|
|
1023
1031
|
name: string;
|
|
1024
1032
|
sourceCode: string;
|
|
@@ -224,6 +224,7 @@ export type ConditionalStepResolver<Row, Value, Else = null> = {
|
|
|
224
224
|
|
|
225
225
|
export type StepOptions<Row> = {
|
|
226
226
|
readonly runIf?: (row: Row, index: number) => boolean | Promise<boolean>;
|
|
227
|
+
readonly staleAfterSeconds?: number;
|
|
227
228
|
};
|
|
228
229
|
|
|
229
230
|
export type StepProgram<Input, Output, Return = Output> = {
|
|
@@ -257,6 +258,7 @@ export type StepProgramResolver<Input, Return> = {
|
|
|
257
258
|
|
|
258
259
|
export type PlayStepProgramStep = {
|
|
259
260
|
readonly name: string;
|
|
261
|
+
readonly staleAfterSeconds?: number;
|
|
260
262
|
readonly resolver:
|
|
261
263
|
| StepResolver<Record<string, unknown>, unknown>
|
|
262
264
|
| ConditionalStepResolver<Record<string, unknown>, unknown>
|
|
@@ -318,7 +320,6 @@ export type DatasetBuilder<
|
|
|
318
320
|
*/
|
|
319
321
|
run(options?: {
|
|
320
322
|
description?: string;
|
|
321
|
-
staleAfterSeconds?: number;
|
|
322
323
|
key?:
|
|
323
324
|
| (keyof InputRow & string)
|
|
324
325
|
| readonly (keyof InputRow & string)[]
|
|
@@ -191,6 +191,7 @@ export async function harnessStartSheetDataset(input: {
|
|
|
191
191
|
runId: string;
|
|
192
192
|
inputOffset?: number;
|
|
193
193
|
userEmail?: string | null;
|
|
194
|
+
cellPolicies?: Record<string, { staleAfterSeconds?: number }>;
|
|
194
195
|
}): Promise<{
|
|
195
196
|
inserted: number;
|
|
196
197
|
skipped: number;
|
|
@@ -50,10 +50,10 @@ export type SdkRelease = {
|
|
|
50
50
|
};
|
|
51
51
|
|
|
52
52
|
export const SDK_RELEASE = {
|
|
53
|
-
version: '0.1.
|
|
54
|
-
apiContract: '2026-06-dataset-column-
|
|
53
|
+
version: '0.1.77',
|
|
54
|
+
apiContract: '2026-06-dataset-column-cell-stale-hard-cutover',
|
|
55
55
|
supportPolicy: {
|
|
56
|
-
latest: '0.1.
|
|
56
|
+
latest: '0.1.77',
|
|
57
57
|
minimumSupported: '0.1.53',
|
|
58
58
|
deprecatedBelow: '0.1.53',
|
|
59
59
|
},
|
|
@@ -128,6 +128,9 @@ class WorkerStepProgram<Input, Output, ReturnValue> implements StepProgram<
|
|
|
128
128
|
{
|
|
129
129
|
name,
|
|
130
130
|
resolver: stepResolver as PlayStepProgramStep['resolver'],
|
|
131
|
+
...(options?.staleAfterSeconds !== undefined
|
|
132
|
+
? { staleAfterSeconds: options.staleAfterSeconds }
|
|
133
|
+
: {}),
|
|
131
134
|
},
|
|
132
135
|
],
|
|
133
136
|
this.returnResolver as StepResolver<
|