deepline 0.1.82 → 0.1.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +93 -24
- package/dist/cli/index.mjs +93 -24
- package/dist/index.d.mts +230 -10
- package/dist/index.d.ts +230 -10
- package/dist/index.js +115 -2
- package/dist/index.mjs +110 -2
- package/dist/repo/apps/play-runner-workers/src/entry.ts +123 -26
- package/dist/repo/sdk/src/index.ts +11 -0
- package/dist/repo/sdk/src/play.ts +61 -5
- package/dist/repo/sdk/src/plays/harness-stub.ts +2 -1
- package/dist/repo/sdk/src/release.ts +2 -2
- package/dist/repo/sdk/src/types.ts +4 -0
- package/dist/repo/shared_libs/play-runtime/cell-staleness.ts +156 -7
- package/dist/repo/shared_libs/play-runtime/execution-plan.ts +3 -3
- package/dist/repo/shared_libs/play-runtime/extractor-targets.ts +106 -0
- package/dist/repo/shared_libs/play-runtime/providers.ts +28 -0
- package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts +11 -0
- package/dist/repo/shared_libs/play-runtime/step-program-dataset-builder.ts +96 -6
- package/dist/repo/shared_libs/play-runtime/tool-result.ts +82 -0
- package/dist/repo/shared_libs/plays/static-pipeline.ts +133 -24
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -22,11 +22,15 @@ var src_exports = {};
|
|
|
22
22
|
__export(src_exports, {
|
|
23
23
|
AuthError: () => AuthError,
|
|
24
24
|
ConfigError: () => ConfigError,
|
|
25
|
+
DEEPLINE_EXTRACTOR_TARGETS: () => DEEPLINE_EXTRACTOR_TARGETS,
|
|
26
|
+
DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS: () => DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS,
|
|
25
27
|
DEEPLINE_TOOL_CATEGORIES: () => DEEPLINE_TOOL_CATEGORIES,
|
|
26
28
|
Deepline: () => Deepline,
|
|
27
29
|
DeeplineClient: () => DeeplineClient,
|
|
28
30
|
DeeplineContext: () => DeeplineContext,
|
|
29
31
|
DeeplineError: () => DeeplineError,
|
|
32
|
+
JOB_CHANGE_STATUS_VALUES: () => JOB_CHANGE_STATUS_VALUES,
|
|
33
|
+
PHONE_STATUS_VALUES: () => PHONE_STATUS_VALUES,
|
|
30
34
|
PLAY_BOOTSTRAP_COMPANY_FIELDS: () => PLAY_BOOTSTRAP_COMPANY_FIELDS,
|
|
31
35
|
PLAY_BOOTSTRAP_COMPANY_PROVIDER_CATEGORY: () => PLAY_BOOTSTRAP_COMPANY_PROVIDER_CATEGORY,
|
|
32
36
|
PLAY_BOOTSTRAP_CONTACT_FIELDS: () => PLAY_BOOTSTRAP_CONTACT_FIELDS,
|
|
@@ -49,6 +53,7 @@ __export(src_exports, {
|
|
|
49
53
|
formatPlayBootstrapFinderKindsForSentence: () => formatPlayBootstrapFinderKindsForSentence,
|
|
50
54
|
formatPlayBootstrapTemplates: () => formatPlayBootstrapTemplates,
|
|
51
55
|
getDefinedPlayMetadata: () => getDefinedPlayMetadata,
|
|
56
|
+
isDeeplineExtractorTarget: () => isDeeplineExtractorTarget,
|
|
52
57
|
isPlayBootstrapFinderKind: () => isPlayBootstrapFinderKind,
|
|
53
58
|
isPlayBootstrapTemplate: () => isPlayBootstrapTemplate,
|
|
54
59
|
resolveConfig: () => resolveConfig,
|
|
@@ -241,10 +246,10 @@ var import_node_path2 = require("path");
|
|
|
241
246
|
|
|
242
247
|
// src/release.ts
|
|
243
248
|
var SDK_RELEASE = {
|
|
244
|
-
version: "0.1.
|
|
249
|
+
version: "0.1.85",
|
|
245
250
|
apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
|
|
246
251
|
supportPolicy: {
|
|
247
|
-
latest: "0.1.
|
|
252
|
+
latest: "0.1.85",
|
|
248
253
|
minimumSupported: "0.1.53",
|
|
249
254
|
deprecatedBelow: "0.1.53"
|
|
250
255
|
}
|
|
@@ -2064,6 +2069,58 @@ function buildEmailStatus({
|
|
|
2064
2069
|
};
|
|
2065
2070
|
}
|
|
2066
2071
|
|
|
2072
|
+
// ../shared_libs/play-runtime/extractor-targets.ts
|
|
2073
|
+
var JOB_CHANGE_STATUS_VALUES = [
|
|
2074
|
+
"moved",
|
|
2075
|
+
"no_change",
|
|
2076
|
+
"left_company",
|
|
2077
|
+
"unknown",
|
|
2078
|
+
"profile_unavailable",
|
|
2079
|
+
"no_new_company"
|
|
2080
|
+
];
|
|
2081
|
+
var PHONE_STATUS_VALUES = ["valid", "invalid", "unknown"];
|
|
2082
|
+
var DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS = {
|
|
2083
|
+
id: { identity: true, valueKind: "string" },
|
|
2084
|
+
name: { identity: true, valueKind: "string" },
|
|
2085
|
+
email: { identity: true, valueKind: "string" },
|
|
2086
|
+
personal_email: { identity: true, valueKind: "string" },
|
|
2087
|
+
phone: { identity: true, valueKind: "string" },
|
|
2088
|
+
linkedin: { identity: true, valueKind: "string" },
|
|
2089
|
+
linkedin_url: { identity: true, valueKind: "string" },
|
|
2090
|
+
domain: { identity: true, valueKind: "string" },
|
|
2091
|
+
website: { identity: true, valueKind: "string" },
|
|
2092
|
+
first_name: { identity: true, valueKind: "string" },
|
|
2093
|
+
last_name: { identity: true, valueKind: "string" },
|
|
2094
|
+
full_name: { identity: true, valueKind: "string" },
|
|
2095
|
+
company: { identity: true, valueKind: "string" },
|
|
2096
|
+
company_name: { identity: true, valueKind: "string" },
|
|
2097
|
+
organization_name: { identity: true, valueKind: "string" },
|
|
2098
|
+
company_domain: { identity: true, valueKind: "string" },
|
|
2099
|
+
company_website: { identity: true, valueKind: "string" },
|
|
2100
|
+
company_linkedin_url: { identity: true, valueKind: "string" },
|
|
2101
|
+
title: { identity: false, valueKind: "string" },
|
|
2102
|
+
industry: { identity: false, valueKind: "string" },
|
|
2103
|
+
status: { identity: false, valueKind: "string" },
|
|
2104
|
+
job_change: { identity: false, valueKind: "job_change" },
|
|
2105
|
+
job_change_status: {
|
|
2106
|
+
identity: false,
|
|
2107
|
+
valueKind: "job_change_status",
|
|
2108
|
+
enum: JOB_CHANGE_STATUS_VALUES
|
|
2109
|
+
},
|
|
2110
|
+
email_status: { identity: false, valueKind: "email_status" },
|
|
2111
|
+
phone_status: {
|
|
2112
|
+
identity: false,
|
|
2113
|
+
valueKind: "phone_status",
|
|
2114
|
+
enum: PHONE_STATUS_VALUES
|
|
2115
|
+
}
|
|
2116
|
+
};
|
|
2117
|
+
var DEEPLINE_EXTRACTOR_TARGETS = Object.keys(
|
|
2118
|
+
DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS
|
|
2119
|
+
);
|
|
2120
|
+
function isDeeplineExtractorTarget(value) {
|
|
2121
|
+
return value in DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS;
|
|
2122
|
+
}
|
|
2123
|
+
|
|
2067
2124
|
// ../shared_libs/play-runtime/tool-result.ts
|
|
2068
2125
|
var TARGET_FALLBACK_KEYS = {
|
|
2069
2126
|
email: [/^email$/i, /^address$/i, /email/i],
|
|
@@ -2337,10 +2394,61 @@ function normalizePhoneStatus(value) {
|
|
|
2337
2394
|
}
|
|
2338
2395
|
return normalized;
|
|
2339
2396
|
}
|
|
2397
|
+
function normalizeJobChangeStatus(value) {
|
|
2398
|
+
if (typeof value === "boolean") return value ? "moved" : "no_change";
|
|
2399
|
+
const normalized = normalizeString(value)?.toLowerCase().replace(/[\s-]+/g, "_");
|
|
2400
|
+
if (!normalized) return "unknown";
|
|
2401
|
+
if (["true", "yes", "moved", "changed", "new_company"].includes(normalized)) {
|
|
2402
|
+
return "moved";
|
|
2403
|
+
}
|
|
2404
|
+
if (["false", "no", "same", "no_change"].includes(normalized))
|
|
2405
|
+
return "no_change";
|
|
2406
|
+
if (["left", "left_company"].includes(normalized)) return "left_company";
|
|
2407
|
+
if (JOB_CHANGE_STATUS_VALUES.includes(normalized)) {
|
|
2408
|
+
return normalized;
|
|
2409
|
+
}
|
|
2410
|
+
return "unknown";
|
|
2411
|
+
}
|
|
2412
|
+
function firstExperienceDate(value) {
|
|
2413
|
+
if (!Array.isArray(value)) return null;
|
|
2414
|
+
for (const entry of value) {
|
|
2415
|
+
if (!isRecord2(entry)) continue;
|
|
2416
|
+
const date = normalizeString(
|
|
2417
|
+
entry.start_date ?? entry.started_at ?? entry.startDate
|
|
2418
|
+
);
|
|
2419
|
+
if (date) return date;
|
|
2420
|
+
}
|
|
2421
|
+
return null;
|
|
2422
|
+
}
|
|
2423
|
+
function normalizeJobChange(value) {
|
|
2424
|
+
const record = isRecord2(value) ? value : {};
|
|
2425
|
+
const nested = isRecord2(record.job_change) ? record.job_change : record;
|
|
2426
|
+
const output = isRecord2(nested.output) ? nested.output : nested;
|
|
2427
|
+
const person = isRecord2(output.person) ? output.person : {};
|
|
2428
|
+
const status = normalizeJobChangeStatus(
|
|
2429
|
+
output.status ?? output.job_change_status ?? output.job_changed ?? output.changed
|
|
2430
|
+
);
|
|
2431
|
+
const moved = status === "moved";
|
|
2432
|
+
return {
|
|
2433
|
+
status,
|
|
2434
|
+
date: moved ? normalizeString(
|
|
2435
|
+
output.date ?? output.job_change_date ?? output.change_date ?? output.changed_at
|
|
2436
|
+
) ?? firstExperienceDate(person.experiences) : null,
|
|
2437
|
+
new_company: moved ? normalizeString(
|
|
2438
|
+
output.new_company ?? output.current_company ?? person.company_name ?? person.current_company
|
|
2439
|
+
) : null,
|
|
2440
|
+
new_title: moved ? normalizeString(
|
|
2441
|
+
output.new_title ?? output.current_title ?? person.title ?? person.headline
|
|
2442
|
+
) : null
|
|
2443
|
+
};
|
|
2444
|
+
}
|
|
2340
2445
|
function applyExtractorTransforms(value, descriptor) {
|
|
2341
2446
|
return (descriptor.transforms ?? []).reduce((current, transform) => {
|
|
2342
2447
|
if (transform.endsWith("emailStatus")) return normalizeEmailStatus(current);
|
|
2343
2448
|
if (transform.endsWith("phoneStatus")) return normalizePhoneStatus(current);
|
|
2449
|
+
if (transform === "jobChange") return normalizeJobChange(current);
|
|
2450
|
+
if (transform === "jobChangeStatus")
|
|
2451
|
+
return normalizeJobChangeStatus(current);
|
|
2344
2452
|
return current;
|
|
2345
2453
|
}, value);
|
|
2346
2454
|
}
|
|
@@ -3191,11 +3299,15 @@ function extractSummaryFields(payload) {
|
|
|
3191
3299
|
0 && (module.exports = {
|
|
3192
3300
|
AuthError,
|
|
3193
3301
|
ConfigError,
|
|
3302
|
+
DEEPLINE_EXTRACTOR_TARGETS,
|
|
3303
|
+
DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS,
|
|
3194
3304
|
DEEPLINE_TOOL_CATEGORIES,
|
|
3195
3305
|
Deepline,
|
|
3196
3306
|
DeeplineClient,
|
|
3197
3307
|
DeeplineContext,
|
|
3198
3308
|
DeeplineError,
|
|
3309
|
+
JOB_CHANGE_STATUS_VALUES,
|
|
3310
|
+
PHONE_STATUS_VALUES,
|
|
3199
3311
|
PLAY_BOOTSTRAP_COMPANY_FIELDS,
|
|
3200
3312
|
PLAY_BOOTSTRAP_COMPANY_PROVIDER_CATEGORY,
|
|
3201
3313
|
PLAY_BOOTSTRAP_CONTACT_FIELDS,
|
|
@@ -3218,6 +3330,7 @@ function extractSummaryFields(payload) {
|
|
|
3218
3330
|
formatPlayBootstrapFinderKindsForSentence,
|
|
3219
3331
|
formatPlayBootstrapTemplates,
|
|
3220
3332
|
getDefinedPlayMetadata,
|
|
3333
|
+
isDeeplineExtractorTarget,
|
|
3221
3334
|
isPlayBootstrapFinderKind,
|
|
3222
3335
|
isPlayBootstrapTemplate,
|
|
3223
3336
|
resolveConfig,
|
package/dist/index.mjs
CHANGED
|
@@ -179,10 +179,10 @@ import { join as join2 } from "path";
|
|
|
179
179
|
|
|
180
180
|
// src/release.ts
|
|
181
181
|
var SDK_RELEASE = {
|
|
182
|
-
version: "0.1.
|
|
182
|
+
version: "0.1.85",
|
|
183
183
|
apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
|
|
184
184
|
supportPolicy: {
|
|
185
|
-
latest: "0.1.
|
|
185
|
+
latest: "0.1.85",
|
|
186
186
|
minimumSupported: "0.1.53",
|
|
187
187
|
deprecatedBelow: "0.1.53"
|
|
188
188
|
}
|
|
@@ -2002,6 +2002,58 @@ function buildEmailStatus({
|
|
|
2002
2002
|
};
|
|
2003
2003
|
}
|
|
2004
2004
|
|
|
2005
|
+
// ../shared_libs/play-runtime/extractor-targets.ts
|
|
2006
|
+
var JOB_CHANGE_STATUS_VALUES = [
|
|
2007
|
+
"moved",
|
|
2008
|
+
"no_change",
|
|
2009
|
+
"left_company",
|
|
2010
|
+
"unknown",
|
|
2011
|
+
"profile_unavailable",
|
|
2012
|
+
"no_new_company"
|
|
2013
|
+
];
|
|
2014
|
+
var PHONE_STATUS_VALUES = ["valid", "invalid", "unknown"];
|
|
2015
|
+
var DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS = {
|
|
2016
|
+
id: { identity: true, valueKind: "string" },
|
|
2017
|
+
name: { identity: true, valueKind: "string" },
|
|
2018
|
+
email: { identity: true, valueKind: "string" },
|
|
2019
|
+
personal_email: { identity: true, valueKind: "string" },
|
|
2020
|
+
phone: { identity: true, valueKind: "string" },
|
|
2021
|
+
linkedin: { identity: true, valueKind: "string" },
|
|
2022
|
+
linkedin_url: { identity: true, valueKind: "string" },
|
|
2023
|
+
domain: { identity: true, valueKind: "string" },
|
|
2024
|
+
website: { identity: true, valueKind: "string" },
|
|
2025
|
+
first_name: { identity: true, valueKind: "string" },
|
|
2026
|
+
last_name: { identity: true, valueKind: "string" },
|
|
2027
|
+
full_name: { identity: true, valueKind: "string" },
|
|
2028
|
+
company: { identity: true, valueKind: "string" },
|
|
2029
|
+
company_name: { identity: true, valueKind: "string" },
|
|
2030
|
+
organization_name: { identity: true, valueKind: "string" },
|
|
2031
|
+
company_domain: { identity: true, valueKind: "string" },
|
|
2032
|
+
company_website: { identity: true, valueKind: "string" },
|
|
2033
|
+
company_linkedin_url: { identity: true, valueKind: "string" },
|
|
2034
|
+
title: { identity: false, valueKind: "string" },
|
|
2035
|
+
industry: { identity: false, valueKind: "string" },
|
|
2036
|
+
status: { identity: false, valueKind: "string" },
|
|
2037
|
+
job_change: { identity: false, valueKind: "job_change" },
|
|
2038
|
+
job_change_status: {
|
|
2039
|
+
identity: false,
|
|
2040
|
+
valueKind: "job_change_status",
|
|
2041
|
+
enum: JOB_CHANGE_STATUS_VALUES
|
|
2042
|
+
},
|
|
2043
|
+
email_status: { identity: false, valueKind: "email_status" },
|
|
2044
|
+
phone_status: {
|
|
2045
|
+
identity: false,
|
|
2046
|
+
valueKind: "phone_status",
|
|
2047
|
+
enum: PHONE_STATUS_VALUES
|
|
2048
|
+
}
|
|
2049
|
+
};
|
|
2050
|
+
var DEEPLINE_EXTRACTOR_TARGETS = Object.keys(
|
|
2051
|
+
DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS
|
|
2052
|
+
);
|
|
2053
|
+
function isDeeplineExtractorTarget(value) {
|
|
2054
|
+
return value in DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS;
|
|
2055
|
+
}
|
|
2056
|
+
|
|
2005
2057
|
// ../shared_libs/play-runtime/tool-result.ts
|
|
2006
2058
|
var TARGET_FALLBACK_KEYS = {
|
|
2007
2059
|
email: [/^email$/i, /^address$/i, /email/i],
|
|
@@ -2275,10 +2327,61 @@ function normalizePhoneStatus(value) {
|
|
|
2275
2327
|
}
|
|
2276
2328
|
return normalized;
|
|
2277
2329
|
}
|
|
2330
|
+
function normalizeJobChangeStatus(value) {
|
|
2331
|
+
if (typeof value === "boolean") return value ? "moved" : "no_change";
|
|
2332
|
+
const normalized = normalizeString(value)?.toLowerCase().replace(/[\s-]+/g, "_");
|
|
2333
|
+
if (!normalized) return "unknown";
|
|
2334
|
+
if (["true", "yes", "moved", "changed", "new_company"].includes(normalized)) {
|
|
2335
|
+
return "moved";
|
|
2336
|
+
}
|
|
2337
|
+
if (["false", "no", "same", "no_change"].includes(normalized))
|
|
2338
|
+
return "no_change";
|
|
2339
|
+
if (["left", "left_company"].includes(normalized)) return "left_company";
|
|
2340
|
+
if (JOB_CHANGE_STATUS_VALUES.includes(normalized)) {
|
|
2341
|
+
return normalized;
|
|
2342
|
+
}
|
|
2343
|
+
return "unknown";
|
|
2344
|
+
}
|
|
2345
|
+
function firstExperienceDate(value) {
|
|
2346
|
+
if (!Array.isArray(value)) return null;
|
|
2347
|
+
for (const entry of value) {
|
|
2348
|
+
if (!isRecord2(entry)) continue;
|
|
2349
|
+
const date = normalizeString(
|
|
2350
|
+
entry.start_date ?? entry.started_at ?? entry.startDate
|
|
2351
|
+
);
|
|
2352
|
+
if (date) return date;
|
|
2353
|
+
}
|
|
2354
|
+
return null;
|
|
2355
|
+
}
|
|
2356
|
+
function normalizeJobChange(value) {
|
|
2357
|
+
const record = isRecord2(value) ? value : {};
|
|
2358
|
+
const nested = isRecord2(record.job_change) ? record.job_change : record;
|
|
2359
|
+
const output = isRecord2(nested.output) ? nested.output : nested;
|
|
2360
|
+
const person = isRecord2(output.person) ? output.person : {};
|
|
2361
|
+
const status = normalizeJobChangeStatus(
|
|
2362
|
+
output.status ?? output.job_change_status ?? output.job_changed ?? output.changed
|
|
2363
|
+
);
|
|
2364
|
+
const moved = status === "moved";
|
|
2365
|
+
return {
|
|
2366
|
+
status,
|
|
2367
|
+
date: moved ? normalizeString(
|
|
2368
|
+
output.date ?? output.job_change_date ?? output.change_date ?? output.changed_at
|
|
2369
|
+
) ?? firstExperienceDate(person.experiences) : null,
|
|
2370
|
+
new_company: moved ? normalizeString(
|
|
2371
|
+
output.new_company ?? output.current_company ?? person.company_name ?? person.current_company
|
|
2372
|
+
) : null,
|
|
2373
|
+
new_title: moved ? normalizeString(
|
|
2374
|
+
output.new_title ?? output.current_title ?? person.title ?? person.headline
|
|
2375
|
+
) : null
|
|
2376
|
+
};
|
|
2377
|
+
}
|
|
2278
2378
|
function applyExtractorTransforms(value, descriptor) {
|
|
2279
2379
|
return (descriptor.transforms ?? []).reduce((current, transform) => {
|
|
2280
2380
|
if (transform.endsWith("emailStatus")) return normalizeEmailStatus(current);
|
|
2281
2381
|
if (transform.endsWith("phoneStatus")) return normalizePhoneStatus(current);
|
|
2382
|
+
if (transform === "jobChange") return normalizeJobChange(current);
|
|
2383
|
+
if (transform === "jobChangeStatus")
|
|
2384
|
+
return normalizeJobChangeStatus(current);
|
|
2282
2385
|
return current;
|
|
2283
2386
|
}, value);
|
|
2284
2387
|
}
|
|
@@ -3128,11 +3231,15 @@ function extractSummaryFields(payload) {
|
|
|
3128
3231
|
export {
|
|
3129
3232
|
AuthError,
|
|
3130
3233
|
ConfigError,
|
|
3234
|
+
DEEPLINE_EXTRACTOR_TARGETS,
|
|
3235
|
+
DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS,
|
|
3131
3236
|
DEEPLINE_TOOL_CATEGORIES,
|
|
3132
3237
|
Deepline,
|
|
3133
3238
|
DeeplineClient,
|
|
3134
3239
|
DeeplineContext,
|
|
3135
3240
|
DeeplineError,
|
|
3241
|
+
JOB_CHANGE_STATUS_VALUES,
|
|
3242
|
+
PHONE_STATUS_VALUES,
|
|
3136
3243
|
PLAY_BOOTSTRAP_COMPANY_FIELDS,
|
|
3137
3244
|
PLAY_BOOTSTRAP_COMPANY_PROVIDER_CATEGORY,
|
|
3138
3245
|
PLAY_BOOTSTRAP_CONTACT_FIELDS,
|
|
@@ -3155,6 +3262,7 @@ export {
|
|
|
3155
3262
|
formatPlayBootstrapFinderKindsForSentence,
|
|
3156
3263
|
formatPlayBootstrapTemplates,
|
|
3157
3264
|
getDefinedPlayMetadata,
|
|
3265
|
+
isDeeplineExtractorTarget,
|
|
3158
3266
|
isPlayBootstrapFinderKind,
|
|
3159
3267
|
isPlayBootstrapTemplate,
|
|
3160
3268
|
resolveConfig,
|
|
@@ -163,11 +163,19 @@ import {
|
|
|
163
163
|
} from './runtime/tool-http-errors';
|
|
164
164
|
import {
|
|
165
165
|
StepProgramDatasetBuilder,
|
|
166
|
+
type StepProgramDatasetColumnInput,
|
|
166
167
|
type StepProgramDatasetOptions,
|
|
167
168
|
} from '../../../shared_libs/play-runtime/step-program-dataset-builder';
|
|
168
169
|
import {
|
|
169
170
|
DEEPLINE_CELL_META_FIELD,
|
|
171
|
+
normalizeCellStalenessPolicy,
|
|
172
|
+
previousCellFromValue,
|
|
173
|
+
resolveCompletedCellStalenessMeta,
|
|
170
174
|
shouldRecomputeCell,
|
|
175
|
+
type AuthoredCellStalenessPolicyByField,
|
|
176
|
+
type AuthoredStaleAfterSeconds,
|
|
177
|
+
type CellStalenessPolicyByField,
|
|
178
|
+
type PreviousCell,
|
|
171
179
|
} from '../../../shared_libs/play-runtime/cell-staleness';
|
|
172
180
|
|
|
173
181
|
// The play's default export. The bundler injects this — see bundle-play-file.ts.
|
|
@@ -717,10 +725,7 @@ function publicCsvStorageRow<T extends Record<string, unknown>>(row: T): T {
|
|
|
717
725
|
storageRow[fieldName] =
|
|
718
726
|
'value' in descriptor ? descriptor.value : publicRow[fieldName];
|
|
719
727
|
}
|
|
720
|
-
for (const runtimeField of [
|
|
721
|
-
'__deeplineRowKey',
|
|
722
|
-
'__deeplineCellMetaPatch',
|
|
723
|
-
]) {
|
|
728
|
+
for (const runtimeField of ['__deeplineRowKey', '__deeplineCellMetaPatch']) {
|
|
724
729
|
if (Object.prototype.hasOwnProperty.call(row, runtimeField)) {
|
|
725
730
|
storageRow[runtimeField] = row[runtimeField];
|
|
726
731
|
}
|
|
@@ -1973,6 +1978,7 @@ type WorkerStepResolver = (
|
|
|
1973
1978
|
row: Record<string, unknown>,
|
|
1974
1979
|
ctx: unknown,
|
|
1975
1980
|
index: number,
|
|
1981
|
+
previousCell?: PreviousCell,
|
|
1976
1982
|
) => Promise<unknown> | unknown;
|
|
1977
1983
|
|
|
1978
1984
|
type WorkerConditionalStepResolver = {
|
|
@@ -1987,7 +1993,7 @@ type WorkerConditionalStepResolver = {
|
|
|
1987
1993
|
|
|
1988
1994
|
type WorkerStepProgramStep = {
|
|
1989
1995
|
name: string;
|
|
1990
|
-
staleAfterSeconds?:
|
|
1996
|
+
staleAfterSeconds?: AuthoredStaleAfterSeconds;
|
|
1991
1997
|
resolver:
|
|
1992
1998
|
| WorkerStepResolver
|
|
1993
1999
|
| WorkerConditionalStepResolver
|
|
@@ -2042,6 +2048,7 @@ async function executeWorkerStepResolver(
|
|
|
2042
2048
|
row: Record<string, unknown>,
|
|
2043
2049
|
ctx: unknown,
|
|
2044
2050
|
index: number,
|
|
2051
|
+
previousCell?: PreviousCell,
|
|
2045
2052
|
recorder?: {
|
|
2046
2053
|
parentField: string;
|
|
2047
2054
|
path: string[];
|
|
@@ -2051,7 +2058,14 @@ async function executeWorkerStepResolver(
|
|
|
2051
2058
|
if (isWorkerConditionalStepResolver(resolver)) {
|
|
2052
2059
|
const shouldRun = await resolver.when(row, index);
|
|
2053
2060
|
return shouldRun
|
|
2054
|
-
? await executeWorkerStepResolver(
|
|
2061
|
+
? await executeWorkerStepResolver(
|
|
2062
|
+
resolver.run,
|
|
2063
|
+
row,
|
|
2064
|
+
ctx,
|
|
2065
|
+
index,
|
|
2066
|
+
previousCell,
|
|
2067
|
+
recorder,
|
|
2068
|
+
)
|
|
2055
2069
|
: {
|
|
2056
2070
|
value: resolver.elseValue ?? null,
|
|
2057
2071
|
status: 'skipped',
|
|
@@ -2069,7 +2083,14 @@ async function executeWorkerStepResolver(
|
|
|
2069
2083
|
};
|
|
2070
2084
|
}
|
|
2071
2085
|
if (typeof resolver === 'function') {
|
|
2072
|
-
return {
|
|
2086
|
+
return {
|
|
2087
|
+
value: await (resolver as WorkerStepResolver)(
|
|
2088
|
+
row,
|
|
2089
|
+
ctx,
|
|
2090
|
+
index,
|
|
2091
|
+
previousCell,
|
|
2092
|
+
),
|
|
2093
|
+
};
|
|
2073
2094
|
}
|
|
2074
2095
|
return { value: resolver };
|
|
2075
2096
|
}
|
|
@@ -2095,6 +2116,7 @@ async function executeWorkerStepProgram(
|
|
|
2095
2116
|
currentRow,
|
|
2096
2117
|
ctx,
|
|
2097
2118
|
index,
|
|
2119
|
+
undefined,
|
|
2098
2120
|
recorder
|
|
2099
2121
|
? {
|
|
2100
2122
|
...recorder,
|
|
@@ -3005,7 +3027,7 @@ async function prepareMapRows(input: {
|
|
|
3005
3027
|
tableNamespace: string;
|
|
3006
3028
|
rows: Record<string, unknown>[];
|
|
3007
3029
|
outputFields: string[];
|
|
3008
|
-
cellPolicies?:
|
|
3030
|
+
cellPolicies?: CellStalenessPolicyByField;
|
|
3009
3031
|
}): Promise<{
|
|
3010
3032
|
inserted: number;
|
|
3011
3033
|
skipped: number;
|
|
@@ -3161,18 +3183,21 @@ function createCoordinatorRatePort(req: RunRequest): CoordinatorRatePort {
|
|
|
3161
3183
|
if (!coordinatorUrl) {
|
|
3162
3184
|
throw new Error('Coordinator rate acquire is unavailable.');
|
|
3163
3185
|
}
|
|
3164
|
-
const res = await fetch(
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
'
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3186
|
+
const res = await fetch(
|
|
3187
|
+
`${coordinatorUrl.replace(/\/$/, '')}/rate-acquire`,
|
|
3188
|
+
{
|
|
3189
|
+
method: 'POST',
|
|
3190
|
+
headers: {
|
|
3191
|
+
'x-deepline-request-id': makeRequestId(),
|
|
3192
|
+
...coordinatorRequestHeaders({
|
|
3193
|
+
runId: req.runId,
|
|
3194
|
+
contentType: 'application/json',
|
|
3195
|
+
internalToken: req.coordinatorInternalToken,
|
|
3196
|
+
}),
|
|
3197
|
+
},
|
|
3198
|
+
body: JSON.stringify(input),
|
|
3173
3199
|
},
|
|
3174
|
-
|
|
3175
|
-
});
|
|
3200
|
+
);
|
|
3176
3201
|
if (!res.ok) {
|
|
3177
3202
|
const text = await res.text().catch(() => '');
|
|
3178
3203
|
throw new Error(
|
|
@@ -3494,7 +3519,8 @@ function createMinimalWorkerCtx(
|
|
|
3494
3519
|
index: number,
|
|
3495
3520
|
) => Promise<unknown> | unknown)
|
|
3496
3521
|
>,
|
|
3497
|
-
cellPolicies?:
|
|
3522
|
+
cellPolicies?: CellStalenessPolicyByField,
|
|
3523
|
+
authoredCellPolicies?: AuthoredCellStalenessPolicyByField,
|
|
3498
3524
|
opts?: WorkerMapOptions,
|
|
3499
3525
|
): Promise<unknown> => {
|
|
3500
3526
|
const mapStartedAt = nowMs();
|
|
@@ -3760,6 +3786,8 @@ function createMinimalWorkerCtx(
|
|
|
3760
3786
|
reused?: boolean;
|
|
3761
3787
|
runId?: string;
|
|
3762
3788
|
completedAt?: number;
|
|
3789
|
+
staleAt?: number | null;
|
|
3790
|
+
staleAfterSeconds?: number | null;
|
|
3763
3791
|
}
|
|
3764
3792
|
>
|
|
3765
3793
|
| undefined
|
|
@@ -3786,10 +3814,18 @@ function createMinimalWorkerCtx(
|
|
|
3786
3814
|
});
|
|
3787
3815
|
try {
|
|
3788
3816
|
const entry = uniqueRowsToExecuteEntries[myIndex]!;
|
|
3789
|
-
const
|
|
3817
|
+
const pendingRow = pendingRowsByKey.get(entry.rowKey);
|
|
3818
|
+
const row = pendingRow
|
|
3790
3819
|
? ({
|
|
3791
3820
|
...entry.row,
|
|
3792
|
-
...publicCsvInputRow(
|
|
3821
|
+
...publicCsvInputRow(pendingRow),
|
|
3822
|
+
...(pendingRow[DEEPLINE_CELL_META_FIELD] &&
|
|
3823
|
+
typeof pendingRow[DEEPLINE_CELL_META_FIELD] === 'object'
|
|
3824
|
+
? {
|
|
3825
|
+
[DEEPLINE_CELL_META_FIELD]:
|
|
3826
|
+
pendingRow[DEEPLINE_CELL_META_FIELD],
|
|
3827
|
+
}
|
|
3828
|
+
: {}),
|
|
3793
3829
|
} as T & Record<string, unknown>)
|
|
3794
3830
|
: entry.row;
|
|
3795
3831
|
const absoluteIndex = entry.absoluteIndex;
|
|
@@ -3804,6 +3840,8 @@ function createMinimalWorkerCtx(
|
|
|
3804
3840
|
reused?: boolean;
|
|
3805
3841
|
runId?: string;
|
|
3806
3842
|
completedAt?: number;
|
|
3843
|
+
staleAt?: number | null;
|
|
3844
|
+
staleAfterSeconds?: number | null;
|
|
3807
3845
|
}
|
|
3808
3846
|
> = {};
|
|
3809
3847
|
const waterfallOutputs: RecordedWaterfallOutput[] = [];
|
|
@@ -3857,10 +3895,25 @@ function createMinimalWorkerCtx(
|
|
|
3857
3895
|
? (rawCellMeta as {
|
|
3858
3896
|
status?: string;
|
|
3859
3897
|
completedAt?: number;
|
|
3898
|
+
staleAt?: number | null;
|
|
3899
|
+
staleAfterSeconds?: number | null;
|
|
3860
3900
|
})
|
|
3861
3901
|
: null,
|
|
3862
3902
|
policy: cellPolicies?.[key],
|
|
3863
3903
|
});
|
|
3904
|
+
const previousCell = previousCellFromValue({
|
|
3905
|
+
hasValue: isCompletedWorkerFieldValue(enriched[key]),
|
|
3906
|
+
value: enriched[key],
|
|
3907
|
+
meta:
|
|
3908
|
+
rawCellMeta && typeof rawCellMeta === 'object'
|
|
3909
|
+
? (rawCellMeta as {
|
|
3910
|
+
status?: string;
|
|
3911
|
+
completedAt?: number;
|
|
3912
|
+
staleAt?: number | null;
|
|
3913
|
+
staleAfterSeconds?: number | null;
|
|
3914
|
+
})
|
|
3915
|
+
: null,
|
|
3916
|
+
});
|
|
3864
3917
|
if (reuseDecision.action === 'reuse') {
|
|
3865
3918
|
cellMetaPatch[key] = {
|
|
3866
3919
|
status: 'cached',
|
|
@@ -3875,6 +3928,7 @@ function createMinimalWorkerCtx(
|
|
|
3875
3928
|
enriched,
|
|
3876
3929
|
rowCtx,
|
|
3877
3930
|
absoluteIndex,
|
|
3931
|
+
previousCell,
|
|
3878
3932
|
isWorkerStepProgram(value)
|
|
3879
3933
|
? {
|
|
3880
3934
|
parentField: key,
|
|
@@ -3892,11 +3946,18 @@ function createMinimalWorkerCtx(
|
|
|
3892
3946
|
runId: req.runId,
|
|
3893
3947
|
};
|
|
3894
3948
|
} else {
|
|
3949
|
+
const completedAt = nowMs();
|
|
3950
|
+
const stalenessMeta = resolveCompletedCellStalenessMeta({
|
|
3951
|
+
policy: authoredCellPolicies?.[key],
|
|
3952
|
+
value: resolved.value,
|
|
3953
|
+
completedAt,
|
|
3954
|
+
});
|
|
3895
3955
|
cellMetaPatch[key] = {
|
|
3896
3956
|
status: 'completed',
|
|
3897
3957
|
stage: key,
|
|
3898
3958
|
runId: req.runId,
|
|
3899
|
-
completedAt
|
|
3959
|
+
completedAt,
|
|
3960
|
+
...stalenessMeta,
|
|
3900
3961
|
};
|
|
3901
3962
|
}
|
|
3902
3963
|
}
|
|
@@ -4266,14 +4327,31 @@ function createMinimalWorkerCtx(
|
|
|
4266
4327
|
program.steps.map((step) => [step.name, step.resolver]),
|
|
4267
4328
|
);
|
|
4268
4329
|
const cellPolicies = Object.fromEntries(
|
|
4330
|
+
program.steps.map((step) => [
|
|
4331
|
+
step.name,
|
|
4332
|
+
step.staleAfterSeconds === undefined
|
|
4333
|
+
? {}
|
|
4334
|
+
: normalizeCellStalenessPolicy({
|
|
4335
|
+
staleAfterSeconds: step.staleAfterSeconds,
|
|
4336
|
+
}),
|
|
4337
|
+
]),
|
|
4338
|
+
) as CellStalenessPolicyByField;
|
|
4339
|
+
const authoredCellPolicies = Object.fromEntries(
|
|
4269
4340
|
program.steps.map((step) => [
|
|
4270
4341
|
step.name,
|
|
4271
4342
|
step.staleAfterSeconds === undefined
|
|
4272
4343
|
? {}
|
|
4273
4344
|
: { staleAfterSeconds: step.staleAfterSeconds },
|
|
4274
4345
|
]),
|
|
4346
|
+
) as AuthoredCellStalenessPolicyByField;
|
|
4347
|
+
return runMap(
|
|
4348
|
+
this.name,
|
|
4349
|
+
this.rows,
|
|
4350
|
+
fields,
|
|
4351
|
+
cellPolicies,
|
|
4352
|
+
authoredCellPolicies,
|
|
4353
|
+
opts,
|
|
4275
4354
|
);
|
|
4276
|
-
return runMap(this.name, this.rows, fields, cellPolicies, opts);
|
|
4277
4355
|
},
|
|
4278
4356
|
{
|
|
4279
4357
|
emptyColumnName:
|
|
@@ -4288,7 +4366,9 @@ function createMinimalWorkerCtx(
|
|
|
4288
4366
|
|
|
4289
4367
|
withColumn(
|
|
4290
4368
|
name: string,
|
|
4291
|
-
resolver:
|
|
4369
|
+
resolver: StepProgramDatasetColumnInput<
|
|
4370
|
+
WorkerStepProgramStep['resolver']
|
|
4371
|
+
>,
|
|
4292
4372
|
options?: StepProgramDatasetOptions,
|
|
4293
4373
|
): this {
|
|
4294
4374
|
this.builder.withColumn(name, resolver, options);
|
|
@@ -4492,14 +4572,31 @@ function createMinimalWorkerCtx(
|
|
|
4492
4572
|
fieldsDef.steps.map((step) => [step.name, step.resolver]),
|
|
4493
4573
|
);
|
|
4494
4574
|
const cellPolicies = Object.fromEntries(
|
|
4575
|
+
fieldsDef.steps.map((step) => [
|
|
4576
|
+
step.name,
|
|
4577
|
+
step.staleAfterSeconds === undefined
|
|
4578
|
+
? {}
|
|
4579
|
+
: normalizeCellStalenessPolicy({
|
|
4580
|
+
staleAfterSeconds: step.staleAfterSeconds,
|
|
4581
|
+
}),
|
|
4582
|
+
]),
|
|
4583
|
+
) as CellStalenessPolicyByField;
|
|
4584
|
+
const authoredCellPolicies = Object.fromEntries(
|
|
4495
4585
|
fieldsDef.steps.map((step) => [
|
|
4496
4586
|
step.name,
|
|
4497
4587
|
step.staleAfterSeconds === undefined
|
|
4498
4588
|
? {}
|
|
4499
4589
|
: { staleAfterSeconds: step.staleAfterSeconds },
|
|
4500
4590
|
]),
|
|
4591
|
+
) as AuthoredCellStalenessPolicyByField;
|
|
4592
|
+
return runMap(
|
|
4593
|
+
name,
|
|
4594
|
+
rows,
|
|
4595
|
+
fields,
|
|
4596
|
+
cellPolicies,
|
|
4597
|
+
authoredCellPolicies,
|
|
4598
|
+
opts,
|
|
4501
4599
|
);
|
|
4502
|
-
return runMap(name, rows, fields, cellPolicies, opts);
|
|
4503
4600
|
}
|
|
4504
4601
|
throw new Error(
|
|
4505
4602
|
'ctx.dataset(key, rows, fields, options) is not supported. Use ctx.dataset(key, rows).withColumn(...).run(options).',
|
|
@@ -91,10 +91,15 @@ export { resolveConfig, PROD_URL } from './config.js';
|
|
|
91
91
|
export {
|
|
92
92
|
Deepline,
|
|
93
93
|
DeeplineContext,
|
|
94
|
+
DEEPLINE_EXTRACTOR_TARGETS,
|
|
95
|
+
DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS,
|
|
94
96
|
defineInput,
|
|
95
97
|
definePlay,
|
|
96
98
|
defineWorkflow,
|
|
99
|
+
JOB_CHANGE_STATUS_VALUES,
|
|
100
|
+
PHONE_STATUS_VALUES,
|
|
97
101
|
getDefinedPlayMetadata,
|
|
102
|
+
isDeeplineExtractorTarget,
|
|
98
103
|
runIf,
|
|
99
104
|
steps,
|
|
100
105
|
} from './play.js';
|
|
@@ -151,4 +156,10 @@ export type {
|
|
|
151
156
|
StepProgramResolver,
|
|
152
157
|
StepResolver,
|
|
153
158
|
ToolExecuteResult,
|
|
159
|
+
DeeplineEmailStatusGetterValue,
|
|
160
|
+
DeeplineExtractorTarget,
|
|
161
|
+
DeeplineGetterValue,
|
|
162
|
+
DeeplineGetterValueMap,
|
|
163
|
+
JobChangeStatus,
|
|
164
|
+
PhoneStatus,
|
|
154
165
|
} from './play.js';
|