deepline 0.1.83 → 0.1.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +481 -79
- package/dist/cli/index.mjs +500 -92
- package/dist/index.d.mts +604 -28
- package/dist/index.d.ts +604 -28
- package/dist/index.js +274 -4
- package/dist/index.mjs +269 -4
- package/dist/repo/apps/play-runner-workers/src/dedup-do.ts +1 -0
- package/dist/repo/apps/play-runner-workers/src/entry.ts +345 -72
- package/dist/repo/sdk/src/client.ts +155 -1
- package/dist/repo/sdk/src/http.ts +11 -0
- package/dist/repo/sdk/src/index.ts +35 -1
- package/dist/repo/sdk/src/play.ts +254 -15
- package/dist/repo/sdk/src/plays/harness-stub.ts +2 -1
- package/dist/repo/sdk/src/release.ts +2 -2
- package/dist/repo/sdk/src/types.ts +65 -0
- package/dist/repo/sdk/src/worker-play-entry.ts +6 -3
- package/dist/repo/shared_libs/play-runtime/cell-staleness.ts +179 -7
- package/dist/repo/shared_libs/play-runtime/coordinator-headers.ts +10 -0
- package/dist/repo/shared_libs/play-runtime/execution-plan.ts +3 -3
- package/dist/repo/shared_libs/play-runtime/extractor-targets.ts +106 -0
- package/dist/repo/shared_libs/play-runtime/providers.ts +28 -0
- package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts +21 -1
- package/dist/repo/shared_libs/play-runtime/step-program-dataset-builder.ts +96 -6
- package/dist/repo/shared_libs/play-runtime/tool-result.ts +82 -0
- package/dist/repo/shared_libs/plays/bundling/index.ts +23 -16
- package/dist/repo/shared_libs/plays/dataset.ts +2 -0
- package/dist/repo/shared_libs/plays/static-pipeline.ts +133 -24
- package/package.json +1 -1
|
@@ -101,7 +101,7 @@ class WorkerStepProgram<Input, Output, ReturnValue> implements StepProgram<
|
|
|
101
101
|
step<Name extends string, Value>(
|
|
102
102
|
name: Name,
|
|
103
103
|
resolver: StepResolver<Output, Value> | StepProgramResolver<Output, Value>,
|
|
104
|
-
options: StepOptions<Output>,
|
|
104
|
+
options: StepOptions<Output, Value>,
|
|
105
105
|
): StepProgram<Input, Output & Record<Name, Value | null>, ReturnValue>;
|
|
106
106
|
step<Name extends string, Value>(
|
|
107
107
|
name: Name,
|
|
@@ -109,7 +109,7 @@ class WorkerStepProgram<Input, Output, ReturnValue> implements StepProgram<
|
|
|
109
109
|
| StepResolver<Output, Value>
|
|
110
110
|
| ConditionalStepResolver<Output, Value>
|
|
111
111
|
| StepProgramResolver<Output, Value>,
|
|
112
|
-
options?: StepOptions<Output>,
|
|
112
|
+
options?: StepOptions<Output, Value>,
|
|
113
113
|
): StepProgram<Input, Output & Record<Name, Value | null>, ReturnValue> {
|
|
114
114
|
if (!name.trim()) {
|
|
115
115
|
throw new Error('Step name required.');
|
|
@@ -129,7 +129,10 @@ class WorkerStepProgram<Input, Output, ReturnValue> implements StepProgram<
|
|
|
129
129
|
name,
|
|
130
130
|
resolver: stepResolver as PlayStepProgramStep['resolver'],
|
|
131
131
|
...(options?.staleAfterSeconds !== undefined
|
|
132
|
-
? {
|
|
132
|
+
? {
|
|
133
|
+
staleAfterSeconds:
|
|
134
|
+
options.staleAfterSeconds as PlayStepProgramStep['staleAfterSeconds'],
|
|
135
|
+
}
|
|
133
136
|
: {}),
|
|
134
137
|
},
|
|
135
138
|
],
|
|
@@ -1,17 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve the next expiry window from a completed cell value.
|
|
3
|
+
*
|
|
4
|
+
* Return a positive whole number of seconds to set the next `staleAt`, or
|
|
5
|
+
* `null` when this particular value should not expire.
|
|
6
|
+
*/
|
|
7
|
+
export type StaleAfterSecondsResolver<Value = unknown> = (
|
|
8
|
+
value: Value,
|
|
9
|
+
) => number | null;
|
|
10
|
+
|
|
11
|
+
/** Authored freshness policy: fixed TTL seconds or a value-based resolver. */
|
|
12
|
+
export type AuthoredStaleAfterSeconds<Value = unknown> =
|
|
13
|
+
| number
|
|
14
|
+
| StaleAfterSecondsResolver<Value>;
|
|
15
|
+
|
|
16
|
+
/** Freshness policy as written by play authors before runtime normalization. */
|
|
17
|
+
export type AuthoredCellStalenessPolicy<Value = unknown> = {
|
|
18
|
+
staleAfterSeconds?: AuthoredStaleAfterSeconds<Value>;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
/** Runtime-normalized freshness policy stored in execution plans. */
|
|
1
22
|
export type CellStalenessPolicy = {
|
|
2
23
|
staleAfterSeconds?: number;
|
|
24
|
+
dynamicStaleAfterSeconds?: boolean;
|
|
3
25
|
};
|
|
4
26
|
|
|
27
|
+
/** Stored per-cell freshness and completion metadata. */
|
|
5
28
|
export type CellStalenessMeta = {
|
|
6
29
|
status?: string | null;
|
|
7
30
|
completedAt?: number | null;
|
|
31
|
+
staleAt?: number | null;
|
|
32
|
+
staleAfterSeconds?: number | null;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Previous durable cell value passed to object-column resolvers.
|
|
37
|
+
*
|
|
38
|
+
* The runtime supplies this when a row+column is being recomputed after a
|
|
39
|
+
* previous value existed. `value` has the same type that the column returns;
|
|
40
|
+
* freshness metadata lives beside it.
|
|
41
|
+
*
|
|
42
|
+
* @sdkReference runtime 120
|
|
43
|
+
*/
|
|
44
|
+
export type PreviousCell<Value = unknown> = {
|
|
45
|
+
/** Previous completed value for this row+column. */
|
|
46
|
+
value: Value;
|
|
47
|
+
/** Millisecond timestamp when the previous value completed. */
|
|
48
|
+
completedAt?: number;
|
|
49
|
+
/** Millisecond timestamp when the previous value becomes stale; `null` means no expiry. */
|
|
50
|
+
staleAt?: number | null;
|
|
51
|
+
/** Resolved numeric TTL in seconds for the previous value, when present. */
|
|
52
|
+
staleAfterSeconds?: number;
|
|
8
53
|
};
|
|
9
54
|
|
|
10
55
|
export type CellStalenessDecision =
|
|
11
56
|
| { action: 'recompute'; reason: 'missing' | 'failed' | 'stale' }
|
|
12
|
-
| { action: 'reuse'; reason: 'fresh' | 'no_policy' };
|
|
57
|
+
| { action: 'reuse'; reason: 'fresh' | 'no_policy' | 'no_expiry' };
|
|
13
58
|
|
|
14
59
|
export type CellStalenessPolicyByField = Record<string, CellStalenessPolicy>;
|
|
60
|
+
export type AuthoredCellStalenessPolicyByField = Record<
|
|
61
|
+
string,
|
|
62
|
+
AuthoredCellStalenessPolicy
|
|
63
|
+
>;
|
|
15
64
|
|
|
16
65
|
export const DEEPLINE_CELL_META_FIELD = '__deeplineCellMeta';
|
|
17
66
|
|
|
@@ -32,12 +81,83 @@ export function validateStaleAfterSeconds(
|
|
|
32
81
|
}
|
|
33
82
|
|
|
34
83
|
export function normalizeCellStalenessPolicy(
|
|
35
|
-
policy:
|
|
84
|
+
policy: AuthoredCellStalenessPolicy | undefined,
|
|
36
85
|
): CellStalenessPolicy {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
86
|
+
const staleAfterSeconds = policy?.staleAfterSeconds;
|
|
87
|
+
if (staleAfterSeconds === undefined) {
|
|
88
|
+
return {};
|
|
89
|
+
}
|
|
90
|
+
if (typeof staleAfterSeconds === 'function') {
|
|
91
|
+
return { dynamicStaleAfterSeconds: true };
|
|
92
|
+
}
|
|
93
|
+
validateStaleAfterSeconds(staleAfterSeconds);
|
|
94
|
+
return { staleAfterSeconds };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function resolveCompletedCellStalenessMeta<Value>(input: {
|
|
98
|
+
policy?: AuthoredCellStalenessPolicy<Value>;
|
|
99
|
+
value: Value;
|
|
100
|
+
completedAt: number;
|
|
101
|
+
}): Pick<CellStalenessMeta, 'staleAfterSeconds' | 'staleAt'> {
|
|
102
|
+
const staleAfterSeconds = input.policy?.staleAfterSeconds;
|
|
103
|
+
if (staleAfterSeconds === undefined) {
|
|
104
|
+
return {};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const resolved =
|
|
108
|
+
typeof staleAfterSeconds === 'function'
|
|
109
|
+
? staleAfterSeconds(input.value)
|
|
110
|
+
: staleAfterSeconds;
|
|
111
|
+
if (resolved === null) {
|
|
112
|
+
return { staleAt: null };
|
|
113
|
+
}
|
|
114
|
+
if (typeof resolved !== 'number') {
|
|
115
|
+
throw new Error(
|
|
116
|
+
'staleAfterSeconds(value) must return a positive whole number of seconds or null.',
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
validateStaleAfterSeconds(resolved, 'staleAfterSeconds(value)');
|
|
120
|
+
return {
|
|
121
|
+
staleAfterSeconds: resolved,
|
|
122
|
+
staleAt: input.completedAt + resolved * 1000,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export function previousCellFromValue<Value>(input: {
|
|
127
|
+
hasValue: boolean;
|
|
128
|
+
value: Value;
|
|
129
|
+
meta?: CellStalenessMeta | null;
|
|
130
|
+
}): PreviousCell<Value> | undefined {
|
|
131
|
+
if (!input.hasValue) {
|
|
132
|
+
return undefined;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const previous: PreviousCell<Value> = {
|
|
136
|
+
value: input.value,
|
|
137
|
+
};
|
|
138
|
+
if (
|
|
139
|
+
typeof input.meta?.completedAt === 'number' &&
|
|
140
|
+
Number.isFinite(input.meta.completedAt)
|
|
141
|
+
) {
|
|
142
|
+
previous.completedAt = input.meta.completedAt;
|
|
143
|
+
}
|
|
144
|
+
if (
|
|
145
|
+
input.meta &&
|
|
146
|
+
Object.prototype.hasOwnProperty.call(input.meta, 'staleAt')
|
|
147
|
+
) {
|
|
148
|
+
previous.staleAt =
|
|
149
|
+
typeof input.meta.staleAt === 'number' &&
|
|
150
|
+
Number.isFinite(input.meta.staleAt)
|
|
151
|
+
? input.meta.staleAt
|
|
152
|
+
: null;
|
|
153
|
+
}
|
|
154
|
+
if (
|
|
155
|
+
typeof input.meta?.staleAfterSeconds === 'number' &&
|
|
156
|
+
Number.isFinite(input.meta.staleAfterSeconds)
|
|
157
|
+
) {
|
|
158
|
+
previous.staleAfterSeconds = input.meta.staleAfterSeconds;
|
|
159
|
+
}
|
|
160
|
+
return previous;
|
|
41
161
|
}
|
|
42
162
|
|
|
43
163
|
export function shouldRecomputeCell(input: {
|
|
@@ -55,6 +175,23 @@ export function shouldRecomputeCell(input: {
|
|
|
55
175
|
return { action: 'recompute', reason: 'failed' };
|
|
56
176
|
}
|
|
57
177
|
|
|
178
|
+
const staleAt =
|
|
179
|
+
input.meta && Object.prototype.hasOwnProperty.call(input.meta, 'staleAt')
|
|
180
|
+
? input.meta.staleAt
|
|
181
|
+
: undefined;
|
|
182
|
+
if (staleAt === null) {
|
|
183
|
+
return { action: 'reuse', reason: 'no_expiry' };
|
|
184
|
+
}
|
|
185
|
+
if (typeof staleAt === 'number' && Number.isFinite(staleAt)) {
|
|
186
|
+
return (input.nowMs ?? Date.now()) > staleAt
|
|
187
|
+
? { action: 'recompute', reason: 'stale' }
|
|
188
|
+
: { action: 'reuse', reason: 'fresh' };
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
if (input.policy?.dynamicStaleAfterSeconds) {
|
|
192
|
+
return { action: 'recompute', reason: 'stale' };
|
|
193
|
+
}
|
|
194
|
+
|
|
58
195
|
const staleAfterSeconds = input.policy?.staleAfterSeconds;
|
|
59
196
|
validateStaleAfterSeconds(staleAfterSeconds);
|
|
60
197
|
if (staleAfterSeconds === undefined) {
|
|
@@ -76,6 +213,37 @@ export function shouldRecomputeCell(input: {
|
|
|
76
213
|
: { action: 'reuse', reason: 'fresh' };
|
|
77
214
|
}
|
|
78
215
|
|
|
216
|
+
export function resolveReusableCellMetaForCurrentPolicy<Value>(input: {
|
|
217
|
+
hasValue: boolean;
|
|
218
|
+
value: Value;
|
|
219
|
+
meta?: CellStalenessMeta | null;
|
|
220
|
+
policy?: AuthoredCellStalenessPolicy<Value>;
|
|
221
|
+
}): Pick<CellStalenessMeta, 'staleAfterSeconds' | 'staleAt'> {
|
|
222
|
+
if (
|
|
223
|
+
!input.hasValue ||
|
|
224
|
+
typeof input.policy?.staleAfterSeconds !== 'function'
|
|
225
|
+
) {
|
|
226
|
+
return {};
|
|
227
|
+
}
|
|
228
|
+
const status = String(input.meta?.status ?? '').trim();
|
|
229
|
+
if (status === 'failed') {
|
|
230
|
+
return {};
|
|
231
|
+
}
|
|
232
|
+
const completedAt =
|
|
233
|
+
typeof input.meta?.completedAt === 'number' &&
|
|
234
|
+
Number.isFinite(input.meta.completedAt)
|
|
235
|
+
? input.meta.completedAt
|
|
236
|
+
: null;
|
|
237
|
+
if (completedAt === null) {
|
|
238
|
+
return {};
|
|
239
|
+
}
|
|
240
|
+
return resolveCompletedCellStalenessMeta({
|
|
241
|
+
policy: input.policy,
|
|
242
|
+
value: input.value,
|
|
243
|
+
completedAt,
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
|
|
79
247
|
export function cellPolicyFields(
|
|
80
248
|
policies: CellStalenessPolicyByField | undefined,
|
|
81
249
|
): string[] {
|
|
@@ -83,6 +251,10 @@ export function cellPolicyFields(
|
|
|
83
251
|
return [];
|
|
84
252
|
}
|
|
85
253
|
return Object.entries(policies)
|
|
86
|
-
.filter(
|
|
254
|
+
.filter(
|
|
255
|
+
([, policy]) =>
|
|
256
|
+
policy.staleAfterSeconds !== undefined ||
|
|
257
|
+
policy.dynamicStaleAfterSeconds === true,
|
|
258
|
+
)
|
|
87
259
|
.map(([field]) => field);
|
|
88
260
|
}
|
|
@@ -24,6 +24,16 @@ export const COORDINATOR_RUN_SCOPE_HEADER = 'x-deepline-run-scope';
|
|
|
24
24
|
export const COORDINATOR_URL_OVERRIDE_HEADER = 'x-deepline-coordinator-url';
|
|
25
25
|
export const WORKER_CALLBACK_URL_OVERRIDE_HEADER =
|
|
26
26
|
'x-deepline-worker-callback-url';
|
|
27
|
+
/**
|
|
28
|
+
* CLI→app marker (NOT a coordinator header — it lives here only because both
|
|
29
|
+
* the SDK HTTP client and the run route already import this module). Set by
|
|
30
|
+
* automated test harnesses (e.g. `tests/v2-plays`) so their intentionally
|
|
31
|
+
* failing plays — depth-guard probes, error-path scenarios — do not page the
|
|
32
|
+
* SDK CLI error channel as if a real customer run had failed. The run route
|
|
33
|
+
* honors it ONLY in non-prod (see run/route.ts); a forged header from a real
|
|
34
|
+
* prod customer can never suppress their own failure alerts.
|
|
35
|
+
*/
|
|
36
|
+
export const SYNTHETIC_RUN_HEADER = 'x-deepline-synthetic-run';
|
|
27
37
|
|
|
28
38
|
let warnedAboutMissingInternalToken = false;
|
|
29
39
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
2
|
+
flattenStaticPipeline,
|
|
3
3
|
type PlayStaticPipeline,
|
|
4
4
|
type PlayStaticSubstep,
|
|
5
5
|
} from '../plays/static-pipeline';
|
|
@@ -189,7 +189,7 @@ function extractPlanMaps(
|
|
|
189
189
|
pipeline: PlayStaticPipeline | null,
|
|
190
190
|
): ExecutionPlanMap[] {
|
|
191
191
|
if (!pipeline) return [];
|
|
192
|
-
const substeps =
|
|
192
|
+
const substeps = flattenStaticPipeline(pipeline);
|
|
193
193
|
const fallbackWaterfalls = substeps.filter(
|
|
194
194
|
(substep): substep is Extract<PlayStaticSubstep, { type: 'waterfall' }> =>
|
|
195
195
|
substep.type === 'waterfall',
|
|
@@ -252,7 +252,7 @@ function extractToolDeclarations(
|
|
|
252
252
|
if (!pipeline) return [];
|
|
253
253
|
const seen = new Set<string>();
|
|
254
254
|
const declarations: ExecutionPlan['toolDeclarations'] = [];
|
|
255
|
-
for (const substep of
|
|
255
|
+
for (const substep of flattenStaticPipeline(pipeline)) {
|
|
256
256
|
if (substep.type === 'tool') {
|
|
257
257
|
const key = `${substep.toolId}:${substep.field}`;
|
|
258
258
|
if (!seen.has(key)) {
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { EmailStatus, EmailStatusValue } from './email-status';
|
|
2
|
+
|
|
3
|
+
export const JOB_CHANGE_STATUS_VALUES = [
|
|
4
|
+
'moved',
|
|
5
|
+
'no_change',
|
|
6
|
+
'left_company',
|
|
7
|
+
'unknown',
|
|
8
|
+
'profile_unavailable',
|
|
9
|
+
'no_new_company',
|
|
10
|
+
] as const;
|
|
11
|
+
|
|
12
|
+
export type JobChangeStatus = (typeof JOB_CHANGE_STATUS_VALUES)[number];
|
|
13
|
+
|
|
14
|
+
export type JobChangeGetterValue = {
|
|
15
|
+
status: JobChangeStatus;
|
|
16
|
+
date: string | null;
|
|
17
|
+
new_company: string | null;
|
|
18
|
+
new_title: string | null;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export const PHONE_STATUS_VALUES = ['valid', 'invalid', 'unknown'] as const;
|
|
22
|
+
|
|
23
|
+
export type PhoneStatus = (typeof PHONE_STATUS_VALUES)[number];
|
|
24
|
+
|
|
25
|
+
export const DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS = {
|
|
26
|
+
id: { identity: true, valueKind: 'string' },
|
|
27
|
+
name: { identity: true, valueKind: 'string' },
|
|
28
|
+
email: { identity: true, valueKind: 'string' },
|
|
29
|
+
personal_email: { identity: true, valueKind: 'string' },
|
|
30
|
+
phone: { identity: true, valueKind: 'string' },
|
|
31
|
+
linkedin: { identity: true, valueKind: 'string' },
|
|
32
|
+
linkedin_url: { identity: true, valueKind: 'string' },
|
|
33
|
+
domain: { identity: true, valueKind: 'string' },
|
|
34
|
+
website: { identity: true, valueKind: 'string' },
|
|
35
|
+
first_name: { identity: true, valueKind: 'string' },
|
|
36
|
+
last_name: { identity: true, valueKind: 'string' },
|
|
37
|
+
full_name: { identity: true, valueKind: 'string' },
|
|
38
|
+
company: { identity: true, valueKind: 'string' },
|
|
39
|
+
company_name: { identity: true, valueKind: 'string' },
|
|
40
|
+
organization_name: { identity: true, valueKind: 'string' },
|
|
41
|
+
company_domain: { identity: true, valueKind: 'string' },
|
|
42
|
+
company_website: { identity: true, valueKind: 'string' },
|
|
43
|
+
company_linkedin_url: { identity: true, valueKind: 'string' },
|
|
44
|
+
title: { identity: false, valueKind: 'string' },
|
|
45
|
+
industry: { identity: false, valueKind: 'string' },
|
|
46
|
+
status: { identity: false, valueKind: 'string' },
|
|
47
|
+
job_change: { identity: false, valueKind: 'job_change' },
|
|
48
|
+
job_change_status: {
|
|
49
|
+
identity: false,
|
|
50
|
+
valueKind: 'job_change_status',
|
|
51
|
+
enum: JOB_CHANGE_STATUS_VALUES,
|
|
52
|
+
},
|
|
53
|
+
email_status: { identity: false, valueKind: 'email_status' },
|
|
54
|
+
phone_status: {
|
|
55
|
+
identity: false,
|
|
56
|
+
valueKind: 'phone_status',
|
|
57
|
+
enum: PHONE_STATUS_VALUES,
|
|
58
|
+
},
|
|
59
|
+
} as const;
|
|
60
|
+
|
|
61
|
+
export type DeeplineExtractorTarget =
|
|
62
|
+
keyof typeof DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS;
|
|
63
|
+
|
|
64
|
+
export const DEEPLINE_EXTRACTOR_TARGETS = Object.keys(
|
|
65
|
+
DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS,
|
|
66
|
+
) as DeeplineExtractorTarget[];
|
|
67
|
+
|
|
68
|
+
export type DeeplineEmailStatusGetterValue = EmailStatus | EmailStatusValue;
|
|
69
|
+
|
|
70
|
+
export type DeeplineGetterValueMap = {
|
|
71
|
+
id: string;
|
|
72
|
+
name: string;
|
|
73
|
+
email: string;
|
|
74
|
+
personal_email: string;
|
|
75
|
+
phone: string;
|
|
76
|
+
linkedin: string;
|
|
77
|
+
linkedin_url: string;
|
|
78
|
+
domain: string;
|
|
79
|
+
website: string;
|
|
80
|
+
first_name: string;
|
|
81
|
+
last_name: string;
|
|
82
|
+
full_name: string;
|
|
83
|
+
company: string;
|
|
84
|
+
company_name: string;
|
|
85
|
+
organization_name: string;
|
|
86
|
+
company_domain: string;
|
|
87
|
+
company_website: string;
|
|
88
|
+
company_linkedin_url: string;
|
|
89
|
+
title: string;
|
|
90
|
+
industry: string;
|
|
91
|
+
status: string;
|
|
92
|
+
job_change: JobChangeGetterValue;
|
|
93
|
+
job_change_status: JobChangeStatus;
|
|
94
|
+
email_status: DeeplineEmailStatusGetterValue;
|
|
95
|
+
phone_status: PhoneStatus;
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
export type DeeplineGetterValue<
|
|
99
|
+
TTarget extends DeeplineExtractorTarget = DeeplineExtractorTarget,
|
|
100
|
+
> = DeeplineGetterValueMap[TTarget];
|
|
101
|
+
|
|
102
|
+
export function isDeeplineExtractorTarget(
|
|
103
|
+
value: string,
|
|
104
|
+
): value is DeeplineExtractorTarget {
|
|
105
|
+
return value in DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS;
|
|
106
|
+
}
|
|
@@ -12,6 +12,9 @@ import {
|
|
|
12
12
|
|
|
13
13
|
export const PLAY_RUNTIME_PROVIDER_IDS = {
|
|
14
14
|
workersEdge: 'workers_edge',
|
|
15
|
+
postgresFast: 'postgres_fast',
|
|
16
|
+
postgresFastSandbox: 'postgres_fast_sandbox',
|
|
17
|
+
postgresFastWorkers: 'postgres_fast_workers',
|
|
15
18
|
local: 'local',
|
|
16
19
|
} as const;
|
|
17
20
|
|
|
@@ -39,6 +42,31 @@ export const PLAY_RUNTIME_PROVIDERS: Record<
|
|
|
39
42
|
artifactKind: PLAY_ARTIFACT_KINDS.esmWorkers,
|
|
40
43
|
label: 'Cloudflare Dynamic Workflows + Dynamic Workers + DO dedup',
|
|
41
44
|
},
|
|
45
|
+
postgres_fast: {
|
|
46
|
+
id: PLAY_RUNTIME_PROVIDER_IDS.postgresFast,
|
|
47
|
+
scheduler: PLAY_SCHEDULER_BACKENDS.postgres,
|
|
48
|
+
runner: PLAY_RUNTIME_BACKENDS.daytona,
|
|
49
|
+
dedup: PLAY_DEDUP_BACKENDS.durableObject,
|
|
50
|
+
artifactKind: PLAY_ARTIFACT_KINDS.cjsNode20,
|
|
51
|
+
label: 'Experimental Postgres Scheduler + warm sandbox runner + DO dedup',
|
|
52
|
+
},
|
|
53
|
+
postgres_fast_sandbox: {
|
|
54
|
+
id: PLAY_RUNTIME_PROVIDER_IDS.postgresFastSandbox,
|
|
55
|
+
scheduler: PLAY_SCHEDULER_BACKENDS.postgres,
|
|
56
|
+
runner: PLAY_RUNTIME_BACKENDS.daytona,
|
|
57
|
+
dedup: PLAY_DEDUP_BACKENDS.durableObject,
|
|
58
|
+
artifactKind: PLAY_ARTIFACT_KINDS.cjsNode20,
|
|
59
|
+
label: 'Experimental Postgres Scheduler + warm sandbox runner + DO dedup',
|
|
60
|
+
},
|
|
61
|
+
postgres_fast_workers: {
|
|
62
|
+
id: PLAY_RUNTIME_PROVIDER_IDS.postgresFastWorkers,
|
|
63
|
+
scheduler: PLAY_SCHEDULER_BACKENDS.postgres,
|
|
64
|
+
runner: PLAY_RUNTIME_BACKENDS.cloudflareWorkers,
|
|
65
|
+
dedup: PLAY_DEDUP_BACKENDS.durableObject,
|
|
66
|
+
artifactKind: PLAY_ARTIFACT_KINDS.esmWorkers,
|
|
67
|
+
label:
|
|
68
|
+
'Experimental Postgres Scheduler + Queue/DO-woken Workers + DO dedup',
|
|
69
|
+
},
|
|
42
70
|
local: {
|
|
43
71
|
id: PLAY_RUNTIME_PROVIDER_IDS.local,
|
|
44
72
|
scheduler: PLAY_SCHEDULER_BACKENDS.temporal,
|
|
@@ -22,6 +22,7 @@ import type { PreloadedRuntimeDbSession } from './db-session';
|
|
|
22
22
|
export const PLAY_SCHEDULER_BACKENDS = {
|
|
23
23
|
temporal: 'temporal',
|
|
24
24
|
cfWorkflows: 'cf-workflows',
|
|
25
|
+
postgres: 'postgres',
|
|
25
26
|
inProcess: 'in-process',
|
|
26
27
|
} as const;
|
|
27
28
|
|
|
@@ -65,6 +66,7 @@ export type PlaySchedulerSubmitInput = {
|
|
|
65
66
|
runId: string;
|
|
66
67
|
playId: string;
|
|
67
68
|
playName: string;
|
|
69
|
+
workflowFamilyKey?: string | null;
|
|
68
70
|
artifactStorageKey: string;
|
|
69
71
|
/**
|
|
70
72
|
* Optional inline artifact for schedulers that run the legacy Temporal
|
|
@@ -119,6 +121,8 @@ export type PlaySchedulerSubmitInput = {
|
|
|
119
121
|
orgId: string;
|
|
120
122
|
userEmail: string;
|
|
121
123
|
userId?: string | null;
|
|
124
|
+
source?: 'published' | 'ad_hoc' | 'draft';
|
|
125
|
+
executionProfile?: string | null;
|
|
122
126
|
/** runner backend to use for executing attempts */
|
|
123
127
|
runtimeBackend: string;
|
|
124
128
|
/** dedup backend for cross-attempt cross-process idempotency */
|
|
@@ -132,7 +136,16 @@ export type PlaySchedulerSubmitInput = {
|
|
|
132
136
|
};
|
|
133
137
|
|
|
134
138
|
export type PlaySchedulerProgressEvent =
|
|
135
|
-
| {
|
|
139
|
+
| {
|
|
140
|
+
type: 'status';
|
|
141
|
+
status: string;
|
|
142
|
+
logs?: string[];
|
|
143
|
+
ts: number;
|
|
144
|
+
activeNodeId?: string | null;
|
|
145
|
+
activeArtifactTableNamespace?: string | null;
|
|
146
|
+
updatedAt?: number | null;
|
|
147
|
+
liveNodeProgress?: unknown;
|
|
148
|
+
}
|
|
136
149
|
| { type: 'log'; line: string; ts: number }
|
|
137
150
|
| { type: 'row'; update: PlayRowUpdate; ts: number }
|
|
138
151
|
| { type: 'execution_event'; event: PlayExecutionEvent; ts: number }
|
|
@@ -200,6 +213,13 @@ export function normalizePlaySchedulerBackend(
|
|
|
200
213
|
if (normalized === 'cf-workflows' || normalized === 'cf_workflows') {
|
|
201
214
|
return PLAY_SCHEDULER_BACKENDS.cfWorkflows;
|
|
202
215
|
}
|
|
216
|
+
if (
|
|
217
|
+
normalized === 'postgres' ||
|
|
218
|
+
normalized === 'postgres-scheduler' ||
|
|
219
|
+
normalized === 'postgres_scheduler'
|
|
220
|
+
) {
|
|
221
|
+
return PLAY_SCHEDULER_BACKENDS.postgres;
|
|
222
|
+
}
|
|
203
223
|
if (normalized === 'in-process' || normalized === 'in_process') {
|
|
204
224
|
return PLAY_SCHEDULER_BACKENDS.inProcess;
|
|
205
225
|
}
|
|
@@ -1,15 +1,55 @@
|
|
|
1
|
+
import type { AuthoredStaleAfterSeconds, PreviousCell } from './cell-staleness';
|
|
2
|
+
|
|
1
3
|
export type StepProgramDatasetOptions = {
|
|
2
4
|
runIf?: (
|
|
3
5
|
row: Record<string, unknown>,
|
|
4
6
|
index: number,
|
|
5
7
|
) => boolean | Promise<boolean>;
|
|
6
|
-
staleAfterSeconds?:
|
|
8
|
+
staleAfterSeconds?: AuthoredStaleAfterSeconds;
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
export type StepProgramDatasetColumnRunInput<Value = unknown> = {
|
|
12
|
+
row: Record<string, unknown>;
|
|
13
|
+
ctx: unknown;
|
|
14
|
+
index: number;
|
|
15
|
+
previousCell?: PreviousCell<Value>;
|
|
7
16
|
};
|
|
8
17
|
|
|
18
|
+
export type StepProgramDatasetColumnDefinition<Value = unknown> =
|
|
19
|
+
StepProgramDatasetOptions & {
|
|
20
|
+
run: (
|
|
21
|
+
input: StepProgramDatasetColumnRunInput<Value>,
|
|
22
|
+
) => Value | Promise<Value>;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
export type StepProgramDatasetColumnInput<TResolver> =
|
|
26
|
+
| TResolver
|
|
27
|
+
| StepProgramDatasetColumnDefinition;
|
|
28
|
+
|
|
29
|
+
function isStepProgramDatasetColumnDefinition(
|
|
30
|
+
value: unknown,
|
|
31
|
+
): value is StepProgramDatasetColumnDefinition {
|
|
32
|
+
return (
|
|
33
|
+
value !== null &&
|
|
34
|
+
typeof value === 'object' &&
|
|
35
|
+
typeof (value as { run?: unknown }).run === 'function'
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
40
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function isPreviousCell(value: unknown): value is PreviousCell {
|
|
44
|
+
return (
|
|
45
|
+
isRecord(value) && Object.prototype.hasOwnProperty.call(value, 'value')
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
|
|
9
49
|
export type StepProgramDatasetStep<TResolver> = {
|
|
10
50
|
name: string;
|
|
11
51
|
resolver: TResolver;
|
|
12
|
-
staleAfterSeconds?:
|
|
52
|
+
staleAfterSeconds?: AuthoredStaleAfterSeconds;
|
|
13
53
|
};
|
|
14
54
|
|
|
15
55
|
export type StepProgramDatasetProgram<TStep> = {
|
|
@@ -75,19 +115,23 @@ export class StepProgramDatasetBuilder<
|
|
|
75
115
|
|
|
76
116
|
withColumn(
|
|
77
117
|
name: string,
|
|
78
|
-
|
|
118
|
+
columnInput: StepProgramDatasetColumnInput<TResolver>,
|
|
79
119
|
options?: StepProgramDatasetOptions,
|
|
80
120
|
): this {
|
|
81
121
|
if (!name.trim()) {
|
|
82
122
|
throw new Error(this.messages.emptyColumnName);
|
|
83
123
|
}
|
|
124
|
+
const normalized = this.normalizeColumnInput(columnInput, options);
|
|
84
125
|
this.program.steps = [
|
|
85
126
|
...this.program.steps,
|
|
86
127
|
{
|
|
87
128
|
name,
|
|
88
|
-
resolver: this.applyDerivationOptions(
|
|
89
|
-
|
|
90
|
-
|
|
129
|
+
resolver: this.applyDerivationOptions(
|
|
130
|
+
normalized.resolver,
|
|
131
|
+
normalized.options,
|
|
132
|
+
),
|
|
133
|
+
...(normalized.options?.staleAfterSeconds !== undefined
|
|
134
|
+
? { staleAfterSeconds: normalized.options.staleAfterSeconds }
|
|
91
135
|
: {}),
|
|
92
136
|
} as TStep,
|
|
93
137
|
];
|
|
@@ -127,4 +171,50 @@ export class StepProgramDatasetBuilder<
|
|
|
127
171
|
elseValue: null,
|
|
128
172
|
} as TResolver;
|
|
129
173
|
}
|
|
174
|
+
|
|
175
|
+
private normalizeColumnInput(
|
|
176
|
+
columnInput: StepProgramDatasetColumnInput<TResolver>,
|
|
177
|
+
options?: StepProgramDatasetOptions,
|
|
178
|
+
): { resolver: TResolver; options?: StepProgramDatasetOptions } {
|
|
179
|
+
if (!isStepProgramDatasetColumnDefinition(columnInput)) {
|
|
180
|
+
return { resolver: columnInput, options };
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const { run, ...definitionOptions } = columnInput;
|
|
184
|
+
const resolver = ((
|
|
185
|
+
row: Record<string, unknown>,
|
|
186
|
+
ctx: unknown,
|
|
187
|
+
third?: unknown,
|
|
188
|
+
fourth?: unknown,
|
|
189
|
+
fifth?: unknown,
|
|
190
|
+
) => {
|
|
191
|
+
const index =
|
|
192
|
+
typeof third === 'number'
|
|
193
|
+
? third
|
|
194
|
+
: typeof fourth === 'number'
|
|
195
|
+
? fourth
|
|
196
|
+
: 0;
|
|
197
|
+
const rowForRun =
|
|
198
|
+
isRecord(third) && typeof fourth === 'number' ? third : row;
|
|
199
|
+
const previousCell = isPreviousCell(fifth)
|
|
200
|
+
? fifth
|
|
201
|
+
: isPreviousCell(fourth)
|
|
202
|
+
? fourth
|
|
203
|
+
: undefined;
|
|
204
|
+
return run({
|
|
205
|
+
row: rowForRun,
|
|
206
|
+
ctx,
|
|
207
|
+
index,
|
|
208
|
+
...(previousCell ? { previousCell } : {}),
|
|
209
|
+
});
|
|
210
|
+
}) as TResolver;
|
|
211
|
+
|
|
212
|
+
return {
|
|
213
|
+
resolver,
|
|
214
|
+
options: {
|
|
215
|
+
...definitionOptions,
|
|
216
|
+
...(options ?? {}),
|
|
217
|
+
},
|
|
218
|
+
};
|
|
219
|
+
}
|
|
130
220
|
}
|