deepline 0.1.109 → 0.1.111
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +2634 -1532
- package/dist/cli/index.mjs +2547 -1451
- package/dist/index.d.mts +21 -14
- package/dist/index.d.ts +21 -14
- package/dist/index.js +97 -23
- package/dist/index.mjs +97 -23
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +192 -121
- package/dist/repo/apps/play-runner-workers/src/entry.ts +254 -65
- package/dist/repo/apps/play-runner-workers/src/runtime/receipts.ts +18 -27
- package/dist/repo/apps/play-runner-workers/src/workflow-instance-create.ts +44 -0
- package/dist/repo/apps/play-runner-workers/src/workflow-retry.ts +7 -11
- package/dist/repo/sdk/src/client.ts +35 -12
- package/dist/repo/sdk/src/errors.ts +2 -2
- package/dist/repo/sdk/src/http.ts +87 -7
- package/dist/repo/sdk/src/play.ts +1 -1
- package/dist/repo/sdk/src/plays/bundle-play-file.ts +5 -1
- package/dist/repo/sdk/src/release.ts +13 -10
- package/dist/repo/sdk/src/tool-output.ts +2 -2
- package/dist/repo/sdk/src/types.ts +9 -6
- package/dist/repo/shared_libs/play-runtime/fullenrich-batching.ts +229 -0
- package/dist/repo/shared_libs/play-runtime/governor/policy.ts +1 -1
- package/dist/repo/shared_libs/play-runtime/play-runtime-batching-registry.ts +20 -0
- package/dist/repo/shared_libs/play-runtime/run-failure.ts +20 -12
- package/dist/repo/shared_libs/play-runtime/run-ledger.ts +147 -70
- package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts +6 -2
- package/dist/repo/shared_libs/play-runtime/secret-redaction.ts +15 -0
- package/dist/repo/shared_libs/play-runtime/work-receipts.ts +1 -0
- package/dist/repo/shared_libs/plays/bundling/index.ts +193 -21
- package/dist/repo/shared_libs/plays/static-pipeline.ts +1 -3
- package/dist/repo/shared_libs/security/outbound-url-policy.ts +238 -0
- package/dist/repo/shared_libs/security/safe-fetch.ts +118 -0
- package/dist/viewer/viewer.css +617 -0
- package/dist/viewer/viewer.js +1496 -0
- package/package.json +5 -1
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
import {
|
|
2
|
+
defineBatchStrategyMap,
|
|
3
|
+
type BatchOperationStrategy,
|
|
4
|
+
} from './batching-types';
|
|
5
|
+
|
|
6
|
+
const FULLENRICH_BATCH_ITEM_KEY = 'deepline_batch_item_key';
|
|
7
|
+
const FULLENRICH_BATCH_SIZE = 100;
|
|
8
|
+
|
|
9
|
+
type FullEnrichContactRow = Record<string, unknown> & {
|
|
10
|
+
custom?: Record<string, string>;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
type FullEnrichBulkPayload = {
|
|
14
|
+
name: string;
|
|
15
|
+
webhook_url?: string;
|
|
16
|
+
wait_for_completion?: boolean;
|
|
17
|
+
poll_interval_ms?: number;
|
|
18
|
+
max_wait_ms?: number;
|
|
19
|
+
data: FullEnrichContactRow[];
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
type FullEnrichBulkResult = Record<string, unknown> & {
|
|
23
|
+
data?: unknown[];
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
function sortRecord(value: Record<string, unknown>): Record<string, unknown> {
|
|
27
|
+
const entries = Object.entries(value)
|
|
28
|
+
.filter(([, entry]) => entry !== undefined)
|
|
29
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
30
|
+
.map(([key, entry]) => [key, stableValue(entry)]);
|
|
31
|
+
return Object.fromEntries(entries);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function stableValue(value: unknown): unknown {
|
|
35
|
+
if (Array.isArray(value)) {
|
|
36
|
+
return value.map((entry) => stableValue(entry));
|
|
37
|
+
}
|
|
38
|
+
if (value && typeof value === 'object') {
|
|
39
|
+
return sortRecord(value as Record<string, unknown>);
|
|
40
|
+
}
|
|
41
|
+
return value;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function stableStringify(value: unknown): string {
|
|
45
|
+
return JSON.stringify(stableValue(value));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function readPath(root: unknown, path: string): unknown {
|
|
49
|
+
const parts = path.split('.');
|
|
50
|
+
let current: unknown = root;
|
|
51
|
+
for (const part of parts) {
|
|
52
|
+
if (!current || typeof current !== 'object') {
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
current = (current as Record<string, unknown>)[part];
|
|
56
|
+
}
|
|
57
|
+
return current;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function readArrayAtPaths(root: unknown, paths: string[]): unknown[] | null {
|
|
61
|
+
for (const path of paths) {
|
|
62
|
+
const value = readPath(root, path);
|
|
63
|
+
if (Array.isArray(value)) {
|
|
64
|
+
return value;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function readRows(payload: FullEnrichBulkPayload): FullEnrichContactRow[] {
|
|
71
|
+
return Array.isArray(payload.data) ? payload.data : [];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function isOneRowPayload(payload: FullEnrichBulkPayload): boolean {
|
|
75
|
+
return readRows(payload).length === 1;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function controlKey(payload: FullEnrichBulkPayload): string {
|
|
79
|
+
return stableStringify({
|
|
80
|
+
webhook_url: payload.webhook_url ?? null,
|
|
81
|
+
wait_for_completion: payload.wait_for_completion ?? null,
|
|
82
|
+
poll_interval_ms: payload.poll_interval_ms ?? null,
|
|
83
|
+
max_wait_ms: payload.max_wait_ms ?? null,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function rowIdentity(row: FullEnrichContactRow): string {
|
|
88
|
+
return stableStringify({
|
|
89
|
+
first_name: row.first_name ?? null,
|
|
90
|
+
last_name: row.last_name ?? null,
|
|
91
|
+
domain: row.domain ?? null,
|
|
92
|
+
company_name: row.company_name ?? null,
|
|
93
|
+
linkedin_url: row.linkedin_url ?? null,
|
|
94
|
+
enrich_fields: row.enrich_fields ?? null,
|
|
95
|
+
custom: row.custom ?? null,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function shortStableHash(value: string): string {
|
|
100
|
+
let hash = 0x811c9dc5;
|
|
101
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
102
|
+
hash ^= value.charCodeAt(index);
|
|
103
|
+
hash = Math.imul(hash, 0x01000193) >>> 0;
|
|
104
|
+
}
|
|
105
|
+
return hash.toString(36);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function itemKey(payload: FullEnrichBulkPayload, index = 0): string {
|
|
109
|
+
return `dl_${index}_${shortStableHash(rowIdentity(readRows(payload)[0] ?? {}))}`;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function withBatchItemKey(
|
|
113
|
+
row: FullEnrichContactRow,
|
|
114
|
+
key: string,
|
|
115
|
+
): FullEnrichContactRow {
|
|
116
|
+
return {
|
|
117
|
+
...row,
|
|
118
|
+
custom: {
|
|
119
|
+
...(row.custom ?? {}),
|
|
120
|
+
[FULLENRICH_BATCH_ITEM_KEY]: key,
|
|
121
|
+
},
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function extractResultRows(fullResult: FullEnrichBulkResult | unknown) {
|
|
126
|
+
return readArrayAtPaths(fullResult, ['data']) ?? [];
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function readResultItemKey(row: unknown): string | null {
|
|
130
|
+
if (!row || typeof row !== 'object' || Array.isArray(row)) {
|
|
131
|
+
return null;
|
|
132
|
+
}
|
|
133
|
+
const custom = (row as Record<string, unknown>).custom;
|
|
134
|
+
if (!custom || typeof custom !== 'object' || Array.isArray(custom)) {
|
|
135
|
+
return null;
|
|
136
|
+
}
|
|
137
|
+
const key = (custom as Record<string, unknown>)[FULLENRICH_BATCH_ITEM_KEY];
|
|
138
|
+
return typeof key === 'string' && key.trim() ? key.trim() : null;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const fullenrichBulkSelfBatchStrategy: BatchOperationStrategy<
|
|
142
|
+
FullEnrichBulkPayload,
|
|
143
|
+
FullEnrichBulkPayload,
|
|
144
|
+
FullEnrichBulkResult,
|
|
145
|
+
{ data: FullEnrichBulkResult },
|
|
146
|
+
unknown
|
|
147
|
+
> = {
|
|
148
|
+
sourceOperation: 'fullenrich_bulk_enrich',
|
|
149
|
+
batchOperation: 'fullenrich_bulk_enrich',
|
|
150
|
+
kind: 'identifier_batch',
|
|
151
|
+
maxBatchSize: FULLENRICH_BATCH_SIZE,
|
|
152
|
+
canBatchWith(left, right) {
|
|
153
|
+
return (
|
|
154
|
+
isOneRowPayload(left) &&
|
|
155
|
+
isOneRowPayload(right) &&
|
|
156
|
+
controlKey(left) === controlKey(right)
|
|
157
|
+
);
|
|
158
|
+
},
|
|
159
|
+
toBucketKey(payload) {
|
|
160
|
+
if (!isOneRowPayload(payload)) {
|
|
161
|
+
return `fullenrich_bulk_enrich:passthrough:${stableStringify(payload)}`;
|
|
162
|
+
}
|
|
163
|
+
return `fullenrich_bulk_enrich:${controlKey(payload)}`;
|
|
164
|
+
},
|
|
165
|
+
toItemKey(payload) {
|
|
166
|
+
return itemKey(payload);
|
|
167
|
+
},
|
|
168
|
+
compile(payloads) {
|
|
169
|
+
const first = payloads[0];
|
|
170
|
+
const rows = payloads.flatMap((payload, index) =>
|
|
171
|
+
readRows(payload).map((row) =>
|
|
172
|
+
withBatchItemKey(row, itemKey(payload, index)),
|
|
173
|
+
),
|
|
174
|
+
);
|
|
175
|
+
return {
|
|
176
|
+
batchOperation: 'fullenrich_bulk_enrich',
|
|
177
|
+
batchPayload: {
|
|
178
|
+
name:
|
|
179
|
+
payloads.length === 1
|
|
180
|
+
? first?.name || 'deepline-fullenrich-batch'
|
|
181
|
+
: `deepline-fullenrich-batch-${payloads.length}`,
|
|
182
|
+
...(first?.webhook_url ? { webhook_url: first.webhook_url } : {}),
|
|
183
|
+
...(first?.wait_for_completion !== undefined
|
|
184
|
+
? { wait_for_completion: first.wait_for_completion }
|
|
185
|
+
: {}),
|
|
186
|
+
...(first?.poll_interval_ms !== undefined
|
|
187
|
+
? { poll_interval_ms: first.poll_interval_ms }
|
|
188
|
+
: {}),
|
|
189
|
+
...(first?.max_wait_ms !== undefined
|
|
190
|
+
? { max_wait_ms: first.max_wait_ms }
|
|
191
|
+
: {}),
|
|
192
|
+
data: rows,
|
|
193
|
+
},
|
|
194
|
+
items: payloads.map((payload, index) => ({
|
|
195
|
+
itemKey: itemKey(payload, index),
|
|
196
|
+
payload,
|
|
197
|
+
})),
|
|
198
|
+
};
|
|
199
|
+
},
|
|
200
|
+
splitResult(fullResult, compiled) {
|
|
201
|
+
const resultRows = extractResultRows(fullResult);
|
|
202
|
+
const rowsByItemKey = new Map<string, unknown>();
|
|
203
|
+
for (const row of resultRows) {
|
|
204
|
+
const key = readResultItemKey(row);
|
|
205
|
+
if (key) {
|
|
206
|
+
rowsByItemKey.set(key, row);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return compiled.items.map((item, index) => {
|
|
211
|
+
const matchedRow =
|
|
212
|
+
rowsByItemKey.get(item.itemKey) ??
|
|
213
|
+
(index < resultRows.length ? resultRows[index] : null);
|
|
214
|
+
const singleResult = {
|
|
215
|
+
...fullResult,
|
|
216
|
+
data: matchedRow ? [matchedRow] : [],
|
|
217
|
+
};
|
|
218
|
+
return {
|
|
219
|
+
itemKey: item.itemKey,
|
|
220
|
+
result: { data: singleResult },
|
|
221
|
+
rawResult: matchedRow,
|
|
222
|
+
};
|
|
223
|
+
});
|
|
224
|
+
},
|
|
225
|
+
};
|
|
226
|
+
|
|
227
|
+
export const fullenrichBatchStrategies = defineBatchStrategyMap({
|
|
228
|
+
fullenrich_bulk_enrich: fullenrichBulkSelfBatchStrategy,
|
|
229
|
+
});
|
|
@@ -116,7 +116,7 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
|
|
|
116
116
|
},
|
|
117
117
|
pacing: {
|
|
118
118
|
// Undeclared providers; declared providers (rate-limit-definitions.ts) win.
|
|
119
|
-
defaultProviderRequestsPerSecond:
|
|
119
|
+
defaultProviderRequestsPerSecond: 10,
|
|
120
120
|
suggestedMaxParallelism: 50,
|
|
121
121
|
},
|
|
122
122
|
};
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { AnyBatchOperationStrategy } from './batching-types';
|
|
2
|
+
import { DEFAULT_PLAY_RUNTIME_BATCH_STRATEGIES } from './default-batch-strategies';
|
|
3
|
+
import { fullenrichBatchStrategies } from './fullenrich-batching';
|
|
4
|
+
|
|
5
|
+
export const PLAY_RUNTIME_BATCH_OPERATION_REGISTRY: Record<
|
|
6
|
+
string,
|
|
7
|
+
AnyBatchOperationStrategy
|
|
8
|
+
> = {
|
|
9
|
+
...DEFAULT_PLAY_RUNTIME_BATCH_STRATEGIES,
|
|
10
|
+
...fullenrichBatchStrategies,
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export function getPlayRuntimeBatchStrategy(
|
|
14
|
+
operation: string | null | undefined,
|
|
15
|
+
): AnyBatchOperationStrategy | null {
|
|
16
|
+
if (!operation) {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
return PLAY_RUNTIME_BATCH_OPERATION_REGISTRY[operation] ?? null;
|
|
20
|
+
}
|
|
@@ -1,9 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
const CLOUDFLARE_DURABLE_OBJECT_RESET_RE =
|
|
2
|
+
/Durable Object.*(?:code was updated|storage caused object)/;
|
|
3
3
|
|
|
4
4
|
export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE =
|
|
5
5
|
'Run interrupted by a platform deploy and was not retried automatically. Re-run the same command; the input is unchanged.';
|
|
6
6
|
|
|
7
|
+
export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE =
|
|
8
|
+
'Internal play runtime storage failed. Please retry the run; if this keeps happening, contact Deepline support with the run ID.';
|
|
9
|
+
|
|
7
10
|
export type PlayRunFailureDetails = {
|
|
8
11
|
code: string;
|
|
9
12
|
phase: string;
|
|
@@ -19,23 +22,15 @@ function toErrorText(error: unknown): string {
|
|
|
19
22
|
return String(error);
|
|
20
23
|
}
|
|
21
24
|
|
|
22
|
-
export function isCloudflareDurableObjectCodeUpdatedError(
|
|
23
|
-
error: unknown,
|
|
24
|
-
): boolean {
|
|
25
|
-
return toErrorText(error).includes(
|
|
26
|
-
CLOUDFLARE_DURABLE_OBJECT_CODE_UPDATED_ERROR,
|
|
27
|
-
);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
25
|
export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
|
|
31
26
|
const cause = toErrorText(error);
|
|
32
|
-
if (
|
|
27
|
+
if (CLOUDFLARE_DURABLE_OBJECT_RESET_RE.test(cause)) {
|
|
33
28
|
return {
|
|
34
29
|
code: 'PLATFORM_DEPLOY_INTERRUPTED',
|
|
35
30
|
phase: 'runtime',
|
|
36
31
|
message: PLATFORM_DEPLOY_INTERRUPTED_MESSAGE,
|
|
37
32
|
retryable: true,
|
|
38
|
-
cause
|
|
33
|
+
cause,
|
|
39
34
|
};
|
|
40
35
|
}
|
|
41
36
|
const playDepthBudgetMatch = cause.match(
|
|
@@ -50,6 +45,19 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
|
|
|
50
45
|
cause,
|
|
51
46
|
};
|
|
52
47
|
}
|
|
48
|
+
const lowerCause = cause.toLowerCase();
|
|
49
|
+
if (
|
|
50
|
+
lowerCause.includes('neondberror') ||
|
|
51
|
+
lowerCause.includes('bind message supplies') ||
|
|
52
|
+
lowerCause.includes('prepared statement')
|
|
53
|
+
) {
|
|
54
|
+
return {
|
|
55
|
+
code: 'RUN_STORAGE_FAILED',
|
|
56
|
+
phase: 'storage',
|
|
57
|
+
message: INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE,
|
|
58
|
+
retryable: true,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
53
61
|
return {
|
|
54
62
|
code: 'RUN_FAILED',
|
|
55
63
|
phase: 'runtime',
|
|
@@ -549,22 +549,17 @@ function appendLogLines(
|
|
|
549
549
|
|
|
550
550
|
/**
|
|
551
551
|
* Terminal-status precedence. Re-delivery of the same terminal status is a
|
|
552
|
-
* benign no-op handled by the regular reduction.
|
|
553
|
-
*
|
|
554
|
-
*
|
|
555
|
-
*
|
|
556
|
-
*
|
|
557
|
-
*
|
|
558
|
-
* (compute billing finalize), so a billing business denial there — e.g. the
|
|
559
|
-
* per-run credit cap (maxCreditsPerRun) — arrives as a later run.failed
|
|
560
|
-
* that MUST fail the run. Blanket first-terminal-wins silently completed
|
|
561
|
-
* capped runs (regression pinned by
|
|
562
|
-
* tests/v2-plays/plays/44-compute-billing-cap.play.ts). A cancelled run
|
|
563
|
-
* stays cancelled, and a failed run can never be flipped to completed.
|
|
552
|
+
* benign no-op handled by the regular reduction. Conflicting terminal events
|
|
553
|
+
* use newest-terminal-wins by event time: older events are ignored and logged,
|
|
554
|
+
* newer events reconcile the snapshot. This is the idempotency model callers
|
|
555
|
+
* expect when duplicate/retried work completes out of order. It also preserves
|
|
556
|
+
* post-completion billing cap demotion because that denial arrives as a newer
|
|
557
|
+
* run.failed event.
|
|
564
558
|
*/
|
|
565
559
|
function conflictingTerminalSnapshot(
|
|
566
560
|
base: PlayRunLedgerSnapshot,
|
|
567
561
|
eventType: keyof typeof TERMINAL_STATUS_BY_EVENT_TYPE,
|
|
562
|
+
occurredAt: number,
|
|
568
563
|
): PlayRunLedgerSnapshot | null {
|
|
569
564
|
if (!isTerminalPlayRunLedgerStatus(base.status)) {
|
|
570
565
|
return null;
|
|
@@ -572,14 +567,16 @@ function conflictingTerminalSnapshot(
|
|
|
572
567
|
if (TERMINAL_STATUS_BY_EVENT_TYPE[eventType] === base.status) {
|
|
573
568
|
return null;
|
|
574
569
|
}
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
//
|
|
570
|
+
const terminalAt = base.finishedAt ?? base.updatedAt ?? 0;
|
|
571
|
+
if (occurredAt > terminalAt) {
|
|
572
|
+
// Newer terminal evidence reconciles the run. This covers replay/receipt
|
|
573
|
+
// races where an earlier attempt failed but a later attempt recovered and
|
|
574
|
+
// completed with the durable result.
|
|
578
575
|
return null;
|
|
579
576
|
}
|
|
580
577
|
return withTiming(
|
|
581
578
|
appendLogLines(base, [
|
|
582
|
-
`[ledger] conflicting terminal event ${eventType} ignored; status already ${base.status}`,
|
|
579
|
+
`[ledger] stale conflicting terminal event ${eventType} ignored; status already ${base.status}`,
|
|
583
580
|
]),
|
|
584
581
|
);
|
|
585
582
|
}
|
|
@@ -617,6 +614,23 @@ function settleRunningStepsOnTerminal(
|
|
|
617
614
|
return changed ? { ...snapshot, stepsById } : snapshot;
|
|
618
615
|
}
|
|
619
616
|
|
|
617
|
+
function terminalFinishedAt(
|
|
618
|
+
base: PlayRunLedgerSnapshot,
|
|
619
|
+
eventType: keyof typeof TERMINAL_STATUS_BY_EVENT_TYPE,
|
|
620
|
+
occurredAt: number,
|
|
621
|
+
): number {
|
|
622
|
+
const nextStatus = TERMINAL_STATUS_BY_EVENT_TYPE[eventType];
|
|
623
|
+
const terminalAt = base.finishedAt ?? base.updatedAt ?? 0;
|
|
624
|
+
if (
|
|
625
|
+
isTerminalPlayRunLedgerStatus(base.status) &&
|
|
626
|
+
nextStatus !== base.status &&
|
|
627
|
+
occurredAt > terminalAt
|
|
628
|
+
) {
|
|
629
|
+
return occurredAt;
|
|
630
|
+
}
|
|
631
|
+
return base.finishedAt ?? occurredAt;
|
|
632
|
+
}
|
|
633
|
+
|
|
620
634
|
export function reducePlayRunLedgerEvent(
|
|
621
635
|
snapshot: PlayRunLedgerSnapshot,
|
|
622
636
|
event: PlayRunLedgerEvent,
|
|
@@ -654,52 +668,46 @@ export function reducePlayRunLedgerEvent(
|
|
|
654
668
|
});
|
|
655
669
|
case 'run.completed':
|
|
656
670
|
return (
|
|
657
|
-
conflictingTerminalSnapshot(base, event.type) ??
|
|
658
|
-
withTiming(
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
},
|
|
670
|
-
)
|
|
671
|
+
conflictingTerminalSnapshot(base, event.type, occurredAt) ??
|
|
672
|
+
withTiming({
|
|
673
|
+
...settleRunningStepsOnTerminal(base, 'completed', occurredAt),
|
|
674
|
+
status: 'completed',
|
|
675
|
+
error: null,
|
|
676
|
+
startedAt: base.startedAt ?? occurredAt,
|
|
677
|
+
finishedAt: terminalFinishedAt(base, event.type, occurredAt),
|
|
678
|
+
activeStepId: null,
|
|
679
|
+
activeArtifactTableNamespace: null,
|
|
680
|
+
result: event.result ?? base.result,
|
|
681
|
+
resultSummary: event.resultSummary ?? base.resultSummary,
|
|
682
|
+
})
|
|
671
683
|
);
|
|
672
684
|
case 'run.failed':
|
|
673
685
|
return (
|
|
674
|
-
conflictingTerminalSnapshot(base, event.type) ??
|
|
675
|
-
withTiming(
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
},
|
|
686
|
-
)
|
|
686
|
+
conflictingTerminalSnapshot(base, event.type, occurredAt) ??
|
|
687
|
+
withTiming({
|
|
688
|
+
...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
|
|
689
|
+
status: 'failed',
|
|
690
|
+
error: event.error ?? base.error ?? null,
|
|
691
|
+
startedAt: base.startedAt ?? occurredAt,
|
|
692
|
+
finishedAt: terminalFinishedAt(base, event.type, occurredAt),
|
|
693
|
+
activeStepId: null,
|
|
694
|
+
activeArtifactTableNamespace: null,
|
|
695
|
+
result: event.result ?? base.result,
|
|
696
|
+
})
|
|
687
697
|
);
|
|
688
698
|
case 'run.cancelled':
|
|
689
699
|
return (
|
|
690
|
-
conflictingTerminalSnapshot(base, event.type) ??
|
|
691
|
-
withTiming(
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
},
|
|
702
|
-
)
|
|
700
|
+
conflictingTerminalSnapshot(base, event.type, occurredAt) ??
|
|
701
|
+
withTiming({
|
|
702
|
+
...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
|
|
703
|
+
status: 'cancelled',
|
|
704
|
+
error: event.error ?? base.error ?? null,
|
|
705
|
+
startedAt: base.startedAt ?? occurredAt,
|
|
706
|
+
finishedAt: terminalFinishedAt(base, event.type, occurredAt),
|
|
707
|
+
activeStepId: null,
|
|
708
|
+
activeArtifactTableNamespace: null,
|
|
709
|
+
result: event.result ?? base.result,
|
|
710
|
+
})
|
|
703
711
|
);
|
|
704
712
|
case 'log.appended':
|
|
705
713
|
return withTiming(
|
|
@@ -712,6 +720,11 @@ export function reducePlayRunLedgerEvent(
|
|
|
712
720
|
);
|
|
713
721
|
case 'step.started': {
|
|
714
722
|
const current = base.stepsById[event.stepId];
|
|
723
|
+
const shouldRefineSyntheticStart =
|
|
724
|
+
current?.startedAt != null &&
|
|
725
|
+
current.completedAt != null &&
|
|
726
|
+
current.startedAt === current.completedAt &&
|
|
727
|
+
occurredAt < current.startedAt;
|
|
715
728
|
const nextStep: PlayRunLedgerStepSnapshot = {
|
|
716
729
|
...(current ?? { stepId: event.stepId, status: 'running' as const }),
|
|
717
730
|
stepId: event.stepId,
|
|
@@ -725,8 +738,10 @@ export function reducePlayRunLedgerEvent(
|
|
|
725
738
|
event.artifactTableNamespace ??
|
|
726
739
|
current?.artifactTableNamespace ??
|
|
727
740
|
null,
|
|
728
|
-
startedAt:
|
|
729
|
-
|
|
741
|
+
startedAt: shouldRefineSyntheticStart
|
|
742
|
+
? occurredAt
|
|
743
|
+
: (current?.startedAt ?? occurredAt),
|
|
744
|
+
updatedAt: Math.max(current?.updatedAt ?? 0, occurredAt),
|
|
730
745
|
};
|
|
731
746
|
return withTiming({
|
|
732
747
|
...base,
|
|
@@ -767,6 +782,25 @@ export function reducePlayRunLedgerEvent(
|
|
|
767
782
|
: event.status === 'running' && inferredStatus === 'completed'
|
|
768
783
|
? 'completed'
|
|
769
784
|
: (event.status ?? current?.status ?? inferredStatus);
|
|
785
|
+
const completedAt =
|
|
786
|
+
current?.completedAt ??
|
|
787
|
+
event.progress.completedAt ??
|
|
788
|
+
(status === 'completed'
|
|
789
|
+
? (event.progress.updatedAt ?? occurredAt)
|
|
790
|
+
: null);
|
|
791
|
+
const shouldRefineSyntheticStart =
|
|
792
|
+
current?.startedAt != null &&
|
|
793
|
+
current.completedAt != null &&
|
|
794
|
+
current.startedAt === current.completedAt &&
|
|
795
|
+
event.progress.startedAt != null &&
|
|
796
|
+
event.progress.startedAt < current.startedAt;
|
|
797
|
+
const startedAt =
|
|
798
|
+
(shouldRefineSyntheticStart ? event.progress.startedAt : null) ??
|
|
799
|
+
current?.startedAt ??
|
|
800
|
+
event.progress.startedAt ??
|
|
801
|
+
(status === 'completed' || status === 'failed'
|
|
802
|
+
? (completedAt ?? event.progress.updatedAt ?? occurredAt)
|
|
803
|
+
: null);
|
|
770
804
|
const nextStep: PlayRunLedgerStepSnapshot = {
|
|
771
805
|
...(current ?? { stepId: event.stepId, status }),
|
|
772
806
|
stepId: event.stepId,
|
|
@@ -777,13 +811,8 @@ export function reducePlayRunLedgerEvent(
|
|
|
777
811
|
progress.artifactTableNamespace ??
|
|
778
812
|
current?.artifactTableNamespace ??
|
|
779
813
|
null,
|
|
780
|
-
startedAt
|
|
781
|
-
completedAt
|
|
782
|
-
current?.completedAt ??
|
|
783
|
-
event.progress.completedAt ??
|
|
784
|
-
(status === 'completed'
|
|
785
|
-
? (event.progress.updatedAt ?? occurredAt)
|
|
786
|
-
: null),
|
|
814
|
+
startedAt,
|
|
815
|
+
completedAt,
|
|
787
816
|
updatedAt: occurredAt,
|
|
788
817
|
progress,
|
|
789
818
|
};
|
|
@@ -818,6 +847,10 @@ export function reducePlayRunLedgerEvent(
|
|
|
818
847
|
: event.type === 'step.failed'
|
|
819
848
|
? 'failed'
|
|
820
849
|
: 'skipped';
|
|
850
|
+
const completedAt =
|
|
851
|
+
status === 'skipped' || preserveFailedStatus
|
|
852
|
+
? (current?.completedAt ?? null)
|
|
853
|
+
: occurredAt;
|
|
821
854
|
const nextStep: PlayRunLedgerStepSnapshot = {
|
|
822
855
|
...(current ?? { stepId: event.stepId, status }),
|
|
823
856
|
stepId: event.stepId,
|
|
@@ -828,11 +861,12 @@ export function reducePlayRunLedgerEvent(
|
|
|
828
861
|
event.artifactTableNamespace ??
|
|
829
862
|
current?.artifactTableNamespace ??
|
|
830
863
|
null,
|
|
831
|
-
startedAt:
|
|
832
|
-
|
|
833
|
-
status === '
|
|
834
|
-
? (
|
|
835
|
-
:
|
|
864
|
+
startedAt:
|
|
865
|
+
current?.startedAt ??
|
|
866
|
+
(status === 'completed' || status === 'failed'
|
|
867
|
+
? (completedAt ?? occurredAt)
|
|
868
|
+
: null),
|
|
869
|
+
completedAt,
|
|
836
870
|
updatedAt: occurredAt,
|
|
837
871
|
progress: current?.progress ?? null,
|
|
838
872
|
};
|
|
@@ -927,6 +961,20 @@ export function slicePositionalLogLines(input: {
|
|
|
927
961
|
return { lines: [...lines], channelOffset: sendFromOffset };
|
|
928
962
|
}
|
|
929
963
|
|
|
964
|
+
function terminalLogReplayChannelOffset(input: {
|
|
965
|
+
liveLogTotalCount?: number;
|
|
966
|
+
liveLogsLength: number;
|
|
967
|
+
}): number | null {
|
|
968
|
+
if (
|
|
969
|
+
typeof input.liveLogTotalCount !== 'number' ||
|
|
970
|
+
!Number.isFinite(input.liveLogTotalCount) ||
|
|
971
|
+
input.liveLogTotalCount < input.liveLogsLength
|
|
972
|
+
) {
|
|
973
|
+
return null;
|
|
974
|
+
}
|
|
975
|
+
return input.liveLogTotalCount - input.liveLogsLength;
|
|
976
|
+
}
|
|
977
|
+
|
|
930
978
|
/**
|
|
931
979
|
* Forward producer log lines as one `log.appended` event.
|
|
932
980
|
*
|
|
@@ -968,6 +1016,35 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
|
|
|
968
1016
|
];
|
|
969
1017
|
}
|
|
970
1018
|
|
|
1019
|
+
/**
|
|
1020
|
+
* Re-send a terminal retained log tail through terminal transport recovery.
|
|
1021
|
+
*
|
|
1022
|
+
* Live worker flushes use positional channel offsets. Terminal replay is a
|
|
1023
|
+
* different transport: it replays the runner's retained output after terminal
|
|
1024
|
+
* state is known. When the producer knows the retained tail's offset, Convex
|
|
1025
|
+
* uses that offset only to bound occurrence-count reconciliation to the same
|
|
1026
|
+
* tail window; it does not route the replay through the worker cursor.
|
|
1027
|
+
*/
|
|
1028
|
+
export function buildTerminalLogReplayEvents(input: {
|
|
1029
|
+
runId: string;
|
|
1030
|
+
lines: readonly string[];
|
|
1031
|
+
source?: PlayRunLedgerEventSource;
|
|
1032
|
+
occurredAt?: number;
|
|
1033
|
+
liveLogTotalCount?: number;
|
|
1034
|
+
}): PlayRunLedgerEvent[] {
|
|
1035
|
+
const channelOffset = terminalLogReplayChannelOffset({
|
|
1036
|
+
liveLogTotalCount: input.liveLogTotalCount,
|
|
1037
|
+
liveLogsLength: input.lines.length,
|
|
1038
|
+
});
|
|
1039
|
+
return buildPlayRunLedgerEventsFromLogLines({
|
|
1040
|
+
runId: input.runId,
|
|
1041
|
+
lines: input.lines,
|
|
1042
|
+
source: 'coordinator',
|
|
1043
|
+
occurredAt: input.occurredAt,
|
|
1044
|
+
channelOffset,
|
|
1045
|
+
});
|
|
1046
|
+
}
|
|
1047
|
+
|
|
971
1048
|
export function buildPlayRunLedgerEventsFromStatusPatch(input: {
|
|
972
1049
|
patch: PlayRunLedgerStatusPatch;
|
|
973
1050
|
previousSnapshot: PlayRunLedgerSnapshot;
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
* One of three pluggable axes (alongside runner-backends and dedup-backends).
|
|
5
5
|
* Selected per-run via PlayExecutionProfile.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
7
|
+
* Cloudflare Workflows is the default production scheduler through the
|
|
8
|
+
* workers_edge profile. Hatchet is selected explicitly by the hatchet profile.
|
|
9
9
|
*
|
|
10
10
|
* Customer plays are unaffected — this is purely the orchestration layer.
|
|
11
11
|
*/
|
|
@@ -18,6 +18,7 @@ import type {
|
|
|
18
18
|
import type { ExecutionPlan } from './execution-plan';
|
|
19
19
|
import type { PlayRuntimeManifestMap } from '../plays/compiler-manifest';
|
|
20
20
|
import type { PreloadedRuntimeDbSession } from './db-session';
|
|
21
|
+
import type { PlayRunnerRuntimeTiming } from './protocol';
|
|
21
22
|
|
|
22
23
|
export const PLAY_SCHEDULER_BACKENDS = {
|
|
23
24
|
temporal: 'temporal',
|
|
@@ -128,6 +129,8 @@ export type PlaySchedulerSubmitInput = {
|
|
|
128
129
|
userId?: string | null;
|
|
129
130
|
source?: 'published' | 'ad_hoc' | 'draft';
|
|
130
131
|
executionProfile?: string | null;
|
|
132
|
+
/** Durable per-workspace active run cap enforced when the run row is projected. */
|
|
133
|
+
activeRunLimit?: number | null;
|
|
131
134
|
/** runner backend to use for executing attempts */
|
|
132
135
|
runtimeBackend: string;
|
|
133
136
|
/** dedup backend for cross-attempt cross-process idempotency */
|
|
@@ -195,6 +198,7 @@ export type PlaySchedulerResultEnvelope = {
|
|
|
195
198
|
finalCheckpoint?: PlayCheckpoint;
|
|
196
199
|
totalRows?: number;
|
|
197
200
|
durationMs?: number;
|
|
201
|
+
runtimeTiming?: PlayRunnerRuntimeTiming;
|
|
198
202
|
};
|
|
199
203
|
|
|
200
204
|
export interface PlaySchedulerBackend {
|