deepline 0.1.109 → 0.1.111

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/cli/index.js +2634 -1532
  2. package/dist/cli/index.mjs +2547 -1451
  3. package/dist/index.d.mts +21 -14
  4. package/dist/index.d.ts +21 -14
  5. package/dist/index.js +97 -23
  6. package/dist/index.mjs +97 -23
  7. package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +192 -121
  8. package/dist/repo/apps/play-runner-workers/src/entry.ts +254 -65
  9. package/dist/repo/apps/play-runner-workers/src/runtime/receipts.ts +18 -27
  10. package/dist/repo/apps/play-runner-workers/src/workflow-instance-create.ts +44 -0
  11. package/dist/repo/apps/play-runner-workers/src/workflow-retry.ts +7 -11
  12. package/dist/repo/sdk/src/client.ts +35 -12
  13. package/dist/repo/sdk/src/errors.ts +2 -2
  14. package/dist/repo/sdk/src/http.ts +87 -7
  15. package/dist/repo/sdk/src/play.ts +1 -1
  16. package/dist/repo/sdk/src/plays/bundle-play-file.ts +5 -1
  17. package/dist/repo/sdk/src/release.ts +13 -10
  18. package/dist/repo/sdk/src/tool-output.ts +2 -2
  19. package/dist/repo/sdk/src/types.ts +9 -6
  20. package/dist/repo/shared_libs/play-runtime/fullenrich-batching.ts +229 -0
  21. package/dist/repo/shared_libs/play-runtime/governor/policy.ts +1 -1
  22. package/dist/repo/shared_libs/play-runtime/play-runtime-batching-registry.ts +20 -0
  23. package/dist/repo/shared_libs/play-runtime/run-failure.ts +20 -12
  24. package/dist/repo/shared_libs/play-runtime/run-ledger.ts +147 -70
  25. package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts +6 -2
  26. package/dist/repo/shared_libs/play-runtime/secret-redaction.ts +15 -0
  27. package/dist/repo/shared_libs/play-runtime/work-receipts.ts +1 -0
  28. package/dist/repo/shared_libs/plays/bundling/index.ts +193 -21
  29. package/dist/repo/shared_libs/plays/static-pipeline.ts +1 -3
  30. package/dist/repo/shared_libs/security/outbound-url-policy.ts +238 -0
  31. package/dist/repo/shared_libs/security/safe-fetch.ts +118 -0
  32. package/dist/viewer/viewer.css +617 -0
  33. package/dist/viewer/viewer.js +1496 -0
  34. package/package.json +5 -1
@@ -0,0 +1,229 @@
1
+ import {
2
+ defineBatchStrategyMap,
3
+ type BatchOperationStrategy,
4
+ } from './batching-types';
5
+
6
+ const FULLENRICH_BATCH_ITEM_KEY = 'deepline_batch_item_key';
7
+ const FULLENRICH_BATCH_SIZE = 100;
8
+
9
+ type FullEnrichContactRow = Record<string, unknown> & {
10
+ custom?: Record<string, string>;
11
+ };
12
+
13
+ type FullEnrichBulkPayload = {
14
+ name: string;
15
+ webhook_url?: string;
16
+ wait_for_completion?: boolean;
17
+ poll_interval_ms?: number;
18
+ max_wait_ms?: number;
19
+ data: FullEnrichContactRow[];
20
+ };
21
+
22
+ type FullEnrichBulkResult = Record<string, unknown> & {
23
+ data?: unknown[];
24
+ };
25
+
26
+ function sortRecord(value: Record<string, unknown>): Record<string, unknown> {
27
+ const entries = Object.entries(value)
28
+ .filter(([, entry]) => entry !== undefined)
29
+ .sort(([left], [right]) => left.localeCompare(right))
30
+ .map(([key, entry]) => [key, stableValue(entry)]);
31
+ return Object.fromEntries(entries);
32
+ }
33
+
34
+ function stableValue(value: unknown): unknown {
35
+ if (Array.isArray(value)) {
36
+ return value.map((entry) => stableValue(entry));
37
+ }
38
+ if (value && typeof value === 'object') {
39
+ return sortRecord(value as Record<string, unknown>);
40
+ }
41
+ return value;
42
+ }
43
+
44
+ function stableStringify(value: unknown): string {
45
+ return JSON.stringify(stableValue(value));
46
+ }
47
+
48
+ function readPath(root: unknown, path: string): unknown {
49
+ const parts = path.split('.');
50
+ let current: unknown = root;
51
+ for (const part of parts) {
52
+ if (!current || typeof current !== 'object') {
53
+ return undefined;
54
+ }
55
+ current = (current as Record<string, unknown>)[part];
56
+ }
57
+ return current;
58
+ }
59
+
60
+ function readArrayAtPaths(root: unknown, paths: string[]): unknown[] | null {
61
+ for (const path of paths) {
62
+ const value = readPath(root, path);
63
+ if (Array.isArray(value)) {
64
+ return value;
65
+ }
66
+ }
67
+ return null;
68
+ }
69
+
70
+ function readRows(payload: FullEnrichBulkPayload): FullEnrichContactRow[] {
71
+ return Array.isArray(payload.data) ? payload.data : [];
72
+ }
73
+
74
+ function isOneRowPayload(payload: FullEnrichBulkPayload): boolean {
75
+ return readRows(payload).length === 1;
76
+ }
77
+
78
+ function controlKey(payload: FullEnrichBulkPayload): string {
79
+ return stableStringify({
80
+ webhook_url: payload.webhook_url ?? null,
81
+ wait_for_completion: payload.wait_for_completion ?? null,
82
+ poll_interval_ms: payload.poll_interval_ms ?? null,
83
+ max_wait_ms: payload.max_wait_ms ?? null,
84
+ });
85
+ }
86
+
87
+ function rowIdentity(row: FullEnrichContactRow): string {
88
+ return stableStringify({
89
+ first_name: row.first_name ?? null,
90
+ last_name: row.last_name ?? null,
91
+ domain: row.domain ?? null,
92
+ company_name: row.company_name ?? null,
93
+ linkedin_url: row.linkedin_url ?? null,
94
+ enrich_fields: row.enrich_fields ?? null,
95
+ custom: row.custom ?? null,
96
+ });
97
+ }
98
+
99
+ function shortStableHash(value: string): string {
100
+ let hash = 0x811c9dc5;
101
+ for (let index = 0; index < value.length; index += 1) {
102
+ hash ^= value.charCodeAt(index);
103
+ hash = Math.imul(hash, 0x01000193) >>> 0;
104
+ }
105
+ return hash.toString(36);
106
+ }
107
+
108
+ function itemKey(payload: FullEnrichBulkPayload, index = 0): string {
109
+ return `dl_${index}_${shortStableHash(rowIdentity(readRows(payload)[0] ?? {}))}`;
110
+ }
111
+
112
+ function withBatchItemKey(
113
+ row: FullEnrichContactRow,
114
+ key: string,
115
+ ): FullEnrichContactRow {
116
+ return {
117
+ ...row,
118
+ custom: {
119
+ ...(row.custom ?? {}),
120
+ [FULLENRICH_BATCH_ITEM_KEY]: key,
121
+ },
122
+ };
123
+ }
124
+
125
+ function extractResultRows(fullResult: FullEnrichBulkResult | unknown) {
126
+ return readArrayAtPaths(fullResult, ['data']) ?? [];
127
+ }
128
+
129
+ function readResultItemKey(row: unknown): string | null {
130
+ if (!row || typeof row !== 'object' || Array.isArray(row)) {
131
+ return null;
132
+ }
133
+ const custom = (row as Record<string, unknown>).custom;
134
+ if (!custom || typeof custom !== 'object' || Array.isArray(custom)) {
135
+ return null;
136
+ }
137
+ const key = (custom as Record<string, unknown>)[FULLENRICH_BATCH_ITEM_KEY];
138
+ return typeof key === 'string' && key.trim() ? key.trim() : null;
139
+ }
140
+
141
+ const fullenrichBulkSelfBatchStrategy: BatchOperationStrategy<
142
+ FullEnrichBulkPayload,
143
+ FullEnrichBulkPayload,
144
+ FullEnrichBulkResult,
145
+ { data: FullEnrichBulkResult },
146
+ unknown
147
+ > = {
148
+ sourceOperation: 'fullenrich_bulk_enrich',
149
+ batchOperation: 'fullenrich_bulk_enrich',
150
+ kind: 'identifier_batch',
151
+ maxBatchSize: FULLENRICH_BATCH_SIZE,
152
+ canBatchWith(left, right) {
153
+ return (
154
+ isOneRowPayload(left) &&
155
+ isOneRowPayload(right) &&
156
+ controlKey(left) === controlKey(right)
157
+ );
158
+ },
159
+ toBucketKey(payload) {
160
+ if (!isOneRowPayload(payload)) {
161
+ return `fullenrich_bulk_enrich:passthrough:${stableStringify(payload)}`;
162
+ }
163
+ return `fullenrich_bulk_enrich:${controlKey(payload)}`;
164
+ },
165
+ toItemKey(payload) {
166
+ return itemKey(payload);
167
+ },
168
+ compile(payloads) {
169
+ const first = payloads[0];
170
+ const rows = payloads.flatMap((payload, index) =>
171
+ readRows(payload).map((row) =>
172
+ withBatchItemKey(row, itemKey(payload, index)),
173
+ ),
174
+ );
175
+ return {
176
+ batchOperation: 'fullenrich_bulk_enrich',
177
+ batchPayload: {
178
+ name:
179
+ payloads.length === 1
180
+ ? first?.name || 'deepline-fullenrich-batch'
181
+ : `deepline-fullenrich-batch-${payloads.length}`,
182
+ ...(first?.webhook_url ? { webhook_url: first.webhook_url } : {}),
183
+ ...(first?.wait_for_completion !== undefined
184
+ ? { wait_for_completion: first.wait_for_completion }
185
+ : {}),
186
+ ...(first?.poll_interval_ms !== undefined
187
+ ? { poll_interval_ms: first.poll_interval_ms }
188
+ : {}),
189
+ ...(first?.max_wait_ms !== undefined
190
+ ? { max_wait_ms: first.max_wait_ms }
191
+ : {}),
192
+ data: rows,
193
+ },
194
+ items: payloads.map((payload, index) => ({
195
+ itemKey: itemKey(payload, index),
196
+ payload,
197
+ })),
198
+ };
199
+ },
200
+ splitResult(fullResult, compiled) {
201
+ const resultRows = extractResultRows(fullResult);
202
+ const rowsByItemKey = new Map<string, unknown>();
203
+ for (const row of resultRows) {
204
+ const key = readResultItemKey(row);
205
+ if (key) {
206
+ rowsByItemKey.set(key, row);
207
+ }
208
+ }
209
+
210
+ return compiled.items.map((item, index) => {
211
+ const matchedRow =
212
+ rowsByItemKey.get(item.itemKey) ??
213
+ (index < resultRows.length ? resultRows[index] : null);
214
+ const singleResult = {
215
+ ...fullResult,
216
+ data: matchedRow ? [matchedRow] : [],
217
+ };
218
+ return {
219
+ itemKey: item.itemKey,
220
+ result: { data: singleResult },
221
+ rawResult: matchedRow,
222
+ };
223
+ });
224
+ },
225
+ };
226
+
227
+ export const fullenrichBatchStrategies = defineBatchStrategyMap({
228
+ fullenrich_bulk_enrich: fullenrichBulkSelfBatchStrategy,
229
+ });
@@ -116,7 +116,7 @@ export const SHARED_EXECUTION_POLICY: ResolvedExecutionPolicy = {
116
116
  },
117
117
  pacing: {
118
118
  // Undeclared providers; declared providers (rate-limit-definitions.ts) win.
119
- defaultProviderRequestsPerSecond: 30,
119
+ defaultProviderRequestsPerSecond: 10,
120
120
  suggestedMaxParallelism: 50,
121
121
  },
122
122
  };
@@ -0,0 +1,20 @@
1
+ import type { AnyBatchOperationStrategy } from './batching-types';
2
+ import { DEFAULT_PLAY_RUNTIME_BATCH_STRATEGIES } from './default-batch-strategies';
3
+ import { fullenrichBatchStrategies } from './fullenrich-batching';
4
+
5
+ export const PLAY_RUNTIME_BATCH_OPERATION_REGISTRY: Record<
6
+ string,
7
+ AnyBatchOperationStrategy
8
+ > = {
9
+ ...DEFAULT_PLAY_RUNTIME_BATCH_STRATEGIES,
10
+ ...fullenrichBatchStrategies,
11
+ };
12
+
13
+ export function getPlayRuntimeBatchStrategy(
14
+ operation: string | null | undefined,
15
+ ): AnyBatchOperationStrategy | null {
16
+ if (!operation) {
17
+ return null;
18
+ }
19
+ return PLAY_RUNTIME_BATCH_OPERATION_REGISTRY[operation] ?? null;
20
+ }
@@ -1,9 +1,12 @@
1
- export const CLOUDFLARE_DURABLE_OBJECT_CODE_UPDATED_ERROR =
2
- 'Durable Object reset because its code was updated.';
1
+ const CLOUDFLARE_DURABLE_OBJECT_RESET_RE =
2
+ /Durable Object.*(?:code was updated|storage caused object)/;
3
3
 
4
4
  export const PLATFORM_DEPLOY_INTERRUPTED_MESSAGE =
5
5
  'Run interrupted by a platform deploy and was not retried automatically. Re-run the same command; the input is unchanged.';
6
6
 
7
+ export const INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE =
8
+ 'Internal play runtime storage failed. Please retry the run; if this keeps happening, contact Deepline support with the run ID.';
9
+
7
10
  export type PlayRunFailureDetails = {
8
11
  code: string;
9
12
  phase: string;
@@ -19,23 +22,15 @@ function toErrorText(error: unknown): string {
19
22
  return String(error);
20
23
  }
21
24
 
22
- export function isCloudflareDurableObjectCodeUpdatedError(
23
- error: unknown,
24
- ): boolean {
25
- return toErrorText(error).includes(
26
- CLOUDFLARE_DURABLE_OBJECT_CODE_UPDATED_ERROR,
27
- );
28
- }
29
-
30
25
  export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
31
26
  const cause = toErrorText(error);
32
- if (isCloudflareDurableObjectCodeUpdatedError(cause)) {
27
+ if (CLOUDFLARE_DURABLE_OBJECT_RESET_RE.test(cause)) {
33
28
  return {
34
29
  code: 'PLATFORM_DEPLOY_INTERRUPTED',
35
30
  phase: 'runtime',
36
31
  message: PLATFORM_DEPLOY_INTERRUPTED_MESSAGE,
37
32
  retryable: true,
38
- cause: CLOUDFLARE_DURABLE_OBJECT_CODE_UPDATED_ERROR,
33
+ cause,
39
34
  };
40
35
  }
41
36
  const playDepthBudgetMatch = cause.match(
@@ -50,6 +45,19 @@ export function normalizePlayRunFailure(error: unknown): PlayRunFailureDetails {
50
45
  cause,
51
46
  };
52
47
  }
48
+ const lowerCause = cause.toLowerCase();
49
+ if (
50
+ lowerCause.includes('neondberror') ||
51
+ lowerCause.includes('bind message supplies') ||
52
+ lowerCause.includes('prepared statement')
53
+ ) {
54
+ return {
55
+ code: 'RUN_STORAGE_FAILED',
56
+ phase: 'storage',
57
+ message: INTERNAL_RUNTIME_STORAGE_ERROR_MESSAGE,
58
+ retryable: true,
59
+ };
60
+ }
53
61
  return {
54
62
  code: 'RUN_FAILED',
55
63
  phase: 'runtime',
@@ -549,22 +549,17 @@ function appendLogLines(
549
549
 
550
550
  /**
551
551
  * Terminal-status precedence. Re-delivery of the same terminal status is a
552
- * benign no-op handled by the regular reduction. A terminal event that
553
- * disagrees with an already-terminal snapshot is ignored (keeping e.g. an
554
- * explicit user cancellation from being flipped to completed by a late
555
- * worker terminal), and the conflict is recorded as one anomaly log line —
556
- * with ONE exception: `run.failed` DEMOTES a `completed` snapshot. The
557
- * worker appends run.completed BEFORE awaiting post-completion accounting
558
- * (compute billing finalize), so a billing business denial there — e.g. the
559
- * per-run credit cap (maxCreditsPerRun) — arrives as a later run.failed
560
- * that MUST fail the run. Blanket first-terminal-wins silently completed
561
- * capped runs (regression pinned by
562
- * tests/v2-plays/plays/44-compute-billing-cap.play.ts). A cancelled run
563
- * stays cancelled, and a failed run can never be flipped to completed.
552
+ * benign no-op handled by the regular reduction. Conflicting terminal events
553
+ * use newest-terminal-wins by event time: older events are ignored and logged,
554
+ * newer events reconcile the snapshot. This is the idempotency model callers
555
+ * expect when duplicate/retried work completes out of order. It also preserves
556
+ * post-completion billing cap demotion because that denial arrives as a newer
557
+ * run.failed event.
564
558
  */
565
559
  function conflictingTerminalSnapshot(
566
560
  base: PlayRunLedgerSnapshot,
567
561
  eventType: keyof typeof TERMINAL_STATUS_BY_EVENT_TYPE,
562
+ occurredAt: number,
568
563
  ): PlayRunLedgerSnapshot | null {
569
564
  if (!isTerminalPlayRunLedgerStatus(base.status)) {
570
565
  return null;
@@ -572,14 +567,16 @@ function conflictingTerminalSnapshot(
572
567
  if (TERMINAL_STATUS_BY_EVENT_TYPE[eventType] === base.status) {
573
568
  return null;
574
569
  }
575
- if (base.status === 'completed' && eventType === 'run.failed') {
576
- // Post-completion accounting demotion (e.g. per-run billing cap denial):
577
- // let the regular run.failed reduction flip the run to failed.
570
+ const terminalAt = base.finishedAt ?? base.updatedAt ?? 0;
571
+ if (occurredAt > terminalAt) {
572
+ // Newer terminal evidence reconciles the run. This covers replay/receipt
573
+ // races where an earlier attempt failed but a later attempt recovered and
574
+ // completed with the durable result.
578
575
  return null;
579
576
  }
580
577
  return withTiming(
581
578
  appendLogLines(base, [
582
- `[ledger] conflicting terminal event ${eventType} ignored; status already ${base.status}`,
579
+ `[ledger] stale conflicting terminal event ${eventType} ignored; status already ${base.status}`,
583
580
  ]),
584
581
  );
585
582
  }
@@ -617,6 +614,23 @@ function settleRunningStepsOnTerminal(
617
614
  return changed ? { ...snapshot, stepsById } : snapshot;
618
615
  }
619
616
 
617
+ function terminalFinishedAt(
618
+ base: PlayRunLedgerSnapshot,
619
+ eventType: keyof typeof TERMINAL_STATUS_BY_EVENT_TYPE,
620
+ occurredAt: number,
621
+ ): number {
622
+ const nextStatus = TERMINAL_STATUS_BY_EVENT_TYPE[eventType];
623
+ const terminalAt = base.finishedAt ?? base.updatedAt ?? 0;
624
+ if (
625
+ isTerminalPlayRunLedgerStatus(base.status) &&
626
+ nextStatus !== base.status &&
627
+ occurredAt > terminalAt
628
+ ) {
629
+ return occurredAt;
630
+ }
631
+ return base.finishedAt ?? occurredAt;
632
+ }
633
+
620
634
  export function reducePlayRunLedgerEvent(
621
635
  snapshot: PlayRunLedgerSnapshot,
622
636
  event: PlayRunLedgerEvent,
@@ -654,52 +668,46 @@ export function reducePlayRunLedgerEvent(
654
668
  });
655
669
  case 'run.completed':
656
670
  return (
657
- conflictingTerminalSnapshot(base, event.type) ??
658
- withTiming(
659
- {
660
- ...settleRunningStepsOnTerminal(base, 'completed', occurredAt),
661
- status: 'completed',
662
- error: null,
663
- startedAt: base.startedAt ?? occurredAt,
664
- finishedAt: base.finishedAt ?? occurredAt,
665
- activeStepId: null,
666
- activeArtifactTableNamespace: null,
667
- result: event.result ?? base.result,
668
- resultSummary: event.resultSummary ?? base.resultSummary,
669
- },
670
- )
671
+ conflictingTerminalSnapshot(base, event.type, occurredAt) ??
672
+ withTiming({
673
+ ...settleRunningStepsOnTerminal(base, 'completed', occurredAt),
674
+ status: 'completed',
675
+ error: null,
676
+ startedAt: base.startedAt ?? occurredAt,
677
+ finishedAt: terminalFinishedAt(base, event.type, occurredAt),
678
+ activeStepId: null,
679
+ activeArtifactTableNamespace: null,
680
+ result: event.result ?? base.result,
681
+ resultSummary: event.resultSummary ?? base.resultSummary,
682
+ })
671
683
  );
672
684
  case 'run.failed':
673
685
  return (
674
- conflictingTerminalSnapshot(base, event.type) ??
675
- withTiming(
676
- {
677
- ...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
678
- status: 'failed',
679
- error: event.error ?? base.error ?? null,
680
- startedAt: base.startedAt ?? occurredAt,
681
- finishedAt: base.finishedAt ?? occurredAt,
682
- activeStepId: null,
683
- activeArtifactTableNamespace: null,
684
- result: event.result ?? base.result,
685
- },
686
- )
686
+ conflictingTerminalSnapshot(base, event.type, occurredAt) ??
687
+ withTiming({
688
+ ...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
689
+ status: 'failed',
690
+ error: event.error ?? base.error ?? null,
691
+ startedAt: base.startedAt ?? occurredAt,
692
+ finishedAt: terminalFinishedAt(base, event.type, occurredAt),
693
+ activeStepId: null,
694
+ activeArtifactTableNamespace: null,
695
+ result: event.result ?? base.result,
696
+ })
687
697
  );
688
698
  case 'run.cancelled':
689
699
  return (
690
- conflictingTerminalSnapshot(base, event.type) ??
691
- withTiming(
692
- {
693
- ...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
694
- status: 'cancelled',
695
- error: event.error ?? base.error ?? null,
696
- startedAt: base.startedAt ?? occurredAt,
697
- finishedAt: base.finishedAt ?? occurredAt,
698
- activeStepId: null,
699
- activeArtifactTableNamespace: null,
700
- result: event.result ?? base.result,
701
- },
702
- )
700
+ conflictingTerminalSnapshot(base, event.type, occurredAt) ??
701
+ withTiming({
702
+ ...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
703
+ status: 'cancelled',
704
+ error: event.error ?? base.error ?? null,
705
+ startedAt: base.startedAt ?? occurredAt,
706
+ finishedAt: terminalFinishedAt(base, event.type, occurredAt),
707
+ activeStepId: null,
708
+ activeArtifactTableNamespace: null,
709
+ result: event.result ?? base.result,
710
+ })
703
711
  );
704
712
  case 'log.appended':
705
713
  return withTiming(
@@ -712,6 +720,11 @@ export function reducePlayRunLedgerEvent(
712
720
  );
713
721
  case 'step.started': {
714
722
  const current = base.stepsById[event.stepId];
723
+ const shouldRefineSyntheticStart =
724
+ current?.startedAt != null &&
725
+ current.completedAt != null &&
726
+ current.startedAt === current.completedAt &&
727
+ occurredAt < current.startedAt;
715
728
  const nextStep: PlayRunLedgerStepSnapshot = {
716
729
  ...(current ?? { stepId: event.stepId, status: 'running' as const }),
717
730
  stepId: event.stepId,
@@ -725,8 +738,10 @@ export function reducePlayRunLedgerEvent(
725
738
  event.artifactTableNamespace ??
726
739
  current?.artifactTableNamespace ??
727
740
  null,
728
- startedAt: current?.startedAt ?? occurredAt,
729
- updatedAt: occurredAt,
741
+ startedAt: shouldRefineSyntheticStart
742
+ ? occurredAt
743
+ : (current?.startedAt ?? occurredAt),
744
+ updatedAt: Math.max(current?.updatedAt ?? 0, occurredAt),
730
745
  };
731
746
  return withTiming({
732
747
  ...base,
@@ -767,6 +782,25 @@ export function reducePlayRunLedgerEvent(
767
782
  : event.status === 'running' && inferredStatus === 'completed'
768
783
  ? 'completed'
769
784
  : (event.status ?? current?.status ?? inferredStatus);
785
+ const completedAt =
786
+ current?.completedAt ??
787
+ event.progress.completedAt ??
788
+ (status === 'completed'
789
+ ? (event.progress.updatedAt ?? occurredAt)
790
+ : null);
791
+ const shouldRefineSyntheticStart =
792
+ current?.startedAt != null &&
793
+ current.completedAt != null &&
794
+ current.startedAt === current.completedAt &&
795
+ event.progress.startedAt != null &&
796
+ event.progress.startedAt < current.startedAt;
797
+ const startedAt =
798
+ (shouldRefineSyntheticStart ? event.progress.startedAt : null) ??
799
+ current?.startedAt ??
800
+ event.progress.startedAt ??
801
+ (status === 'completed' || status === 'failed'
802
+ ? (completedAt ?? event.progress.updatedAt ?? occurredAt)
803
+ : null);
770
804
  const nextStep: PlayRunLedgerStepSnapshot = {
771
805
  ...(current ?? { stepId: event.stepId, status }),
772
806
  stepId: event.stepId,
@@ -777,13 +811,8 @@ export function reducePlayRunLedgerEvent(
777
811
  progress.artifactTableNamespace ??
778
812
  current?.artifactTableNamespace ??
779
813
  null,
780
- startedAt: current?.startedAt ?? event.progress.startedAt ?? null,
781
- completedAt:
782
- current?.completedAt ??
783
- event.progress.completedAt ??
784
- (status === 'completed'
785
- ? (event.progress.updatedAt ?? occurredAt)
786
- : null),
814
+ startedAt,
815
+ completedAt,
787
816
  updatedAt: occurredAt,
788
817
  progress,
789
818
  };
@@ -818,6 +847,10 @@ export function reducePlayRunLedgerEvent(
818
847
  : event.type === 'step.failed'
819
848
  ? 'failed'
820
849
  : 'skipped';
850
+ const completedAt =
851
+ status === 'skipped' || preserveFailedStatus
852
+ ? (current?.completedAt ?? null)
853
+ : occurredAt;
821
854
  const nextStep: PlayRunLedgerStepSnapshot = {
822
855
  ...(current ?? { stepId: event.stepId, status }),
823
856
  stepId: event.stepId,
@@ -828,11 +861,12 @@ export function reducePlayRunLedgerEvent(
828
861
  event.artifactTableNamespace ??
829
862
  current?.artifactTableNamespace ??
830
863
  null,
831
- startedAt: current?.startedAt ?? null,
832
- completedAt:
833
- status === 'skipped' || preserveFailedStatus
834
- ? (current?.completedAt ?? null)
835
- : occurredAt,
864
+ startedAt:
865
+ current?.startedAt ??
866
+ (status === 'completed' || status === 'failed'
867
+ ? (completedAt ?? occurredAt)
868
+ : null),
869
+ completedAt,
836
870
  updatedAt: occurredAt,
837
871
  progress: current?.progress ?? null,
838
872
  };
@@ -927,6 +961,20 @@ export function slicePositionalLogLines(input: {
927
961
  return { lines: [...lines], channelOffset: sendFromOffset };
928
962
  }
929
963
 
964
+ function terminalLogReplayChannelOffset(input: {
965
+ liveLogTotalCount?: number;
966
+ liveLogsLength: number;
967
+ }): number | null {
968
+ if (
969
+ typeof input.liveLogTotalCount !== 'number' ||
970
+ !Number.isFinite(input.liveLogTotalCount) ||
971
+ input.liveLogTotalCount < input.liveLogsLength
972
+ ) {
973
+ return null;
974
+ }
975
+ return input.liveLogTotalCount - input.liveLogsLength;
976
+ }
977
+
930
978
  /**
931
979
  * Forward producer log lines as one `log.appended` event.
932
980
  *
@@ -968,6 +1016,35 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
968
1016
  ];
969
1017
  }
970
1018
 
1019
+ /**
1020
+ * Re-send a terminal retained log tail through terminal transport recovery.
1021
+ *
1022
+ * Live worker flushes use positional channel offsets. Terminal replay is a
1023
+ * different transport: it replays the runner's retained output after terminal
1024
+ * state is known. When the producer knows the retained tail's offset, Convex
1025
+ * uses that offset only to bound occurrence-count reconciliation to the same
1026
+ * tail window; it does not route the replay through the worker cursor.
1027
+ */
1028
+ export function buildTerminalLogReplayEvents(input: {
1029
+ runId: string;
1030
+ lines: readonly string[];
1031
+ source?: PlayRunLedgerEventSource;
1032
+ occurredAt?: number;
1033
+ liveLogTotalCount?: number;
1034
+ }): PlayRunLedgerEvent[] {
1035
+ const channelOffset = terminalLogReplayChannelOffset({
1036
+ liveLogTotalCount: input.liveLogTotalCount,
1037
+ liveLogsLength: input.lines.length,
1038
+ });
1039
+ return buildPlayRunLedgerEventsFromLogLines({
1040
+ runId: input.runId,
1041
+ lines: input.lines,
1042
+ source: 'coordinator',
1043
+ occurredAt: input.occurredAt,
1044
+ channelOffset,
1045
+ });
1046
+ }
1047
+
971
1048
  export function buildPlayRunLedgerEventsFromStatusPatch(input: {
972
1049
  patch: PlayRunLedgerStatusPatch;
973
1050
  previousSnapshot: PlayRunLedgerSnapshot;
@@ -4,8 +4,8 @@
4
4
  * One of three pluggable axes (alongside runner-backends and dedup-backends).
5
5
  * Selected per-run via PlayExecutionProfile.
6
6
  *
7
- * Temporal is the existing production scheduler. Cloudflare Workflows is the
8
- * edge scheduler used by the workers_edge profile.
7
+ * Cloudflare Workflows is the default production scheduler through the
8
+ * workers_edge profile. Hatchet is selected explicitly by the hatchet profile.
9
9
  *
10
10
  * Customer plays are unaffected — this is purely the orchestration layer.
11
11
  */
@@ -18,6 +18,7 @@ import type {
18
18
  import type { ExecutionPlan } from './execution-plan';
19
19
  import type { PlayRuntimeManifestMap } from '../plays/compiler-manifest';
20
20
  import type { PreloadedRuntimeDbSession } from './db-session';
21
+ import type { PlayRunnerRuntimeTiming } from './protocol';
21
22
 
22
23
  export const PLAY_SCHEDULER_BACKENDS = {
23
24
  temporal: 'temporal',
@@ -128,6 +129,8 @@ export type PlaySchedulerSubmitInput = {
128
129
  userId?: string | null;
129
130
  source?: 'published' | 'ad_hoc' | 'draft';
130
131
  executionProfile?: string | null;
132
+ /** Durable per-workspace active run cap enforced when the run row is projected. */
133
+ activeRunLimit?: number | null;
131
134
  /** runner backend to use for executing attempts */
132
135
  runtimeBackend: string;
133
136
  /** dedup backend for cross-attempt cross-process idempotency */
@@ -195,6 +198,7 @@ export type PlaySchedulerResultEnvelope = {
195
198
  finalCheckpoint?: PlayCheckpoint;
196
199
  totalRows?: number;
197
200
  durationMs?: number;
201
+ runtimeTiming?: PlayRunnerRuntimeTiming;
198
202
  };
199
203
 
200
204
  export interface PlaySchedulerBackend {