deepline 0.1.210 → 0.1.212

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +203 -12
  2. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +23 -0
  3. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +15 -0
  4. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts +90 -0
  5. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +162 -100
  6. package/dist/bundling-sources/sdk/src/client.ts +118 -3
  7. package/dist/bundling-sources/sdk/src/play.ts +2 -0
  8. package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
  9. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  10. package/dist/bundling-sources/sdk/src/types.ts +47 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +253 -30
  12. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +365 -103
  13. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +12 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts +10 -4
  15. package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts +9 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +45 -8
  17. package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +14 -2
  18. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +4 -2
  19. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +21 -1
  20. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +128 -6
  21. package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +7 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +5 -1
  23. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +37 -5
  24. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +19 -10
  25. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +63 -7
  26. package/dist/bundling-sources/shared_libs/play-runtime/secret-capability.ts +18 -4
  27. package/dist/bundling-sources/shared_libs/play-runtime/single-flight.ts +31 -0
  28. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +12 -1
  29. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +29 -9
  30. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +22 -1
  31. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +79 -1
  32. package/dist/bundling-sources/shared_libs/plays/dataset.ts +41 -5
  33. package/dist/cli/index.js +728 -239
  34. package/dist/cli/index.mjs +728 -239
  35. package/dist/index.d.mts +93 -7
  36. package/dist/index.d.ts +93 -7
  37. package/dist/index.js +245 -46
  38. package/dist/index.mjs +245 -46
  39. package/dist/plays/bundle-play-file.mjs +65 -4
  40. package/package.json +1 -1
@@ -380,6 +380,18 @@ export interface MapExecutionScope {
380
380
  }
381
381
 
382
382
  export type PlayExecutionEvent =
383
+ | {
384
+ type: 'dataset.lifecycle';
385
+ datasetId: string;
386
+ path: string;
387
+ tableNamespace: string;
388
+ phase: 'registered' | 'available' | 'failed';
389
+ persistedRows?: number;
390
+ succeededRows?: number;
391
+ failedRows?: number;
392
+ complete?: boolean;
393
+ at: number;
394
+ }
383
395
  | {
384
396
  type: 'map.preparing';
385
397
  mapInvocationId: string;
@@ -1,10 +1,16 @@
1
- import { createHash } from 'node:crypto';
1
+ import {
2
+ normalizePlayNameForSheet,
3
+ normalizeTableNamespace,
4
+ sha256Hex,
5
+ } from '../plays/row-identity';
2
6
 
3
7
  export function createRuntimeDatasetId(
4
8
  playName: string,
5
9
  tableNamespace: string,
6
10
  ): string {
7
- return createHash('sha1')
8
- .update(`${playName}:${tableNamespace}`)
9
- .digest('hex');
11
+ const normalizedPlayName = normalizePlayNameForSheet(playName);
12
+ const normalizedTableNamespace = normalizeTableNamespace(tableNamespace);
13
+ return sha256Hex(
14
+ `${normalizedPlayName}:${normalizedTableNamespace}`,
15
+ ).slice(0, 40);
10
16
  }
@@ -56,6 +56,7 @@ export type CreateDbSessionRequest = {
56
56
 
57
57
  export type CreateDbSessionResponse = {
58
58
  sessionId: string;
59
+ executionRole?: string;
59
60
  postgresUrl?: string;
60
61
  encryptedPostgresUrl?: EncryptedPostgresUrl;
61
62
  postgres?: {
@@ -398,6 +399,14 @@ export const createDbSessionResponseSchema =
398
399
  }
399
400
  return {
400
401
  sessionId: parseString(value.sessionId, 'sessionId'),
402
+ ...(value.executionRole !== undefined
403
+ ? {
404
+ executionRole: parseOptionalString(
405
+ value.executionRole,
406
+ 'executionRole',
407
+ ),
408
+ }
409
+ : {}),
401
410
  ...(value.postgresUrl !== undefined
402
411
  ? { postgresUrl: parseOptionalString(value.postgresUrl, 'postgresUrl') }
403
412
  : {}),
@@ -243,9 +243,11 @@ async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
243
243
  rejectLeaseLost(error as RuntimeReceiptLeaseLostError),
244
244
  });
245
245
 
246
- if ((await heartbeatOnce()) === 'active') {
247
- supervisor.start();
248
- }
246
+ // markRunning is the authoritative pre-dispatch ownership fence. Renewal is
247
+ // only needed when the lease approaches expiry; an immediate read/renewal
248
+ // adds a round trip to every short tool call without extending ownership.
249
+ // Completion remains conditional on this lease, and long work still renews.
250
+ supervisor.start();
249
251
  try {
250
252
  return await Promise.race([input.execute(), leaseLost]);
251
253
  } finally {
@@ -294,6 +296,7 @@ export async function waitForCompletedRuntimeReceipts(input: {
294
296
  maxAttempts?: number;
295
297
  delayMs?: number;
296
298
  abortSignal?: AbortSignal;
299
+ log?: (message: string) => void;
297
300
  }): Promise<{
298
301
  completed: Map<string, RuntimeStepReceipt>;
299
302
  failed: Map<string, Error>;
@@ -326,6 +329,20 @@ export async function waitForCompletedRuntimeReceipts(input: {
326
329
  }
327
330
  throw error;
328
331
  }
332
+ const statuses = [...receipts.values()].reduce<Record<string, number>>(
333
+ (counts, receipt) => {
334
+ counts[receipt.status] = (counts[receipt.status] ?? 0) + 1;
335
+ return counts;
336
+ },
337
+ {},
338
+ );
339
+ input.log?.(
340
+ `[perf] runtime-receipt-wait attempt=${attempt} requested=${pending.size} indexed=${receipts.size} statuses=${
341
+ Object.entries(statuses)
342
+ .map(([status, count]) => `${status}:${count}`)
343
+ .join(',') || 'none'
344
+ }`,
345
+ );
329
346
  for (const key of [...pending]) {
330
347
  const receipt = receipts.get(key);
331
348
  if (receipt?.status === 'completed' || receipt?.status === 'skipped') {
@@ -343,6 +360,9 @@ export async function waitForCompletedRuntimeReceipts(input: {
343
360
  pending.delete(key);
344
361
  }
345
362
  }
363
+ input.log?.(
364
+ `[perf] runtime-receipt-wait attempt=${attempt} remaining=${pending.size}`,
365
+ );
346
366
  }
347
367
 
348
368
  return { completed, failed, timedOut: pending };
@@ -374,6 +394,16 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
374
394
  log: (message: string) => void;
375
395
  execute: (context: { leaseId: string | null }) => Promise<T>;
376
396
  }): Promise<T> {
397
+ const lifecycleStartedAt = Date.now();
398
+ const logPhase = (phase: string, startedAt: number): void => {
399
+ // Keep this at the semantic receipt layer. Transport timing below this
400
+ // function explains HTTP cost; these markers expose gaps before/after the
401
+ // store calls and prove whether user code or receipt orchestration owns the
402
+ // elapsed time. Do not log receipt keys because they encode tool inputs.
403
+ input.log(
404
+ `[perf] durable receipt operation=${input.operation} id=${input.id} phase=${phase} elapsed_ms=${Date.now() - startedAt} lifecycle_ms=${Date.now() - lifecycleStartedAt}`,
405
+ );
406
+ };
377
407
  if (!input.store.enabled) {
378
408
  return await input.execute({ leaseId: null });
379
409
  }
@@ -382,9 +412,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
382
412
  receipt: RuntimeStepReceipt,
383
413
  source: DurableReceiptRecoverySource = 'cache',
384
414
  ): Promise<T> => {
385
- input.log(
386
- `ctx.${input.operation}(${input.id}): recovered result from receipt`,
387
- );
415
+ input.log(`ctx.${input.operation}(${input.id}): reused completed work`);
388
416
  if (receipt.output === undefined) {
389
417
  return receipt.output as T;
390
418
  }
@@ -417,7 +445,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
417
445
  }
418
446
  if (reclaimed?.status === 'failed') {
419
447
  throw new Error(
420
- `ctx.${input.operation}(${input.id}): receipt is failed and could not be reclaimed: ${reclaimed.error ?? 'unknown error'}.`,
448
+ `ctx.${input.operation}(${input.id}): previous execution failed and cannot be reused: ${reclaimed.error ?? 'unknown error'}.`,
421
449
  );
422
450
  }
423
451
  if (
@@ -479,16 +507,18 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
479
507
  receipt: RuntimeStepReceipt,
480
508
  ): Promise<never> => {
481
509
  throw new Error(
482
- `ctx.${input.operation}(${input.id}): receipt is failed and could not be claimed: ${receipt.error ?? 'unknown error'}.`,
510
+ `ctx.${input.operation}(${input.id}): previous execution failed and cannot be reused: ${receipt.error ?? 'unknown error'}.`,
483
511
  );
484
512
  };
485
513
 
514
+ const claimStartedAt = Date.now();
486
515
  const claimed = await input.store.claim(
487
516
  input.receiptKey,
488
517
  input.runId,
489
518
  input.reclaimRunning === true || input.force === true,
490
519
  input.force === true,
491
520
  );
521
+ logPhase('claim', claimStartedAt);
492
522
  if (
493
523
  input.force !== true &&
494
524
  (claimed?.status === 'completed' || claimed?.status === 'skipped')
@@ -525,11 +555,13 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
525
555
  let result: T;
526
556
  try {
527
557
  if (input.markRunningBeforeExecute !== false && input.store.markRunning) {
558
+ const markRunningStartedAt = Date.now();
528
559
  const running = await input.store.markRunning(
529
560
  input.receiptKey,
530
561
  input.runId,
531
562
  ownedLeaseId,
532
563
  );
564
+ logPhase('mark_running', markRunningStartedAt);
533
565
  if (!running || running.status !== 'running') {
534
566
  throw new RuntimeReceiptLeaseLostError({
535
567
  receiptKey: input.receiptKey,
@@ -540,6 +572,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
540
572
  ownedLeaseId = running.leaseId ?? ownedLeaseId;
541
573
  ownedLeaseExpiresAt = running.leaseExpiresAt ?? ownedLeaseExpiresAt;
542
574
  }
575
+ const executeStartedAt = Date.now();
543
576
  const executed =
544
577
  ownedLeaseId && input.store.heartbeat
545
578
  ? await executeWithDurableRuntimeReceiptHeartbeat({
@@ -561,6 +594,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
561
594
  execute: () => input.execute({ leaseId: ownedLeaseId }),
562
595
  })
563
596
  : await input.execute({ leaseId: ownedLeaseId });
597
+ logPhase('execute', executeStartedAt);
564
598
  result = input.onClaimedResult
565
599
  ? input.onClaimedResult(executed, input.receiptKey)
566
600
  : executed;
@@ -603,12 +637,14 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
603
637
  throw error;
604
638
  }
605
639
 
640
+ const completeStartedAt = Date.now();
606
641
  const completed = await input.store.complete(
607
642
  input.receiptKey,
608
643
  input.runId,
609
644
  isToolExecuteResult(result) ? serializeToolExecuteResult(result) : result,
610
645
  ownedLeaseId,
611
646
  );
647
+ logPhase('complete', completeStartedAt);
612
648
  if (
613
649
  !completed ||
614
650
  (completed.status !== 'completed' && completed.status !== 'skipped')
@@ -633,6 +669,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
633
669
  completed &&
634
670
  (completed.status === 'completed' || completed.status === 'skipped')
635
671
  ) {
672
+ logPhase('total', lifecycleStartedAt);
636
673
  return await recoverCompletedReceipt(completed, 'owner');
637
674
  }
638
675
  if (input.store.canPersistCompletion) {
@@ -1,7 +1,10 @@
1
1
  import { createSecretRedactionContext } from './secret-redaction';
2
- import { RUNTIME_RECEIPT_OUTPUT_MAX_BYTES } from './output-size-limits';
2
+ import {
3
+ assertRuntimeReceiptOutputWithinLimit,
4
+ TERMINAL_RUN_RESULT_MAX_BYTES,
5
+ } from './output-size-limits';
3
6
 
4
- export const MAX_LEDGER_RESULT_PAYLOAD_BYTES = RUNTIME_RECEIPT_OUTPUT_MAX_BYTES;
7
+ export const MAX_LEDGER_RESULT_PAYLOAD_BYTES = TERMINAL_RUN_RESULT_MAX_BYTES;
5
8
  export const MAX_LEDGER_LOG_LINES_PER_EVENT = 500;
6
9
  export const MAX_LEDGER_LOG_LINE_LENGTH = 2_000;
7
10
 
@@ -21,6 +24,15 @@ export function sanitizeLedgerPayload(value: unknown): unknown {
21
24
  return ledgerIngressRedactor.redact(value);
22
25
  }
23
26
 
27
+ /** Secret-redact a receipt payload without coupling its 10 MiB cell limit to the ledger. */
28
+ export function sanitizeRuntimeReceiptPayload(value: unknown): unknown {
29
+ assertRuntimeReceiptOutputWithinLimit({
30
+ output: value,
31
+ path: 'runtime receipt output',
32
+ });
33
+ return ledgerIngressRedactor.redact(value);
34
+ }
35
+
24
36
  export function sanitizeLedgerLogLines(value: unknown): string[] {
25
37
  if (!Array.isArray(value)) return [];
26
38
  return value
@@ -3,8 +3,10 @@ const encoder = typeof TextEncoder === 'undefined' ? null : new TextEncoder();
3
3
  export const TERMINAL_RUN_RESULT_MAX_BYTES = 2_500_000;
4
4
  export const CUSTOMER_OUTPUT_VALUE_MAX_BYTES = 1 * 1024 * 1024;
5
5
  export const CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 2 * 1024 * 1024;
6
- export const RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 2 * 1024 * 1024;
7
- export const RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 16 * 1024 * 1024;
6
+ /** Inline Postgres receipt contract: one serialized output is at most 10 MiB. */
7
+ export const RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
8
+ /** One completion request may contain multiple receipts up to 32 MiB total. */
9
+ export const RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
8
10
 
9
11
  export class OutputTooLargeError extends Error {
10
12
  readonly code = 'OUTPUT_TOO_LARGE';
@@ -1,6 +1,26 @@
1
1
  import { RECEIPT_STATUS_CODE } from './receipt-status';
2
2
  import { workReceiptFailureKindCode } from './work-receipts';
3
3
 
4
+ export function workReceiptRepairableFailurePredicateSql(
5
+ receiptTable: string,
6
+ ): string {
7
+ return `(
8
+ ${receiptTable}.failure_kind = ${workReceiptFailureKindCode('repairable')}::smallint
9
+ OR (
10
+ ${receiptTable}.failure_kind = ${workReceiptFailureKindCode('terminal')}::smallint
11
+ AND ${receiptTable}.error IS NOT NULL
12
+ AND lower(${receiptTable}.error) NOT LIKE '%billing cap%'
13
+ AND lower(${receiptTable}.error) NOT LIKE '%insufficient credits%'
14
+ AND lower(${receiptTable}.error) NOT LIKE '%monthly billing limit%'
15
+ AND (
16
+ ${receiptTable}.error ~* '(^|[^0-9])5[0-9]{2}([^0-9]|$)'
17
+ OR lower(${receiptTable}.error) LIKE '%transport failed calling%'
18
+ OR lower(${receiptTable}.error) LIKE '%runtime api call timed out%'
19
+ )
20
+ )
21
+ )`;
22
+ }
23
+
4
24
  export function workReceiptClaimableStatusCodes(input: {
5
25
  forceRefresh?: boolean;
6
26
  forceFailedRefresh?: boolean;
@@ -47,7 +67,7 @@ export function workReceiptClaimConflictPredicateSql(input: {
47
67
  )
48
68
  OR ${table}.status <> ${RECEIPT_STATUS_CODE.failed}::smallint
49
69
  OR (
50
- ${table}.failure_kind = ${workReceiptFailureKindCode('repairable')}::smallint
70
+ ${workReceiptRepairableFailurePredicateSql(table)}
51
71
  AND (
52
72
  ${table}.run_id IS DISTINCT FROM ${input.claimantRunIdSql}
53
73
  OR COALESCE(${table}.lease_owner_attempt, 0) < ${claimantRunAttemptSql}::integer
@@ -7,6 +7,7 @@ import {
7
7
  nextPlayRunLedgerTerminalSource,
8
8
  normalizePlayRunLedgerTerminalSource,
9
9
  } from './run-terminal-source';
10
+ import { MAX_LEDGER_LOG_LINES_PER_EVENT } from './ledger-safe-payload';
10
11
 
11
12
  export type PlayRunLedgerStatus =
12
13
  | 'queued'
@@ -84,6 +85,18 @@ export type PlayRunLedgerStepSnapshot = {
84
85
  progress?: PlayRunLedgerStepProgress | null;
85
86
  };
86
87
 
88
+ export type PlayRunLedgerDatasetSnapshot = {
89
+ datasetId: string;
90
+ path: string;
91
+ tableNamespace: string;
92
+ phase: 'registered' | 'available' | 'failed';
93
+ persistedRows: number;
94
+ succeededRows: number;
95
+ failedRows: number;
96
+ complete: boolean;
97
+ updatedAt: number;
98
+ };
99
+
87
100
  export type PlayRunLedgerSnapshot = {
88
101
  runId: string;
89
102
  playName?: string | null;
@@ -96,6 +109,8 @@ export type PlayRunLedgerSnapshot = {
96
109
  durationMs?: number | null;
97
110
  orderedStepIds: string[];
98
111
  stepsById: Record<string, PlayRunLedgerStepSnapshot>;
112
+ orderedDatasetIds: string[];
113
+ datasetsById: Record<string, PlayRunLedgerDatasetSnapshot>;
99
114
  /**
100
115
  * Bounded tail of the run's log lines (last {@link LOG_TAIL_LIMIT}).
101
116
  * Full retention lives in the Run Log Stream (Convex `playRunLogChunks`);
@@ -217,10 +232,34 @@ export type PlayRunLedgerEvent =
217
232
  * repeated identical lines while absorbing redundant re-sends.
218
233
  */
219
234
  channelOffset?: number;
235
+ /** Scheduler attempt that owns this positional log channel. */
236
+ producerAttempt?: number;
237
+ /**
238
+ * Terminal transport replay normally matches ctx.log lines after
239
+ * removing timestamps. Detached-attempt recovery uses exact matching so
240
+ * the same message from a later attempt remains a distinct occurrence.
241
+ */
242
+ transportDedupe?: 'semantic' | 'exact';
220
243
  /** Set by ingestion when this append crossed the retention cap. */
221
244
  logsTruncated?: boolean;
222
245
  })
223
246
  | (PlayRunLedgerBaseEvent & {
247
+ type: 'dataset.lifecycle';
248
+ datasetId: string;
249
+ path: string;
250
+ tableNamespace: string;
251
+ phase: 'registered' | 'available' | 'failed';
252
+ persistedRows?: number;
253
+ succeededRows?: number;
254
+ failedRows?: number;
255
+ complete?: boolean;
256
+ })
257
+ | (PlayRunLedgerBaseEvent & {
258
+ /**
259
+ * @deprecated Mixed-version deploy seam only. Owner: Play Runtime.
260
+ * Remove after 2026-07-19, once pre-dataset-lifecycle workers have
261
+ * exceeded the maximum run duration plus the seven-day drain window.
262
+ */
224
263
  type: 'sheet.summary.updated';
225
264
  tableNamespace: string;
226
265
  deltaCursor?: number;
@@ -241,6 +280,7 @@ export type PlayRunLedgerStatusPatch = {
241
280
  * (see `slicePositionalLogLines`), forwarded onto the `log.appended` event.
242
281
  */
243
282
  liveLogsChannelOffset?: number | null;
283
+ liveLogsProducerAttempt?: number | null;
244
284
  liveNodeProgress?: unknown;
245
285
  result?: unknown;
246
286
  };
@@ -401,6 +441,8 @@ export function createEmptyPlayRunLedgerSnapshot(input: {
401
441
  : null,
402
442
  orderedStepIds: [],
403
443
  stepsById: {},
444
+ orderedDatasetIds: [],
445
+ datasetsById: {},
404
446
  logTail: [],
405
447
  totalLogCount: 0,
406
448
  logsTruncated: false,
@@ -447,6 +489,26 @@ export function normalizePlayRunLedgerSnapshot(
447
489
  progress: rawProgress ? normalizeStepProgress(rawProgress) : null,
448
490
  };
449
491
  }
492
+ const rawDatasets = isRecord(value.datasetsById) ? value.datasetsById : {};
493
+ const datasetsById: Record<string, PlayRunLedgerDatasetSnapshot> = {};
494
+ for (const [datasetId, rawDataset] of Object.entries(rawDatasets)) {
495
+ if (!datasetId.trim() || !isRecord(rawDataset)) continue;
496
+ const phase =
497
+ rawDataset.phase === 'available' || rawDataset.phase === 'failed'
498
+ ? rawDataset.phase
499
+ : 'registered';
500
+ datasetsById[datasetId] = {
501
+ datasetId,
502
+ path: optionalString(rawDataset.path) ?? `datasets.${datasetId}`,
503
+ tableNamespace: optionalString(rawDataset.tableNamespace) ?? datasetId,
504
+ phase,
505
+ persistedRows: Math.max(0, finiteNumber(rawDataset.persistedRows) ?? 0),
506
+ succeededRows: Math.max(0, finiteNumber(rawDataset.succeededRows) ?? 0),
507
+ failedRows: Math.max(0, finiteNumber(rawDataset.failedRows) ?? 0),
508
+ complete: rawDataset.complete === true,
509
+ updatedAt: finiteNumber(rawDataset.updatedAt) ?? 0,
510
+ };
511
+ }
450
512
  const createdAt = finiteNumber(value.createdAt) ?? fallback.createdAt ?? null;
451
513
  const startedAt = finiteNumber(value.startedAt) ?? fallback.startedAt ?? null;
452
514
  const finishedAt =
@@ -484,6 +546,14 @@ export function normalizePlayRunLedgerSnapshot(
484
546
  : finiteNumber(value.durationMs),
485
547
  orderedStepIds: orderedStepIds.filter((stepId) => stepsById[stepId]),
486
548
  stepsById,
549
+ orderedDatasetIds: (Array.isArray(value.orderedDatasetIds)
550
+ ? value.orderedDatasetIds
551
+ : Object.keys(datasetsById)
552
+ ).filter(
553
+ (datasetId): datasetId is string =>
554
+ typeof datasetId === 'string' && Boolean(datasetsById[datasetId]),
555
+ ),
556
+ datasetsById,
487
557
  logTail: logTail.slice(-LOG_TAIL_LIMIT),
488
558
  // Snapshots persisted before totalLogCount existed only know the retained
489
559
  // tail, so the best lower bound for the cumulative count is the tail size.
@@ -1269,6 +1339,34 @@ export function reducePlayRunLedgerEvent(
1269
1339
  : null,
1270
1340
  });
1271
1341
  }
1342
+ case 'dataset.lifecycle': {
1343
+ const current = base.datasetsById[event.datasetId];
1344
+ const next: PlayRunLedgerDatasetSnapshot = {
1345
+ datasetId: event.datasetId,
1346
+ path: event.path,
1347
+ tableNamespace: event.tableNamespace,
1348
+ phase: event.phase,
1349
+ persistedRows: Math.max(
1350
+ current?.persistedRows ?? 0,
1351
+ event.persistedRows ?? 0,
1352
+ ),
1353
+ succeededRows: Math.max(
1354
+ current?.succeededRows ?? 0,
1355
+ event.succeededRows ?? 0,
1356
+ ),
1357
+ failedRows: Math.max(current?.failedRows ?? 0, event.failedRows ?? 0),
1358
+ complete: current?.complete === true || event.complete === true,
1359
+ updatedAt: occurredAt,
1360
+ };
1361
+ return withTiming({
1362
+ ...base,
1363
+ orderedDatasetIds: current
1364
+ ? base.orderedDatasetIds
1365
+ : [...base.orderedDatasetIds, event.datasetId],
1366
+ datasetsById: { ...base.datasetsById, [event.datasetId]: next },
1367
+ resultTableNamespace: base.resultTableNamespace ?? event.tableNamespace,
1368
+ });
1369
+ }
1272
1370
  case 'sheet.summary.updated':
1273
1371
  return withTiming({
1274
1372
  ...base,
@@ -1376,6 +1474,8 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
1376
1474
  source?: PlayRunLedgerEventSource;
1377
1475
  occurredAt?: number;
1378
1476
  channelOffset?: number | null;
1477
+ producerAttempt?: number | null;
1478
+ transportDedupe?: 'semantic' | 'exact';
1379
1479
  includeLogAppend?: boolean;
1380
1480
  }): PlayRunLedgerEvent[] {
1381
1481
  const source = input.source ?? 'worker';
@@ -1392,16 +1492,31 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
1392
1492
  if (newLines.length === 0 || input.includeLogAppend === false) {
1393
1493
  return [];
1394
1494
  }
1395
- return [
1396
- {
1495
+ const events: PlayRunLedgerEvent[] = [];
1496
+ for (
1497
+ let index = 0;
1498
+ index < newLines.length;
1499
+ index += MAX_LEDGER_LOG_LINES_PER_EVENT
1500
+ ) {
1501
+ events.push({
1397
1502
  type: 'log.appended',
1398
1503
  runId: input.runId,
1399
1504
  source,
1400
1505
  occurredAt,
1401
- lines: newLines,
1402
- ...(positional ? { channelOffset: input.channelOffset! } : {}),
1403
- },
1404
- ];
1506
+ lines: newLines.slice(index, index + MAX_LEDGER_LOG_LINES_PER_EVENT),
1507
+ ...(positional ? { channelOffset: input.channelOffset! + index } : {}),
1508
+ ...(positional &&
1509
+ typeof input.producerAttempt === 'number' &&
1510
+ Number.isFinite(input.producerAttempt) &&
1511
+ input.producerAttempt >= 0
1512
+ ? { producerAttempt: Math.trunc(input.producerAttempt) }
1513
+ : {}),
1514
+ ...(input.transportDedupe
1515
+ ? { transportDedupe: input.transportDedupe }
1516
+ : {}),
1517
+ });
1518
+ }
1519
+ return events;
1405
1520
  }
1406
1521
 
1407
1522
  /**
@@ -1419,6 +1534,8 @@ export function buildTerminalLogReplayEvents(input: {
1419
1534
  source?: PlayRunLedgerEventSource;
1420
1535
  occurredAt?: number;
1421
1536
  liveLogTotalCount?: number;
1537
+ dedupeMode?: 'semantic' | 'exact';
1538
+ producerAttempt?: number;
1422
1539
  }): PlayRunLedgerEvent[] {
1423
1540
  const channelOffset = terminalLogReplayChannelOffset({
1424
1541
  liveLogTotalCount: input.liveLogTotalCount,
@@ -1430,6 +1547,10 @@ export function buildTerminalLogReplayEvents(input: {
1430
1547
  source: 'coordinator',
1431
1548
  occurredAt: input.occurredAt,
1432
1549
  channelOffset,
1550
+ producerAttempt: input.producerAttempt,
1551
+ transportDedupe:
1552
+ input.dedupeMode ??
1553
+ (typeof input.producerAttempt === 'number' ? 'exact' : undefined),
1433
1554
  });
1434
1555
  }
1435
1556
 
@@ -1466,6 +1587,7 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
1466
1587
  source,
1467
1588
  occurredAt: checkpointAt,
1468
1589
  channelOffset: patch.liveLogsChannelOffset ?? null,
1590
+ producerAttempt: patch.liveLogsProducerAttempt ?? null,
1469
1591
  })
1470
1592
  : [];
1471
1593
  const earliestStartedAt =
@@ -4,6 +4,7 @@ import {
4
4
  normalizePlayRunLedgerSnapshot,
5
5
  summarizeRunRowOutcomes,
6
6
  type PlayRunLedgerSnapshot,
7
+ type PlayRunLedgerDatasetSnapshot,
7
8
  type PlayRunLedgerStepSnapshot,
8
9
  type PlayRunRowOutcomeSummary,
9
10
  } from './run-ledger';
@@ -76,6 +77,7 @@ export type PlayRunLiveSnapshot = {
76
77
  logsTruncated?: boolean;
77
78
  activeArtifactTableNamespace: string | null;
78
79
  resultTableNamespace: string | null;
80
+ datasets?: PlayRunLedgerDatasetSnapshot[];
79
81
  nodeStates: PlayRunStreamNodeState[];
80
82
  activeNodeId: string | null;
81
83
  /**
@@ -212,6 +214,11 @@ function buildSnapshotFromLedger(
212
214
  ...(snapshot.logsTruncated ? { logsTruncated: true } : {}),
213
215
  activeArtifactTableNamespace: snapshot.activeArtifactTableNamespace ?? null,
214
216
  resultTableNamespace: snapshot.resultTableNamespace ?? null,
217
+ datasets: snapshot.orderedDatasetIds
218
+ .map((datasetId) => snapshot.datasetsById[datasetId])
219
+ .filter((dataset): dataset is PlayRunLedgerDatasetSnapshot =>
220
+ Boolean(dataset),
221
+ ),
215
222
  nodeStates,
216
223
  activeNodeId: snapshot.activeStepId ?? null,
217
224
  ...(rowOutcomes ? { rowOutcomes } : {}),
@@ -344,13 +344,17 @@ export async function stageDaytonaRunnerPayload(input: {
344
344
  const outputPath = `${workDir}/deepline-play-output-${randomUUID()}.log`;
345
345
  const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
346
346
  const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
347
+ const runnerTraceEnv =
348
+ process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
349
+ ? 'DEEPLINE_RUNTIME_RECEIPT_TRACE=1 '
350
+ : '';
347
351
  const runnerCommand = `${nodeMaterializePayloadCommand({
348
352
  envelopePath,
349
353
  runnerPath,
350
354
  configPath,
351
355
  artifactCodePath,
352
356
  crashPusherPath,
353
- })} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
357
+ })} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
354
358
  // Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
355
359
  // pushes the parsed (or synthesized) terminal to the gateway so the parked
356
360
  // worker wakes within seconds of ANY runner death — process.exit abuse, OOM
@@ -122,19 +122,34 @@ export async function inspectDetachedDaytonaRunner(input: {
122
122
  cmdId: string;
123
123
  outputPath: string;
124
124
  exitCodePath: string;
125
- }): Promise<{ exitCode: number | null; result: PlayRunnerResult | null }> {
125
+ }): Promise<{
126
+ exitCode: number | null;
127
+ result: PlayRunnerResult | null;
128
+ diagnosis: {
129
+ stage:
130
+ | 'salvaged'
131
+ | 'result_missing'
132
+ | 'command_inspection_failed'
133
+ | 'sandbox_lookup_failed';
134
+ detail: string | null;
135
+ };
136
+ }> {
126
137
  try {
127
138
  const { clientOptions } = loadDaytonaRequiredConfig();
128
139
  const daytona = daytonaSdkClientFactory.createFull(clientOptions);
129
140
  const sandbox = (await daytona.get(input.sandboxId)) as DaytonaSandbox;
130
141
  let exitCode: number | null = null;
142
+ let commandInspectionError: string | null = null;
131
143
  try {
132
144
  const command = await sandbox.process.getSessionCommand(
133
145
  input.sessionId,
134
146
  input.cmdId,
135
147
  );
136
- exitCode = typeof command?.exitCode === 'number' ? command.exitCode : null;
148
+ exitCode =
149
+ typeof command?.exitCode === 'number' ? command.exitCode : null;
137
150
  } catch (error) {
151
+ commandInspectionError =
152
+ error instanceof Error ? error.message : String(error);
138
153
  console.warn('[play-runner.daytona.detached_inspect_command_failed]', {
139
154
  sandboxId: input.sandboxId,
140
155
  cmdId: input.cmdId,
@@ -148,7 +163,16 @@ export async function inspectDetachedDaytonaRunner(input: {
148
163
  timeoutMs: 10_000,
149
164
  });
150
165
  if (!recovered.execution) {
151
- return { exitCode, result: null };
166
+ return {
167
+ exitCode,
168
+ result: null,
169
+ diagnosis: commandInspectionError
170
+ ? {
171
+ stage: 'command_inspection_failed',
172
+ detail: commandInspectionError,
173
+ }
174
+ : { stage: 'result_missing', detail: null },
175
+ };
152
176
  }
153
177
  const result =
154
178
  findPlayRunnerResult(parsePlayRunnerEvents(recovered.execution.result)) ??
@@ -156,14 +180,22 @@ export async function inspectDetachedDaytonaRunner(input: {
156
180
  return {
157
181
  exitCode: exitCode ?? recovered.execution.exitCode,
158
182
  result,
183
+ diagnosis: result
184
+ ? { stage: 'salvaged', detail: null }
185
+ : { stage: 'result_missing', detail: null },
159
186
  };
160
187
  } catch (error) {
188
+ const detail = error instanceof Error ? error.message : String(error);
161
189
  console.warn('[play-runner.daytona.detached_inspect_failed]', {
162
190
  sandboxId: input.sandboxId,
163
191
  cmdId: input.cmdId,
164
- error: error instanceof Error ? error.message : String(error),
192
+ error: detail,
165
193
  });
166
- return { exitCode: null, result: null };
194
+ return {
195
+ exitCode: null,
196
+ result: null,
197
+ diagnosis: { stage: 'sandbox_lookup_failed', detail },
198
+ };
167
199
  }
168
200
  }
169
201