deepline 0.1.210 → 0.1.212
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +203 -12
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +23 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +15 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts +90 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +162 -100
- package/dist/bundling-sources/sdk/src/client.ts +118 -3
- package/dist/bundling-sources/sdk/src/play.ts +2 -0
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +253 -30
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +365 -103
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts +10 -4
- package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +45 -8
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +14 -2
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +4 -2
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +21 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +128 -6
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +5 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +37 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +19 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +63 -7
- package/dist/bundling-sources/shared_libs/play-runtime/secret-capability.ts +18 -4
- package/dist/bundling-sources/shared_libs/play-runtime/single-flight.ts +31 -0
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +12 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +29 -9
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +22 -1
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +79 -1
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +41 -5
- package/dist/cli/index.js +728 -239
- package/dist/cli/index.mjs +728 -239
- package/dist/index.d.mts +93 -7
- package/dist/index.d.ts +93 -7
- package/dist/index.js +245 -46
- package/dist/index.mjs +245 -46
- package/dist/plays/bundle-play-file.mjs +65 -4
- package/package.json +1 -1
|
@@ -380,6 +380,18 @@ export interface MapExecutionScope {
|
|
|
380
380
|
}
|
|
381
381
|
|
|
382
382
|
export type PlayExecutionEvent =
|
|
383
|
+
| {
|
|
384
|
+
type: 'dataset.lifecycle';
|
|
385
|
+
datasetId: string;
|
|
386
|
+
path: string;
|
|
387
|
+
tableNamespace: string;
|
|
388
|
+
phase: 'registered' | 'available' | 'failed';
|
|
389
|
+
persistedRows?: number;
|
|
390
|
+
succeededRows?: number;
|
|
391
|
+
failedRows?: number;
|
|
392
|
+
complete?: boolean;
|
|
393
|
+
at: number;
|
|
394
|
+
}
|
|
383
395
|
| {
|
|
384
396
|
type: 'map.preparing';
|
|
385
397
|
mapInvocationId: string;
|
|
@@ -1,10 +1,16 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
normalizePlayNameForSheet,
|
|
3
|
+
normalizeTableNamespace,
|
|
4
|
+
sha256Hex,
|
|
5
|
+
} from '../plays/row-identity';
|
|
2
6
|
|
|
3
7
|
export function createRuntimeDatasetId(
|
|
4
8
|
playName: string,
|
|
5
9
|
tableNamespace: string,
|
|
6
10
|
): string {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
11
|
+
const normalizedPlayName = normalizePlayNameForSheet(playName);
|
|
12
|
+
const normalizedTableNamespace = normalizeTableNamespace(tableNamespace);
|
|
13
|
+
return sha256Hex(
|
|
14
|
+
`${normalizedPlayName}:${normalizedTableNamespace}`,
|
|
15
|
+
).slice(0, 40);
|
|
10
16
|
}
|
|
@@ -56,6 +56,7 @@ export type CreateDbSessionRequest = {
|
|
|
56
56
|
|
|
57
57
|
export type CreateDbSessionResponse = {
|
|
58
58
|
sessionId: string;
|
|
59
|
+
executionRole?: string;
|
|
59
60
|
postgresUrl?: string;
|
|
60
61
|
encryptedPostgresUrl?: EncryptedPostgresUrl;
|
|
61
62
|
postgres?: {
|
|
@@ -398,6 +399,14 @@ export const createDbSessionResponseSchema =
|
|
|
398
399
|
}
|
|
399
400
|
return {
|
|
400
401
|
sessionId: parseString(value.sessionId, 'sessionId'),
|
|
402
|
+
...(value.executionRole !== undefined
|
|
403
|
+
? {
|
|
404
|
+
executionRole: parseOptionalString(
|
|
405
|
+
value.executionRole,
|
|
406
|
+
'executionRole',
|
|
407
|
+
),
|
|
408
|
+
}
|
|
409
|
+
: {}),
|
|
401
410
|
...(value.postgresUrl !== undefined
|
|
402
411
|
? { postgresUrl: parseOptionalString(value.postgresUrl, 'postgresUrl') }
|
|
403
412
|
: {}),
|
|
@@ -243,9 +243,11 @@ async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
|
|
|
243
243
|
rejectLeaseLost(error as RuntimeReceiptLeaseLostError),
|
|
244
244
|
});
|
|
245
245
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
246
|
+
// markRunning is the authoritative pre-dispatch ownership fence. Renewal is
|
|
247
|
+
// only needed when the lease approaches expiry; an immediate read/renewal
|
|
248
|
+
// adds a round trip to every short tool call without extending ownership.
|
|
249
|
+
// Completion remains conditional on this lease, and long work still renews.
|
|
250
|
+
supervisor.start();
|
|
249
251
|
try {
|
|
250
252
|
return await Promise.race([input.execute(), leaseLost]);
|
|
251
253
|
} finally {
|
|
@@ -294,6 +296,7 @@ export async function waitForCompletedRuntimeReceipts(input: {
|
|
|
294
296
|
maxAttempts?: number;
|
|
295
297
|
delayMs?: number;
|
|
296
298
|
abortSignal?: AbortSignal;
|
|
299
|
+
log?: (message: string) => void;
|
|
297
300
|
}): Promise<{
|
|
298
301
|
completed: Map<string, RuntimeStepReceipt>;
|
|
299
302
|
failed: Map<string, Error>;
|
|
@@ -326,6 +329,20 @@ export async function waitForCompletedRuntimeReceipts(input: {
|
|
|
326
329
|
}
|
|
327
330
|
throw error;
|
|
328
331
|
}
|
|
332
|
+
const statuses = [...receipts.values()].reduce<Record<string, number>>(
|
|
333
|
+
(counts, receipt) => {
|
|
334
|
+
counts[receipt.status] = (counts[receipt.status] ?? 0) + 1;
|
|
335
|
+
return counts;
|
|
336
|
+
},
|
|
337
|
+
{},
|
|
338
|
+
);
|
|
339
|
+
input.log?.(
|
|
340
|
+
`[perf] runtime-receipt-wait attempt=${attempt} requested=${pending.size} indexed=${receipts.size} statuses=${
|
|
341
|
+
Object.entries(statuses)
|
|
342
|
+
.map(([status, count]) => `${status}:${count}`)
|
|
343
|
+
.join(',') || 'none'
|
|
344
|
+
}`,
|
|
345
|
+
);
|
|
329
346
|
for (const key of [...pending]) {
|
|
330
347
|
const receipt = receipts.get(key);
|
|
331
348
|
if (receipt?.status === 'completed' || receipt?.status === 'skipped') {
|
|
@@ -343,6 +360,9 @@ export async function waitForCompletedRuntimeReceipts(input: {
|
|
|
343
360
|
pending.delete(key);
|
|
344
361
|
}
|
|
345
362
|
}
|
|
363
|
+
input.log?.(
|
|
364
|
+
`[perf] runtime-receipt-wait attempt=${attempt} remaining=${pending.size}`,
|
|
365
|
+
);
|
|
346
366
|
}
|
|
347
367
|
|
|
348
368
|
return { completed, failed, timedOut: pending };
|
|
@@ -374,6 +394,16 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
374
394
|
log: (message: string) => void;
|
|
375
395
|
execute: (context: { leaseId: string | null }) => Promise<T>;
|
|
376
396
|
}): Promise<T> {
|
|
397
|
+
const lifecycleStartedAt = Date.now();
|
|
398
|
+
const logPhase = (phase: string, startedAt: number): void => {
|
|
399
|
+
// Keep this at the semantic receipt layer. Transport timing below this
|
|
400
|
+
// function explains HTTP cost; these markers expose gaps before/after the
|
|
401
|
+
// store calls and prove whether user code or receipt orchestration owns the
|
|
402
|
+
// elapsed time. Do not log receipt keys because they encode tool inputs.
|
|
403
|
+
input.log(
|
|
404
|
+
`[perf] durable receipt operation=${input.operation} id=${input.id} phase=${phase} elapsed_ms=${Date.now() - startedAt} lifecycle_ms=${Date.now() - lifecycleStartedAt}`,
|
|
405
|
+
);
|
|
406
|
+
};
|
|
377
407
|
if (!input.store.enabled) {
|
|
378
408
|
return await input.execute({ leaseId: null });
|
|
379
409
|
}
|
|
@@ -382,9 +412,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
382
412
|
receipt: RuntimeStepReceipt,
|
|
383
413
|
source: DurableReceiptRecoverySource = 'cache',
|
|
384
414
|
): Promise<T> => {
|
|
385
|
-
input.log(
|
|
386
|
-
`ctx.${input.operation}(${input.id}): recovered result from receipt`,
|
|
387
|
-
);
|
|
415
|
+
input.log(`ctx.${input.operation}(${input.id}): reused completed work`);
|
|
388
416
|
if (receipt.output === undefined) {
|
|
389
417
|
return receipt.output as T;
|
|
390
418
|
}
|
|
@@ -417,7 +445,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
417
445
|
}
|
|
418
446
|
if (reclaimed?.status === 'failed') {
|
|
419
447
|
throw new Error(
|
|
420
|
-
`ctx.${input.operation}(${input.id}):
|
|
448
|
+
`ctx.${input.operation}(${input.id}): previous execution failed and cannot be reused: ${reclaimed.error ?? 'unknown error'}.`,
|
|
421
449
|
);
|
|
422
450
|
}
|
|
423
451
|
if (
|
|
@@ -479,16 +507,18 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
479
507
|
receipt: RuntimeStepReceipt,
|
|
480
508
|
): Promise<never> => {
|
|
481
509
|
throw new Error(
|
|
482
|
-
`ctx.${input.operation}(${input.id}):
|
|
510
|
+
`ctx.${input.operation}(${input.id}): previous execution failed and cannot be reused: ${receipt.error ?? 'unknown error'}.`,
|
|
483
511
|
);
|
|
484
512
|
};
|
|
485
513
|
|
|
514
|
+
const claimStartedAt = Date.now();
|
|
486
515
|
const claimed = await input.store.claim(
|
|
487
516
|
input.receiptKey,
|
|
488
517
|
input.runId,
|
|
489
518
|
input.reclaimRunning === true || input.force === true,
|
|
490
519
|
input.force === true,
|
|
491
520
|
);
|
|
521
|
+
logPhase('claim', claimStartedAt);
|
|
492
522
|
if (
|
|
493
523
|
input.force !== true &&
|
|
494
524
|
(claimed?.status === 'completed' || claimed?.status === 'skipped')
|
|
@@ -525,11 +555,13 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
525
555
|
let result: T;
|
|
526
556
|
try {
|
|
527
557
|
if (input.markRunningBeforeExecute !== false && input.store.markRunning) {
|
|
558
|
+
const markRunningStartedAt = Date.now();
|
|
528
559
|
const running = await input.store.markRunning(
|
|
529
560
|
input.receiptKey,
|
|
530
561
|
input.runId,
|
|
531
562
|
ownedLeaseId,
|
|
532
563
|
);
|
|
564
|
+
logPhase('mark_running', markRunningStartedAt);
|
|
533
565
|
if (!running || running.status !== 'running') {
|
|
534
566
|
throw new RuntimeReceiptLeaseLostError({
|
|
535
567
|
receiptKey: input.receiptKey,
|
|
@@ -540,6 +572,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
540
572
|
ownedLeaseId = running.leaseId ?? ownedLeaseId;
|
|
541
573
|
ownedLeaseExpiresAt = running.leaseExpiresAt ?? ownedLeaseExpiresAt;
|
|
542
574
|
}
|
|
575
|
+
const executeStartedAt = Date.now();
|
|
543
576
|
const executed =
|
|
544
577
|
ownedLeaseId && input.store.heartbeat
|
|
545
578
|
? await executeWithDurableRuntimeReceiptHeartbeat({
|
|
@@ -561,6 +594,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
561
594
|
execute: () => input.execute({ leaseId: ownedLeaseId }),
|
|
562
595
|
})
|
|
563
596
|
: await input.execute({ leaseId: ownedLeaseId });
|
|
597
|
+
logPhase('execute', executeStartedAt);
|
|
564
598
|
result = input.onClaimedResult
|
|
565
599
|
? input.onClaimedResult(executed, input.receiptKey)
|
|
566
600
|
: executed;
|
|
@@ -603,12 +637,14 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
603
637
|
throw error;
|
|
604
638
|
}
|
|
605
639
|
|
|
640
|
+
const completeStartedAt = Date.now();
|
|
606
641
|
const completed = await input.store.complete(
|
|
607
642
|
input.receiptKey,
|
|
608
643
|
input.runId,
|
|
609
644
|
isToolExecuteResult(result) ? serializeToolExecuteResult(result) : result,
|
|
610
645
|
ownedLeaseId,
|
|
611
646
|
);
|
|
647
|
+
logPhase('complete', completeStartedAt);
|
|
612
648
|
if (
|
|
613
649
|
!completed ||
|
|
614
650
|
(completed.status !== 'completed' && completed.status !== 'skipped')
|
|
@@ -633,6 +669,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
633
669
|
completed &&
|
|
634
670
|
(completed.status === 'completed' || completed.status === 'skipped')
|
|
635
671
|
) {
|
|
672
|
+
logPhase('total', lifecycleStartedAt);
|
|
636
673
|
return await recoverCompletedReceipt(completed, 'owner');
|
|
637
674
|
}
|
|
638
675
|
if (input.store.canPersistCompletion) {
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { createSecretRedactionContext } from './secret-redaction';
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
assertRuntimeReceiptOutputWithinLimit,
|
|
4
|
+
TERMINAL_RUN_RESULT_MAX_BYTES,
|
|
5
|
+
} from './output-size-limits';
|
|
3
6
|
|
|
4
|
-
export const MAX_LEDGER_RESULT_PAYLOAD_BYTES =
|
|
7
|
+
export const MAX_LEDGER_RESULT_PAYLOAD_BYTES = TERMINAL_RUN_RESULT_MAX_BYTES;
|
|
5
8
|
export const MAX_LEDGER_LOG_LINES_PER_EVENT = 500;
|
|
6
9
|
export const MAX_LEDGER_LOG_LINE_LENGTH = 2_000;
|
|
7
10
|
|
|
@@ -21,6 +24,15 @@ export function sanitizeLedgerPayload(value: unknown): unknown {
|
|
|
21
24
|
return ledgerIngressRedactor.redact(value);
|
|
22
25
|
}
|
|
23
26
|
|
|
27
|
+
/** Secret-redact a receipt payload without coupling its 10 MiB cell limit to the ledger. */
|
|
28
|
+
export function sanitizeRuntimeReceiptPayload(value: unknown): unknown {
|
|
29
|
+
assertRuntimeReceiptOutputWithinLimit({
|
|
30
|
+
output: value,
|
|
31
|
+
path: 'runtime receipt output',
|
|
32
|
+
});
|
|
33
|
+
return ledgerIngressRedactor.redact(value);
|
|
34
|
+
}
|
|
35
|
+
|
|
24
36
|
export function sanitizeLedgerLogLines(value: unknown): string[] {
|
|
25
37
|
if (!Array.isArray(value)) return [];
|
|
26
38
|
return value
|
|
@@ -3,8 +3,10 @@ const encoder = typeof TextEncoder === 'undefined' ? null : new TextEncoder();
|
|
|
3
3
|
export const TERMINAL_RUN_RESULT_MAX_BYTES = 2_500_000;
|
|
4
4
|
export const CUSTOMER_OUTPUT_VALUE_MAX_BYTES = 1 * 1024 * 1024;
|
|
5
5
|
export const CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 2 * 1024 * 1024;
|
|
6
|
-
|
|
7
|
-
export const
|
|
6
|
+
/** Inline Postgres receipt contract: one serialized output is at most 10 MiB. */
|
|
7
|
+
export const RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
|
|
8
|
+
/** One completion request may contain multiple receipts up to 32 MiB total. */
|
|
9
|
+
export const RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
|
|
8
10
|
|
|
9
11
|
export class OutputTooLargeError extends Error {
|
|
10
12
|
readonly code = 'OUTPUT_TOO_LARGE';
|
|
@@ -1,6 +1,26 @@
|
|
|
1
1
|
import { RECEIPT_STATUS_CODE } from './receipt-status';
|
|
2
2
|
import { workReceiptFailureKindCode } from './work-receipts';
|
|
3
3
|
|
|
4
|
+
export function workReceiptRepairableFailurePredicateSql(
|
|
5
|
+
receiptTable: string,
|
|
6
|
+
): string {
|
|
7
|
+
return `(
|
|
8
|
+
${receiptTable}.failure_kind = ${workReceiptFailureKindCode('repairable')}::smallint
|
|
9
|
+
OR (
|
|
10
|
+
${receiptTable}.failure_kind = ${workReceiptFailureKindCode('terminal')}::smallint
|
|
11
|
+
AND ${receiptTable}.error IS NOT NULL
|
|
12
|
+
AND lower(${receiptTable}.error) NOT LIKE '%billing cap%'
|
|
13
|
+
AND lower(${receiptTable}.error) NOT LIKE '%insufficient credits%'
|
|
14
|
+
AND lower(${receiptTable}.error) NOT LIKE '%monthly billing limit%'
|
|
15
|
+
AND (
|
|
16
|
+
${receiptTable}.error ~* '(^|[^0-9])5[0-9]{2}([^0-9]|$)'
|
|
17
|
+
OR lower(${receiptTable}.error) LIKE '%transport failed calling%'
|
|
18
|
+
OR lower(${receiptTable}.error) LIKE '%runtime api call timed out%'
|
|
19
|
+
)
|
|
20
|
+
)
|
|
21
|
+
)`;
|
|
22
|
+
}
|
|
23
|
+
|
|
4
24
|
export function workReceiptClaimableStatusCodes(input: {
|
|
5
25
|
forceRefresh?: boolean;
|
|
6
26
|
forceFailedRefresh?: boolean;
|
|
@@ -47,7 +67,7 @@ export function workReceiptClaimConflictPredicateSql(input: {
|
|
|
47
67
|
)
|
|
48
68
|
OR ${table}.status <> ${RECEIPT_STATUS_CODE.failed}::smallint
|
|
49
69
|
OR (
|
|
50
|
-
${table
|
|
70
|
+
${workReceiptRepairableFailurePredicateSql(table)}
|
|
51
71
|
AND (
|
|
52
72
|
${table}.run_id IS DISTINCT FROM ${input.claimantRunIdSql}
|
|
53
73
|
OR COALESCE(${table}.lease_owner_attempt, 0) < ${claimantRunAttemptSql}::integer
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
nextPlayRunLedgerTerminalSource,
|
|
8
8
|
normalizePlayRunLedgerTerminalSource,
|
|
9
9
|
} from './run-terminal-source';
|
|
10
|
+
import { MAX_LEDGER_LOG_LINES_PER_EVENT } from './ledger-safe-payload';
|
|
10
11
|
|
|
11
12
|
export type PlayRunLedgerStatus =
|
|
12
13
|
| 'queued'
|
|
@@ -84,6 +85,18 @@ export type PlayRunLedgerStepSnapshot = {
|
|
|
84
85
|
progress?: PlayRunLedgerStepProgress | null;
|
|
85
86
|
};
|
|
86
87
|
|
|
88
|
+
export type PlayRunLedgerDatasetSnapshot = {
|
|
89
|
+
datasetId: string;
|
|
90
|
+
path: string;
|
|
91
|
+
tableNamespace: string;
|
|
92
|
+
phase: 'registered' | 'available' | 'failed';
|
|
93
|
+
persistedRows: number;
|
|
94
|
+
succeededRows: number;
|
|
95
|
+
failedRows: number;
|
|
96
|
+
complete: boolean;
|
|
97
|
+
updatedAt: number;
|
|
98
|
+
};
|
|
99
|
+
|
|
87
100
|
export type PlayRunLedgerSnapshot = {
|
|
88
101
|
runId: string;
|
|
89
102
|
playName?: string | null;
|
|
@@ -96,6 +109,8 @@ export type PlayRunLedgerSnapshot = {
|
|
|
96
109
|
durationMs?: number | null;
|
|
97
110
|
orderedStepIds: string[];
|
|
98
111
|
stepsById: Record<string, PlayRunLedgerStepSnapshot>;
|
|
112
|
+
orderedDatasetIds: string[];
|
|
113
|
+
datasetsById: Record<string, PlayRunLedgerDatasetSnapshot>;
|
|
99
114
|
/**
|
|
100
115
|
* Bounded tail of the run's log lines (last {@link LOG_TAIL_LIMIT}).
|
|
101
116
|
* Full retention lives in the Run Log Stream (Convex `playRunLogChunks`);
|
|
@@ -217,10 +232,34 @@ export type PlayRunLedgerEvent =
|
|
|
217
232
|
* repeated identical lines while absorbing redundant re-sends.
|
|
218
233
|
*/
|
|
219
234
|
channelOffset?: number;
|
|
235
|
+
/** Scheduler attempt that owns this positional log channel. */
|
|
236
|
+
producerAttempt?: number;
|
|
237
|
+
/**
|
|
238
|
+
* Terminal transport replay normally matches ctx.log lines after
|
|
239
|
+
* removing timestamps. Detached-attempt recovery uses exact matching so
|
|
240
|
+
* the same message from a later attempt remains a distinct occurrence.
|
|
241
|
+
*/
|
|
242
|
+
transportDedupe?: 'semantic' | 'exact';
|
|
220
243
|
/** Set by ingestion when this append crossed the retention cap. */
|
|
221
244
|
logsTruncated?: boolean;
|
|
222
245
|
})
|
|
223
246
|
| (PlayRunLedgerBaseEvent & {
|
|
247
|
+
type: 'dataset.lifecycle';
|
|
248
|
+
datasetId: string;
|
|
249
|
+
path: string;
|
|
250
|
+
tableNamespace: string;
|
|
251
|
+
phase: 'registered' | 'available' | 'failed';
|
|
252
|
+
persistedRows?: number;
|
|
253
|
+
succeededRows?: number;
|
|
254
|
+
failedRows?: number;
|
|
255
|
+
complete?: boolean;
|
|
256
|
+
})
|
|
257
|
+
| (PlayRunLedgerBaseEvent & {
|
|
258
|
+
/**
|
|
259
|
+
* @deprecated Mixed-version deploy seam only. Owner: Play Runtime.
|
|
260
|
+
* Remove after 2026-07-19, once pre-dataset-lifecycle workers have
|
|
261
|
+
* exceeded the maximum run duration plus the seven-day drain window.
|
|
262
|
+
*/
|
|
224
263
|
type: 'sheet.summary.updated';
|
|
225
264
|
tableNamespace: string;
|
|
226
265
|
deltaCursor?: number;
|
|
@@ -241,6 +280,7 @@ export type PlayRunLedgerStatusPatch = {
|
|
|
241
280
|
* (see `slicePositionalLogLines`), forwarded onto the `log.appended` event.
|
|
242
281
|
*/
|
|
243
282
|
liveLogsChannelOffset?: number | null;
|
|
283
|
+
liveLogsProducerAttempt?: number | null;
|
|
244
284
|
liveNodeProgress?: unknown;
|
|
245
285
|
result?: unknown;
|
|
246
286
|
};
|
|
@@ -401,6 +441,8 @@ export function createEmptyPlayRunLedgerSnapshot(input: {
|
|
|
401
441
|
: null,
|
|
402
442
|
orderedStepIds: [],
|
|
403
443
|
stepsById: {},
|
|
444
|
+
orderedDatasetIds: [],
|
|
445
|
+
datasetsById: {},
|
|
404
446
|
logTail: [],
|
|
405
447
|
totalLogCount: 0,
|
|
406
448
|
logsTruncated: false,
|
|
@@ -447,6 +489,26 @@ export function normalizePlayRunLedgerSnapshot(
|
|
|
447
489
|
progress: rawProgress ? normalizeStepProgress(rawProgress) : null,
|
|
448
490
|
};
|
|
449
491
|
}
|
|
492
|
+
const rawDatasets = isRecord(value.datasetsById) ? value.datasetsById : {};
|
|
493
|
+
const datasetsById: Record<string, PlayRunLedgerDatasetSnapshot> = {};
|
|
494
|
+
for (const [datasetId, rawDataset] of Object.entries(rawDatasets)) {
|
|
495
|
+
if (!datasetId.trim() || !isRecord(rawDataset)) continue;
|
|
496
|
+
const phase =
|
|
497
|
+
rawDataset.phase === 'available' || rawDataset.phase === 'failed'
|
|
498
|
+
? rawDataset.phase
|
|
499
|
+
: 'registered';
|
|
500
|
+
datasetsById[datasetId] = {
|
|
501
|
+
datasetId,
|
|
502
|
+
path: optionalString(rawDataset.path) ?? `datasets.${datasetId}`,
|
|
503
|
+
tableNamespace: optionalString(rawDataset.tableNamespace) ?? datasetId,
|
|
504
|
+
phase,
|
|
505
|
+
persistedRows: Math.max(0, finiteNumber(rawDataset.persistedRows) ?? 0),
|
|
506
|
+
succeededRows: Math.max(0, finiteNumber(rawDataset.succeededRows) ?? 0),
|
|
507
|
+
failedRows: Math.max(0, finiteNumber(rawDataset.failedRows) ?? 0),
|
|
508
|
+
complete: rawDataset.complete === true,
|
|
509
|
+
updatedAt: finiteNumber(rawDataset.updatedAt) ?? 0,
|
|
510
|
+
};
|
|
511
|
+
}
|
|
450
512
|
const createdAt = finiteNumber(value.createdAt) ?? fallback.createdAt ?? null;
|
|
451
513
|
const startedAt = finiteNumber(value.startedAt) ?? fallback.startedAt ?? null;
|
|
452
514
|
const finishedAt =
|
|
@@ -484,6 +546,14 @@ export function normalizePlayRunLedgerSnapshot(
|
|
|
484
546
|
: finiteNumber(value.durationMs),
|
|
485
547
|
orderedStepIds: orderedStepIds.filter((stepId) => stepsById[stepId]),
|
|
486
548
|
stepsById,
|
|
549
|
+
orderedDatasetIds: (Array.isArray(value.orderedDatasetIds)
|
|
550
|
+
? value.orderedDatasetIds
|
|
551
|
+
: Object.keys(datasetsById)
|
|
552
|
+
).filter(
|
|
553
|
+
(datasetId): datasetId is string =>
|
|
554
|
+
typeof datasetId === 'string' && Boolean(datasetsById[datasetId]),
|
|
555
|
+
),
|
|
556
|
+
datasetsById,
|
|
487
557
|
logTail: logTail.slice(-LOG_TAIL_LIMIT),
|
|
488
558
|
// Snapshots persisted before totalLogCount existed only know the retained
|
|
489
559
|
// tail, so the best lower bound for the cumulative count is the tail size.
|
|
@@ -1269,6 +1339,34 @@ export function reducePlayRunLedgerEvent(
|
|
|
1269
1339
|
: null,
|
|
1270
1340
|
});
|
|
1271
1341
|
}
|
|
1342
|
+
case 'dataset.lifecycle': {
|
|
1343
|
+
const current = base.datasetsById[event.datasetId];
|
|
1344
|
+
const next: PlayRunLedgerDatasetSnapshot = {
|
|
1345
|
+
datasetId: event.datasetId,
|
|
1346
|
+
path: event.path,
|
|
1347
|
+
tableNamespace: event.tableNamespace,
|
|
1348
|
+
phase: event.phase,
|
|
1349
|
+
persistedRows: Math.max(
|
|
1350
|
+
current?.persistedRows ?? 0,
|
|
1351
|
+
event.persistedRows ?? 0,
|
|
1352
|
+
),
|
|
1353
|
+
succeededRows: Math.max(
|
|
1354
|
+
current?.succeededRows ?? 0,
|
|
1355
|
+
event.succeededRows ?? 0,
|
|
1356
|
+
),
|
|
1357
|
+
failedRows: Math.max(current?.failedRows ?? 0, event.failedRows ?? 0),
|
|
1358
|
+
complete: current?.complete === true || event.complete === true,
|
|
1359
|
+
updatedAt: occurredAt,
|
|
1360
|
+
};
|
|
1361
|
+
return withTiming({
|
|
1362
|
+
...base,
|
|
1363
|
+
orderedDatasetIds: current
|
|
1364
|
+
? base.orderedDatasetIds
|
|
1365
|
+
: [...base.orderedDatasetIds, event.datasetId],
|
|
1366
|
+
datasetsById: { ...base.datasetsById, [event.datasetId]: next },
|
|
1367
|
+
resultTableNamespace: base.resultTableNamespace ?? event.tableNamespace,
|
|
1368
|
+
});
|
|
1369
|
+
}
|
|
1272
1370
|
case 'sheet.summary.updated':
|
|
1273
1371
|
return withTiming({
|
|
1274
1372
|
...base,
|
|
@@ -1376,6 +1474,8 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
|
|
|
1376
1474
|
source?: PlayRunLedgerEventSource;
|
|
1377
1475
|
occurredAt?: number;
|
|
1378
1476
|
channelOffset?: number | null;
|
|
1477
|
+
producerAttempt?: number | null;
|
|
1478
|
+
transportDedupe?: 'semantic' | 'exact';
|
|
1379
1479
|
includeLogAppend?: boolean;
|
|
1380
1480
|
}): PlayRunLedgerEvent[] {
|
|
1381
1481
|
const source = input.source ?? 'worker';
|
|
@@ -1392,16 +1492,31 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
|
|
|
1392
1492
|
if (newLines.length === 0 || input.includeLogAppend === false) {
|
|
1393
1493
|
return [];
|
|
1394
1494
|
}
|
|
1395
|
-
|
|
1396
|
-
|
|
1495
|
+
const events: PlayRunLedgerEvent[] = [];
|
|
1496
|
+
for (
|
|
1497
|
+
let index = 0;
|
|
1498
|
+
index < newLines.length;
|
|
1499
|
+
index += MAX_LEDGER_LOG_LINES_PER_EVENT
|
|
1500
|
+
) {
|
|
1501
|
+
events.push({
|
|
1397
1502
|
type: 'log.appended',
|
|
1398
1503
|
runId: input.runId,
|
|
1399
1504
|
source,
|
|
1400
1505
|
occurredAt,
|
|
1401
|
-
lines: newLines,
|
|
1402
|
-
...(positional ? { channelOffset: input.channelOffset! } : {}),
|
|
1403
|
-
|
|
1404
|
-
|
|
1506
|
+
lines: newLines.slice(index, index + MAX_LEDGER_LOG_LINES_PER_EVENT),
|
|
1507
|
+
...(positional ? { channelOffset: input.channelOffset! + index } : {}),
|
|
1508
|
+
...(positional &&
|
|
1509
|
+
typeof input.producerAttempt === 'number' &&
|
|
1510
|
+
Number.isFinite(input.producerAttempt) &&
|
|
1511
|
+
input.producerAttempt >= 0
|
|
1512
|
+
? { producerAttempt: Math.trunc(input.producerAttempt) }
|
|
1513
|
+
: {}),
|
|
1514
|
+
...(input.transportDedupe
|
|
1515
|
+
? { transportDedupe: input.transportDedupe }
|
|
1516
|
+
: {}),
|
|
1517
|
+
});
|
|
1518
|
+
}
|
|
1519
|
+
return events;
|
|
1405
1520
|
}
|
|
1406
1521
|
|
|
1407
1522
|
/**
|
|
@@ -1419,6 +1534,8 @@ export function buildTerminalLogReplayEvents(input: {
|
|
|
1419
1534
|
source?: PlayRunLedgerEventSource;
|
|
1420
1535
|
occurredAt?: number;
|
|
1421
1536
|
liveLogTotalCount?: number;
|
|
1537
|
+
dedupeMode?: 'semantic' | 'exact';
|
|
1538
|
+
producerAttempt?: number;
|
|
1422
1539
|
}): PlayRunLedgerEvent[] {
|
|
1423
1540
|
const channelOffset = terminalLogReplayChannelOffset({
|
|
1424
1541
|
liveLogTotalCount: input.liveLogTotalCount,
|
|
@@ -1430,6 +1547,10 @@ export function buildTerminalLogReplayEvents(input: {
|
|
|
1430
1547
|
source: 'coordinator',
|
|
1431
1548
|
occurredAt: input.occurredAt,
|
|
1432
1549
|
channelOffset,
|
|
1550
|
+
producerAttempt: input.producerAttempt,
|
|
1551
|
+
transportDedupe:
|
|
1552
|
+
input.dedupeMode ??
|
|
1553
|
+
(typeof input.producerAttempt === 'number' ? 'exact' : undefined),
|
|
1433
1554
|
});
|
|
1434
1555
|
}
|
|
1435
1556
|
|
|
@@ -1466,6 +1587,7 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
|
|
|
1466
1587
|
source,
|
|
1467
1588
|
occurredAt: checkpointAt,
|
|
1468
1589
|
channelOffset: patch.liveLogsChannelOffset ?? null,
|
|
1590
|
+
producerAttempt: patch.liveLogsProducerAttempt ?? null,
|
|
1469
1591
|
})
|
|
1470
1592
|
: [];
|
|
1471
1593
|
const earliestStartedAt =
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
normalizePlayRunLedgerSnapshot,
|
|
5
5
|
summarizeRunRowOutcomes,
|
|
6
6
|
type PlayRunLedgerSnapshot,
|
|
7
|
+
type PlayRunLedgerDatasetSnapshot,
|
|
7
8
|
type PlayRunLedgerStepSnapshot,
|
|
8
9
|
type PlayRunRowOutcomeSummary,
|
|
9
10
|
} from './run-ledger';
|
|
@@ -76,6 +77,7 @@ export type PlayRunLiveSnapshot = {
|
|
|
76
77
|
logsTruncated?: boolean;
|
|
77
78
|
activeArtifactTableNamespace: string | null;
|
|
78
79
|
resultTableNamespace: string | null;
|
|
80
|
+
datasets?: PlayRunLedgerDatasetSnapshot[];
|
|
79
81
|
nodeStates: PlayRunStreamNodeState[];
|
|
80
82
|
activeNodeId: string | null;
|
|
81
83
|
/**
|
|
@@ -212,6 +214,11 @@ function buildSnapshotFromLedger(
|
|
|
212
214
|
...(snapshot.logsTruncated ? { logsTruncated: true } : {}),
|
|
213
215
|
activeArtifactTableNamespace: snapshot.activeArtifactTableNamespace ?? null,
|
|
214
216
|
resultTableNamespace: snapshot.resultTableNamespace ?? null,
|
|
217
|
+
datasets: snapshot.orderedDatasetIds
|
|
218
|
+
.map((datasetId) => snapshot.datasetsById[datasetId])
|
|
219
|
+
.filter((dataset): dataset is PlayRunLedgerDatasetSnapshot =>
|
|
220
|
+
Boolean(dataset),
|
|
221
|
+
),
|
|
215
222
|
nodeStates,
|
|
216
223
|
activeNodeId: snapshot.activeStepId ?? null,
|
|
217
224
|
...(rowOutcomes ? { rowOutcomes } : {}),
|
|
@@ -344,13 +344,17 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
344
344
|
const outputPath = `${workDir}/deepline-play-output-${randomUUID()}.log`;
|
|
345
345
|
const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
|
|
346
346
|
const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
|
|
347
|
+
const runnerTraceEnv =
|
|
348
|
+
process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
|
|
349
|
+
? 'DEEPLINE_RUNTIME_RECEIPT_TRACE=1 '
|
|
350
|
+
: '';
|
|
347
351
|
const runnerCommand = `${nodeMaterializePayloadCommand({
|
|
348
352
|
envelopePath,
|
|
349
353
|
runnerPath,
|
|
350
354
|
configPath,
|
|
351
355
|
artifactCodePath,
|
|
352
356
|
crashPusherPath,
|
|
353
|
-
})} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
|
|
357
|
+
})} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
|
|
354
358
|
// Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
|
|
355
359
|
// pushes the parsed (or synthesized) terminal to the gateway so the parked
|
|
356
360
|
// worker wakes within seconds of ANY runner death — process.exit abuse, OOM
|
|
@@ -122,19 +122,34 @@ export async function inspectDetachedDaytonaRunner(input: {
|
|
|
122
122
|
cmdId: string;
|
|
123
123
|
outputPath: string;
|
|
124
124
|
exitCodePath: string;
|
|
125
|
-
}): Promise<{
|
|
125
|
+
}): Promise<{
|
|
126
|
+
exitCode: number | null;
|
|
127
|
+
result: PlayRunnerResult | null;
|
|
128
|
+
diagnosis: {
|
|
129
|
+
stage:
|
|
130
|
+
| 'salvaged'
|
|
131
|
+
| 'result_missing'
|
|
132
|
+
| 'command_inspection_failed'
|
|
133
|
+
| 'sandbox_lookup_failed';
|
|
134
|
+
detail: string | null;
|
|
135
|
+
};
|
|
136
|
+
}> {
|
|
126
137
|
try {
|
|
127
138
|
const { clientOptions } = loadDaytonaRequiredConfig();
|
|
128
139
|
const daytona = daytonaSdkClientFactory.createFull(clientOptions);
|
|
129
140
|
const sandbox = (await daytona.get(input.sandboxId)) as DaytonaSandbox;
|
|
130
141
|
let exitCode: number | null = null;
|
|
142
|
+
let commandInspectionError: string | null = null;
|
|
131
143
|
try {
|
|
132
144
|
const command = await sandbox.process.getSessionCommand(
|
|
133
145
|
input.sessionId,
|
|
134
146
|
input.cmdId,
|
|
135
147
|
);
|
|
136
|
-
exitCode =
|
|
148
|
+
exitCode =
|
|
149
|
+
typeof command?.exitCode === 'number' ? command.exitCode : null;
|
|
137
150
|
} catch (error) {
|
|
151
|
+
commandInspectionError =
|
|
152
|
+
error instanceof Error ? error.message : String(error);
|
|
138
153
|
console.warn('[play-runner.daytona.detached_inspect_command_failed]', {
|
|
139
154
|
sandboxId: input.sandboxId,
|
|
140
155
|
cmdId: input.cmdId,
|
|
@@ -148,7 +163,16 @@ export async function inspectDetachedDaytonaRunner(input: {
|
|
|
148
163
|
timeoutMs: 10_000,
|
|
149
164
|
});
|
|
150
165
|
if (!recovered.execution) {
|
|
151
|
-
return {
|
|
166
|
+
return {
|
|
167
|
+
exitCode,
|
|
168
|
+
result: null,
|
|
169
|
+
diagnosis: commandInspectionError
|
|
170
|
+
? {
|
|
171
|
+
stage: 'command_inspection_failed',
|
|
172
|
+
detail: commandInspectionError,
|
|
173
|
+
}
|
|
174
|
+
: { stage: 'result_missing', detail: null },
|
|
175
|
+
};
|
|
152
176
|
}
|
|
153
177
|
const result =
|
|
154
178
|
findPlayRunnerResult(parsePlayRunnerEvents(recovered.execution.result)) ??
|
|
@@ -156,14 +180,22 @@ export async function inspectDetachedDaytonaRunner(input: {
|
|
|
156
180
|
return {
|
|
157
181
|
exitCode: exitCode ?? recovered.execution.exitCode,
|
|
158
182
|
result,
|
|
183
|
+
diagnosis: result
|
|
184
|
+
? { stage: 'salvaged', detail: null }
|
|
185
|
+
: { stage: 'result_missing', detail: null },
|
|
159
186
|
};
|
|
160
187
|
} catch (error) {
|
|
188
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
161
189
|
console.warn('[play-runner.daytona.detached_inspect_failed]', {
|
|
162
190
|
sandboxId: input.sandboxId,
|
|
163
191
|
cmdId: input.cmdId,
|
|
164
|
-
error:
|
|
192
|
+
error: detail,
|
|
165
193
|
});
|
|
166
|
-
return {
|
|
194
|
+
return {
|
|
195
|
+
exitCode: null,
|
|
196
|
+
result: null,
|
|
197
|
+
diagnosis: { stage: 'sandbox_lookup_failed', detail },
|
|
198
|
+
};
|
|
167
199
|
}
|
|
168
200
|
}
|
|
169
201
|
|