deepline 0.3.449 → 0.3.451
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/play.ts +1 -1
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/monitors/http-error-response.ts +3 -3
- package/dist/bundling-sources/shared_libs/monitors/provider-monitor-capability.ts +1 -1
- package/dist/bundling-sources/shared_libs/observability/telemetry.ts +2 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +133 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts +80 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +16 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +34 -10
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +14 -11
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +54 -6
- package/dist/cli/auth-WPQBEOES.js +40 -0
- package/dist/cli/{auth-YK26H6J4.mjs → auth-YLBWEMRW.mjs} +4 -4
- package/dist/cli/{billing-CNQTA3GB.mjs → billing-DUFOEOPT.mjs} +4 -4
- package/dist/cli/{billing-4KS7B4U3.js → billing-GWG5D3P6.js} +73 -73
- package/dist/cli/{chunk-7ZIXBQQ7.mjs → chunk-33EM4SMH.mjs} +3 -3
- package/dist/cli/{chunk-UBKYOUWW.mjs → chunk-37CPEQBE.mjs} +3 -3
- package/dist/cli/{chunk-6GZCN5TQ.mjs → chunk-3HT47QPJ.mjs} +2 -2
- package/dist/cli/{chunk-3NDIWD42.mjs → chunk-4C7HPD57.mjs} +1 -1
- package/dist/cli/{chunk-NZEKP7IE.js → chunk-5DUMNKQG.js} +50 -50
- package/dist/cli/{chunk-MPMLO7CZ.js → chunk-6G6KX7I7.js} +6 -6
- package/dist/cli/{chunk-EICL26GT.js → chunk-6TFACCXN.js} +2 -2
- package/dist/cli/{chunk-RPG4M5UD.js → chunk-7VRWERTM.js} +10 -10
- package/dist/cli/{chunk-L4RSMA4V.mjs → chunk-BJOVX7EB.mjs} +2 -2
- package/dist/cli/{chunk-KPTUP6NA.mjs → chunk-BQQLTSGC.mjs} +4 -4
- package/dist/cli/{chunk-XQ3SQONI.js → chunk-C4NPWUZY.js} +52 -52
- package/dist/cli/{chunk-PUXZ2KKO.js → chunk-CH2WDSBC.js} +8 -8
- package/dist/cli/{chunk-DNXWVDL7.mjs → chunk-E7V5CRA2.mjs} +8 -8
- package/dist/cli/{chunk-ZHFSBKLN.js → chunk-EJ6YI46S.js} +2 -2
- package/dist/cli/{chunk-EZTBBDYU.js → chunk-F2DW7DFE.js} +8 -8
- package/dist/cli/{chunk-S4RC3CBU.js → chunk-HSKLAPMD.js} +26 -26
- package/dist/cli/{chunk-NTQRNY4J.js → chunk-JRPNCMMN.js} +35 -35
- package/dist/cli/{chunk-TMPLRRFJ.mjs → chunk-JUCEEIEC.mjs} +1 -1
- package/dist/cli/{chunk-UKEINGF3.js → chunk-L2IUGWV5.js} +1 -1
- package/dist/cli/{chunk-TBOUEQKK.mjs → chunk-LPPAJJJN.mjs} +1 -1
- package/dist/cli/{chunk-UZN4GB6J.js → chunk-O3KVUHCG.js} +20 -20
- package/dist/cli/{chunk-2O4JOOZU.mjs → chunk-OKTCLFKM.mjs} +5 -5
- package/dist/cli/{chunk-56EULWUU.mjs → chunk-PJYTV24I.mjs} +1 -1
- package/dist/cli/{chunk-TEKBWPMT.mjs → chunk-RFPP4Q66.mjs} +3 -3
- package/dist/cli/{chunk-2PQAXRHK.js → chunk-TFCLUWDQ.js} +6 -6
- package/dist/cli/{chunk-HDGN7HC5.js → chunk-WDOWAOYB.js} +4 -4
- package/dist/cli/{chunk-HJ4KEE6R.mjs → chunk-WLVGRU5A.mjs} +2 -2
- package/dist/cli/{chunk-3653HHHS.mjs → chunk-XLOXUW5Y.mjs} +1 -1
- package/dist/cli/{chunk-ZSCS7MC2.mjs → chunk-YXXIFMXI.mjs} +2 -2
- package/dist/cli/{chunk-B77RQVZ3.js → chunk-Z3DBS4IB.js} +81 -81
- package/dist/cli/{chunk-RVB3VVEH.js → chunk-ZNDE2BTH.js} +16 -16
- package/dist/cli/{chunk-BZPSFI7J.mjs → chunk-ZPV7DMAA.mjs} +1 -1
- package/dist/cli/{csv-3CD3TJD6.js → csv-4CHXZQXO.js} +12 -12
- package/dist/cli/{csv-A4V7TFYC.mjs → csv-RL7WQD6Y.mjs} +3 -3
- package/dist/cli/{db-STC5BK4K.js → db-MRI5IM2A.js} +25 -25
- package/dist/cli/{db-VSUPLVTX.mjs → db-ZDBAWV7B.mjs} +4 -4
- package/dist/cli/{describe-ZKG4DMJT.js → describe-2FYKXM5B.js} +16 -16
- package/dist/cli/{describe-GHZKYS4A.mjs → describe-RRQUUJQ3.mjs} +12 -12
- package/dist/cli/{executions-HCC3CNO6.js → executions-KDBD47A5.js} +6 -6
- package/dist/cli/{executions-UJ3IL4UT.mjs → executions-RJL76TJS.mjs} +4 -4
- package/dist/cli/{feedback-ZU77DPV2.js → feedback-2CE24NHD.js} +7 -7
- package/dist/cli/{feedback-SAR53LLO.mjs → feedback-KOYLXHYE.mjs} +3 -3
- package/dist/cli/git-DZ5GROPH.js +24 -0
- package/dist/cli/{git-TVZCW7DR.mjs → git-RDUHU6L6.mjs} +4 -4
- package/dist/cli/index.js +131 -131
- package/dist/cli/index.mjs +36 -36
- package/dist/cli/{legacy-noops-J4ZZXHYM.mjs → legacy-noops-4HS2LYP6.mjs} +4 -4
- package/dist/cli/{legacy-noops-Z3WHNTSA.js → legacy-noops-7FOMGBMN.js} +7 -7
- package/dist/cli/{monitors-LV36TKZ4.js → monitors-IJXNK5BL.js} +93 -92
- package/dist/cli/{monitors-CWXHKFTV.mjs → monitors-JLBTNBVU.mjs} +13 -12
- package/dist/cli/{org-SCZUUDNC.js → org-JL7SW6VY.js} +66 -66
- package/dist/cli/{org-L62Y54LR.mjs → org-OWP6K4AA.mjs} +4 -4
- package/dist/cli/{play-BBPMHBOF.mjs → play-FNOVKU74.mjs} +10 -10
- package/dist/cli/{play-R3DYB7VW.js → play-LLNZWHT6.js} +283 -283
- package/dist/cli/{providers-FCEWCAU5.js → providers-E4XIEUC7.js} +7 -7
- package/dist/cli/{providers-UIIH2I5D.mjs → providers-WBOEOKE7.mjs} +4 -4
- package/dist/cli/{quickstart-RDPEZCYS.js → quickstart-3FUHKGEY.js} +5 -5
- package/dist/cli/{quickstart-TF3GJYKG.mjs → quickstart-6G5G3BVA.mjs} +3 -3
- package/dist/cli/search-4ECC773X.js +30 -0
- package/dist/cli/{search-YXLWFCC2.mjs → search-CNMB4Z6M.mjs} +5 -5
- package/dist/cli/{secrets-5BLHMG6B.mjs → secrets-5FY5APVK.mjs} +4 -4
- package/dist/cli/{secrets-CXK3KSSF.js → secrets-BDSWOYAT.js} +13 -13
- package/dist/cli/{sessions-AFBLVB2P.js → sessions-B7T2CHYK.js} +5 -5
- package/dist/cli/{sessions-LI5LSWTV.mjs → sessions-PMIKVPQN.mjs} +4 -4
- package/dist/cli/{settings-X4L4DAPQ.mjs → settings-4TAYLTPR.mjs} +8 -8
- package/dist/cli/{settings-52OZBLSO.js → settings-RXQJRRLR.js} +75 -75
- package/dist/cli/{setup-V7SWBDE7.mjs → setup-LVUISLVO.mjs} +7 -7
- package/dist/cli/{setup-RZAG66C6.js → setup-ZTMETGSO.js} +42 -42
- package/dist/cli/skills-MIENPTOD.js +29 -0
- package/dist/cli/{skills-SS563CGW.mjs → skills-NSJ43Z5Q.mjs} +5 -5
- package/dist/cli/{tools-BZJQPQAH.mjs → tools-CFCKIGDY.mjs} +12 -12
- package/dist/cli/tools-CXJO5XSH.js +44 -0
- package/dist/cli/update-B2LQGEXF.js +31 -0
- package/dist/cli/{update-B7FAZUAD.mjs → update-HYOCBXJO.mjs} +7 -7
- package/dist/cli/{workflow-ASOCZBWQ.mjs → workflow-RKX45RJ4.mjs} +6 -6
- package/dist/cli/{workflow-66DHJDSN.js → workflow-U6S4SWTF.js} +16 -16
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +6 -6
- package/dist/index.mjs +3 -3
- package/dist/install-integrity.json +78 -78
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +2 -2
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
- package/dist/cli/auth-NVVEYCNF.js +0 -40
- package/dist/cli/git-GJLNAT37.js +0 -24
- package/dist/cli/search-GQK4C6UL.js +0 -30
- package/dist/cli/skills-4DXTSI6L.js +0 -29
- package/dist/cli/tools-63ZIXQJ5.js +0 -44
- package/dist/cli/update-ZDGSS4OW.js +0 -31
|
@@ -181,7 +181,7 @@ export type PlayFetchResponse = PlayAuthoringFetchResponse;
|
|
|
181
181
|
* row to its output stream. This is how you build a play "on top of" a monitor
|
|
182
182
|
* (e.g. run enrichment every time a watched company posts a new job). Each
|
|
183
183
|
* listener binds to a monitor tool id + one of its output stream keys (see
|
|
184
|
-
* `deepline
|
|
184
|
+
* `deepline describe <id>` for a tool's streams and row columns).
|
|
185
185
|
* The changed row is delivered as the listener event's `after` in the play function's input.
|
|
186
186
|
*
|
|
187
187
|
* The default Play runtime is 30 minutes. For bounded long-running batches, add
|
|
@@ -205,7 +205,7 @@ export const SDK_RELEASE = {
|
|
|
205
205
|
// getters keep their established compatibility behavior.
|
|
206
206
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
207
207
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
208
|
-
version: '0.3.
|
|
208
|
+
version: '0.3.451',
|
|
209
209
|
updateSummary: {
|
|
210
210
|
introducedIn: '0.3.90',
|
|
211
211
|
summary:
|
|
@@ -127,10 +127,10 @@ export function monitorErrorNextAction(payload: MonitorErrorPayload): string {
|
|
|
127
127
|
return 'Stop monitor work and contact the Deepline team to request access; then rerun `deepline monitors status`.';
|
|
128
128
|
}
|
|
129
129
|
if (code === 'monitor_tool_not_found' || code === 'unknown_monitor_tool') {
|
|
130
|
-
return 'Run `deepline monitors
|
|
130
|
+
return 'Run `deepline tools list --categories monitors`, choose a returned tool id, then retry with that exact id.';
|
|
131
131
|
}
|
|
132
132
|
if (code === 'monitor_provider_not_offered') {
|
|
133
|
-
return 'This provider monitor is retired for new deployments. Run `deepline monitors
|
|
133
|
+
return 'This provider monitor is retired for new deployments. Run `deepline tools list --categories monitors` to choose a supported monitor. Existing retired-provider monitors can still be inspected and deleted.';
|
|
134
134
|
}
|
|
135
135
|
if (code === 'deployed_monitor_not_found' || code === 'monitor_not_found') {
|
|
136
136
|
return 'Run `deepline monitors list --status all`, choose a returned public monitor key, then retry with that key.';
|
|
@@ -183,7 +183,7 @@ export function monitorErrorNextAction(payload: MonitorErrorPayload): string {
|
|
|
183
183
|
return 'Fix every returned issue using its path and suggestion. Rerun `deepline monitors check` for definitions or the relevant dry-run for delete/reactivate; do not mutate until validation passes.';
|
|
184
184
|
}
|
|
185
185
|
if (code === 'monitor_tool_unavailable') {
|
|
186
|
-
return 'Run `deepline monitors
|
|
186
|
+
return 'Run `deepline tools list --categories monitors`; choose a tool returned for this workspace, or stop and request access to the unavailable capability.';
|
|
187
187
|
}
|
|
188
188
|
if (code === 'monitor_entitlement_limit_exceeded') {
|
|
189
189
|
return 'Run `deepline monitors list --status all`; reuse or disable an existing monitor, or ask the user to change entitlement before retrying.';
|
|
@@ -447,7 +447,7 @@ export function defineProviderMonitorCapability(input: {
|
|
|
447
447
|
fields,
|
|
448
448
|
pricing: {
|
|
449
449
|
presentation:
|
|
450
|
-
'Use the Deepline pricing returned by describe,
|
|
450
|
+
'Use the Deepline pricing returned by describe, check, or deploy --dry-run. Provider spend is not exposed.',
|
|
451
451
|
},
|
|
452
452
|
identity: {
|
|
453
453
|
semantics: input.upstreamCollision
|
|
@@ -332,9 +332,10 @@ function errorFields(
|
|
|
332
332
|
export function createConsoleTelemetryAdapter(input?: {
|
|
333
333
|
console?: Pick<Console, 'debug' | 'info' | 'warn' | 'error'>;
|
|
334
334
|
}): TelemetryAdapter {
|
|
335
|
-
const target = input?.console ?? console;
|
|
336
335
|
return {
|
|
337
336
|
emit(event) {
|
|
337
|
+
// Convex Node actions replace console for each invocation in a warm process.
|
|
338
|
+
const target = input?.console ?? console;
|
|
338
339
|
const line = JSON.stringify(event);
|
|
339
340
|
if (event.telemetryKind === 'log' || event.telemetryKind === 'span')
|
|
340
341
|
target[event.telemetryLevel](line);
|
|
@@ -64,6 +64,14 @@ const TERMINAL_TRANSPORT_OUTPUT_WARNINGS_MAX_BYTES = 32 * 1024;
|
|
|
64
64
|
const TERMINAL_TRANSPORT_RUNTIME_DIAGNOSTICS_MAX_BYTES = 16 * 1024;
|
|
65
65
|
const TERMINAL_TRANSPORT_ERROR_MAX_BYTES = 64 * 1024;
|
|
66
66
|
const TERMINAL_TRANSPORT_TABLE_NAMESPACE_MAX_BYTES = 8 * 1024;
|
|
67
|
+
/**
|
|
68
|
+
* Leaves room in the gateway body for the action, run ID and log offset around
|
|
69
|
+
* the result. `terminalRequestBody` drops progress events to fit on its own.
|
|
70
|
+
*/
|
|
71
|
+
const TERMINAL_SUSPENDED_RESULT_MAX_BYTES =
|
|
72
|
+
RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES - 4 * 1024;
|
|
73
|
+
const RUNTIME_SUSPENDED_CHECKPOINT_TOO_LARGE_CODE =
|
|
74
|
+
'RUNTIME_SUSPENDED_CHECKPOINT_TOO_LARGE';
|
|
67
75
|
|
|
68
76
|
type CompletedRunnerResult = Extract<PlayRunnerResult, { status: 'completed' }>;
|
|
69
77
|
type SuspendedRunnerResult = Extract<PlayRunnerResult, { status: 'suspended' }>;
|
|
@@ -288,6 +296,7 @@ function appendTerminalDiagnosticsMarker(
|
|
|
288
296
|
logContexts?: Array<PlayLogContext | null>,
|
|
289
297
|
logIds?: string[],
|
|
290
298
|
logMetadata?: Array<PlayLogMetadata | null>,
|
|
299
|
+
completeness = 'the authored play output is complete.',
|
|
291
300
|
): {
|
|
292
301
|
logs: string[];
|
|
293
302
|
logContexts?: Array<PlayLogContext | null>;
|
|
@@ -296,7 +305,7 @@ function appendTerminalDiagnosticsMarker(
|
|
|
296
305
|
} {
|
|
297
306
|
const marker =
|
|
298
307
|
`[deepline] terminal diagnostic metadata was bounded (${truncatedFields.join(', ')}); ` +
|
|
299
|
-
|
|
308
|
+
completeness;
|
|
300
309
|
const marked = [marker, ...logs];
|
|
301
310
|
const markedContexts = logContexts ? [null, ...logContexts] : undefined;
|
|
302
311
|
const markedMetadata = logMetadata ? [null, ...logMetadata] : undefined;
|
|
@@ -495,6 +504,126 @@ function boundSettledTerminalTransportDiagnostics(
|
|
|
495
504
|
};
|
|
496
505
|
}
|
|
497
506
|
|
|
507
|
+
/**
|
|
508
|
+
* The gateway rejects a body over its ceiling before reading the action, and
|
|
509
|
+
* that line is also too long for the crash pusher's output tail, so an oversized
|
|
510
|
+
* suspended result surfaces only as an unexplained "exited with code 0". Keep
|
|
511
|
+
* ordinary suspensions byte-identical. When one would not fit, drop runner
|
|
512
|
+
* diagnostics first. The checkpoint is never sliced: without it the run cannot
|
|
513
|
+
* resume, so a checkpoint that alone exceeds the ceiling is an explicit failure.
|
|
514
|
+
*/
|
|
515
|
+
function suspendedResultWithinGatewayBody(
|
|
516
|
+
result: SuspendedRunnerResult & IncludedResumeCheckpoint,
|
|
517
|
+
): PlayRunnerTerminalTransportResult {
|
|
518
|
+
if (fitsJsonByteLimit(result, TERMINAL_SUSPENDED_RESULT_MAX_BYTES)) {
|
|
519
|
+
return result;
|
|
520
|
+
}
|
|
521
|
+
const logProjection = retainTerminalLogTailWithRuntimeDiagnostics(
|
|
522
|
+
result.logs,
|
|
523
|
+
result.logContexts,
|
|
524
|
+
result.logMetadata,
|
|
525
|
+
result.logIds,
|
|
526
|
+
);
|
|
527
|
+
const stats =
|
|
528
|
+
result.stats.runtimeWorkCounterSegment === undefined
|
|
529
|
+
? result.stats
|
|
530
|
+
: {
|
|
531
|
+
// The worker reads this to number the resumed segment, so it must
|
|
532
|
+
// be the first key retained rather than whichever one fits.
|
|
533
|
+
runtimeWorkCounterSegment: result.stats.runtimeWorkCounterSegment,
|
|
534
|
+
...result.stats,
|
|
535
|
+
};
|
|
536
|
+
const statsProjection = retainRecordWithinJsonByteLimit(
|
|
537
|
+
stats,
|
|
538
|
+
TERMINAL_TRANSPORT_STATS_MAX_BYTES,
|
|
539
|
+
);
|
|
540
|
+
const stepsProjection = retainArrayWithinJsonByteLimit(
|
|
541
|
+
result.steps,
|
|
542
|
+
TERMINAL_TRANSPORT_STEPS_MAX_BYTES,
|
|
543
|
+
'head',
|
|
544
|
+
);
|
|
545
|
+
const truncatedFields = [
|
|
546
|
+
...(logProjection.truncated ? ['logs'] : []),
|
|
547
|
+
...(statsProjection.truncated ? ['stats'] : []),
|
|
548
|
+
...(stepsProjection.truncated ? ['steps'] : []),
|
|
549
|
+
];
|
|
550
|
+
const projectAligned = <T>(values: readonly T[] | undefined) =>
|
|
551
|
+
values?.length
|
|
552
|
+
? logProjection.indexes.map((index) => values[index]!)
|
|
553
|
+
: undefined;
|
|
554
|
+
const boundedLogs =
|
|
555
|
+
truncatedFields.length > 0
|
|
556
|
+
? appendTerminalDiagnosticsMarker(
|
|
557
|
+
logProjection.values,
|
|
558
|
+
truncatedFields,
|
|
559
|
+
projectAligned(result.logContexts),
|
|
560
|
+
projectAligned(result.logIds),
|
|
561
|
+
projectAligned(result.logMetadata),
|
|
562
|
+
'the resume checkpoint is complete.',
|
|
563
|
+
)
|
|
564
|
+
: {
|
|
565
|
+
logs: logProjection.values,
|
|
566
|
+
logContexts: projectAligned(result.logContexts),
|
|
567
|
+
logIds: projectAligned(result.logIds),
|
|
568
|
+
logMetadata: projectAligned(result.logMetadata),
|
|
569
|
+
};
|
|
570
|
+
const bounded = {
|
|
571
|
+
...result,
|
|
572
|
+
logs: boundedLogs.logs,
|
|
573
|
+
logContexts: boundedLogs.logContexts,
|
|
574
|
+
logMetadata: boundedLogs.logMetadata,
|
|
575
|
+
logIds: boundedLogs.logIds,
|
|
576
|
+
stats: statsProjection.value,
|
|
577
|
+
steps: stepsProjection.values,
|
|
578
|
+
};
|
|
579
|
+
if (fitsJsonByteLimit(bounded, TERMINAL_SUSPENDED_RESULT_MAX_BYTES)) {
|
|
580
|
+
return bounded;
|
|
581
|
+
}
|
|
582
|
+
// A run that can still resume must not fail over diagnostics, so keep only
|
|
583
|
+
// what the worker needs before giving up on the checkpoint.
|
|
584
|
+
const segment = result.stats.runtimeWorkCounterSegment;
|
|
585
|
+
const withoutDiagnostics = {
|
|
586
|
+
...result,
|
|
587
|
+
logs: [
|
|
588
|
+
'[deepline] terminal diagnostic metadata was dropped (logs, stats, steps); the resume checkpoint is complete.',
|
|
589
|
+
],
|
|
590
|
+
logContexts: undefined,
|
|
591
|
+
logMetadata: undefined,
|
|
592
|
+
logIds: undefined,
|
|
593
|
+
stats: segment === undefined ? {} : { runtimeWorkCounterSegment: segment },
|
|
594
|
+
steps: [],
|
|
595
|
+
};
|
|
596
|
+
if (
|
|
597
|
+
fitsJsonByteLimit(withoutDiagnostics, TERMINAL_SUSPENDED_RESULT_MAX_BYTES)
|
|
598
|
+
) {
|
|
599
|
+
return withoutDiagnostics;
|
|
600
|
+
}
|
|
601
|
+
const { checkpoint: _checkpoint, suspension, ...diagnostics } = bounded;
|
|
602
|
+
const checkpointMiB = (result.checkpointBytes / (1024 * 1024)).toFixed(2);
|
|
603
|
+
const limitMiB = RUNNER_TERMINAL_PUSH_MAX_BODY_BYTES / (1024 * 1024);
|
|
604
|
+
const message =
|
|
605
|
+
`The play paused (${suspension.kind}) with a ${checkpointMiB} MiB resume checkpoint, which does not fit the ${limitMiB} MiB limit, so it cannot resume. ` +
|
|
606
|
+
'Split the work into smaller runs, or finish fewer tool calls before the pause.';
|
|
607
|
+
const error = `${RUNTIME_SUSPENDED_CHECKPOINT_TOO_LARGE_CODE}: ${message}`;
|
|
608
|
+
return {
|
|
609
|
+
...diagnostics,
|
|
610
|
+
status: 'failed',
|
|
611
|
+
error,
|
|
612
|
+
errors: [
|
|
613
|
+
{
|
|
614
|
+
code: RUNTIME_SUSPENDED_CHECKPOINT_TOO_LARGE_CODE,
|
|
615
|
+
phase: 'runtime',
|
|
616
|
+
message,
|
|
617
|
+
retryable: false,
|
|
618
|
+
cause: error,
|
|
619
|
+
},
|
|
620
|
+
],
|
|
621
|
+
checkpointDisposition:
|
|
622
|
+
RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.OMITTED_TERMINAL_REPLAY,
|
|
623
|
+
checkpointBytes: result.checkpointBytes,
|
|
624
|
+
};
|
|
625
|
+
}
|
|
626
|
+
|
|
498
627
|
export function runnerTerminalResultForTransport(
|
|
499
628
|
result: PlayRunnerResult | PlayRunnerTerminalTransportResult,
|
|
500
629
|
): PlayRunnerTerminalTransportResult {
|
|
@@ -534,7 +663,9 @@ export function runnerTerminalResultForTransport(
|
|
|
534
663
|
if (transportResult.status === 'suspended') {
|
|
535
664
|
// A suspended result retains its exact checkpoint because it is needed to
|
|
536
665
|
// resume. Its checkpoint follows the dedicated receipt transport contract.
|
|
537
|
-
return
|
|
666
|
+
return suspendedResultWithinGatewayBody(
|
|
667
|
+
transportResult as SuspendedRunnerResult & IncludedResumeCheckpoint,
|
|
668
|
+
);
|
|
538
669
|
}
|
|
539
670
|
const bounded = boundSettledTerminalTransportDiagnostics(transportResult);
|
|
540
671
|
if (!fitsJsonByteLimit(bounded, TERMINAL_RUN_RESULT_MAX_BYTES)) {
|
|
@@ -62,6 +62,27 @@ const DAYTONA_LOGGED_ERROR_NAMES: ReadonlySet<string> = new Set([
|
|
|
62
62
|
'Error',
|
|
63
63
|
]);
|
|
64
64
|
const DAYTONA_LOGGED_PROVIDER_TEXT_MAX_CHARS = 160;
|
|
65
|
+
// Deliberately unanchored: `insufficient_credits` and `billingError` must match,
|
|
66
|
+
// and withholding an innocent word costs only one message. `suspended` is a
|
|
67
|
+
// sandbox state and `load balancer` is infrastructure, so they stay readable.
|
|
68
|
+
const DAYTONA_PROVIDER_ACCOUNT_TEXT_PATTERN =
|
|
69
|
+
/\$\s*\d|credit|billing|invoice|payment|(?<!su)spen[dt]|charge|cost|\busd\b|balance(?!r)|fund|wallet|top.?up|subscription|upgrade your/i;
|
|
70
|
+
const DAYTONA_PROVIDER_PAYMENT_REQUIRED_STATUS = 402;
|
|
71
|
+
// By name: the SDK ships CJS and ESM builds, and `instanceof` fails across them.
|
|
72
|
+
// Deepline's own `Daytona*` errors carry diagnostics classifiers read in full.
|
|
73
|
+
const DAYTONA_SDK_ERROR_NAMES: ReadonlySet<string> = new Set([
|
|
74
|
+
'DaytonaError',
|
|
75
|
+
'DaytonaNotFoundError',
|
|
76
|
+
'DaytonaRateLimitError',
|
|
77
|
+
'DaytonaAuthenticationError',
|
|
78
|
+
'DaytonaAuthorizationError',
|
|
79
|
+
'DaytonaConflictError',
|
|
80
|
+
'DaytonaValidationError',
|
|
81
|
+
'DaytonaTimeoutError',
|
|
82
|
+
'DaytonaConnectionError',
|
|
83
|
+
]);
|
|
84
|
+
const DAYTONA_PROVIDER_ERROR_CODE_PATTERN = /^[A-Za-z][A-Za-z0-9_.-]{1,63}$/;
|
|
85
|
+
const DAYTONA_PROVIDER_DETAIL_PATTERN = / \[provider [^\]]*\]/g;
|
|
65
86
|
|
|
66
87
|
/** A provider state from Daytona's published enum, or `unrecognized`. */
|
|
67
88
|
export function daytonaSandboxStateForLog(state: unknown): string | null {
|
|
@@ -167,6 +188,60 @@ export function boundedDaytonaProviderTextForLog(
|
|
|
167
188
|
return bounded || null;
|
|
168
189
|
}
|
|
169
190
|
|
|
191
|
+
/**
|
|
192
|
+
* Provider text for a run's error, which customers read. Same bounds as the
|
|
193
|
+
* log form, and text about the provider account is withheld so provider spend
|
|
194
|
+
* never reaches a customer. Status and code still identify the failure.
|
|
195
|
+
*/
|
|
196
|
+
export function daytonaProviderTextForRunError(value: string): string | null {
|
|
197
|
+
return DAYTONA_PROVIDER_ACCOUNT_TEXT_PATTERN.test(value)
|
|
198
|
+
? 'provider account detail withheld'
|
|
199
|
+
: boundedDaytonaProviderTextForLog(value);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** A provider-authored error, with or without an HTTP status. */
|
|
203
|
+
export function isDaytonaProviderError(error: unknown): error is DaytonaError {
|
|
204
|
+
return error instanceof Error && DAYTONA_SDK_ERROR_NAMES.has(error.name);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** A provider error's message as a run's error states it. */
|
|
208
|
+
export function daytonaProviderMessageForRunError(
|
|
209
|
+
error: DaytonaError,
|
|
210
|
+
): string | null {
|
|
211
|
+
return daytonaErrorStatusForLog(error) ===
|
|
212
|
+
DAYTONA_PROVIDER_PAYMENT_REQUIRED_STATUS
|
|
213
|
+
? 'provider account detail withheld'
|
|
214
|
+
: daytonaProviderTextForRunError(error.message);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* The provider's status and code as a suffix for a run's error text. The key is
|
|
219
|
+
* `errorCode=` because failure taxonomies read a bare `code=` as Deepline's own.
|
|
220
|
+
*/
|
|
221
|
+
export function daytonaProviderDetailForRunError(error: unknown): string {
|
|
222
|
+
const status = daytonaErrorStatusForLog(error);
|
|
223
|
+
const errorCode = (error as { errorCode?: unknown } | null)?.errorCode;
|
|
224
|
+
const fields = [
|
|
225
|
+
...(status === null ? [] : [`status=${status}`]),
|
|
226
|
+
...(typeof errorCode === 'string' &&
|
|
227
|
+
status !== DAYTONA_PROVIDER_PAYMENT_REQUIRED_STATUS &&
|
|
228
|
+
DAYTONA_PROVIDER_ERROR_CODE_PATTERN.test(errorCode) &&
|
|
229
|
+
!DAYTONA_PROVIDER_ACCOUNT_TEXT_PATTERN.test(errorCode)
|
|
230
|
+
? [`errorCode=${errorCode}`]
|
|
231
|
+
: []),
|
|
232
|
+
];
|
|
233
|
+
return fields.length > 0 ? ` [provider ${fields.join(' ')}]` : '';
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* The run error as retry classification reads it. A diagnostic must not choose
|
|
238
|
+
* a retry: `errorCode=ETIMEDOUT` here would re-run a start whose outcome is
|
|
239
|
+
* unknown.
|
|
240
|
+
*/
|
|
241
|
+
export function withoutDaytonaProviderDetail(error: string): string {
|
|
242
|
+
return error.replace(DAYTONA_PROVIDER_DETAIL_PATTERN, '');
|
|
243
|
+
}
|
|
244
|
+
|
|
170
245
|
/** What this client knew when a sent create ended without a verdict. */
|
|
171
246
|
export type DaytonaLaunchOutcomeDiagnostic = {
|
|
172
247
|
/** Whether the create request returned a response before the deadline. */
|
|
@@ -239,12 +314,16 @@ function providerError(error: unknown): unknown {
|
|
|
239
314
|
if (!axios.isAxiosError(error)) return error;
|
|
240
315
|
const status = error.response?.status;
|
|
241
316
|
const data = error.response?.data;
|
|
242
|
-
const
|
|
317
|
+
const providerMessage =
|
|
243
318
|
typeof data === 'string'
|
|
244
319
|
? data
|
|
245
320
|
: typeof data?.message === 'string'
|
|
246
321
|
? data.message
|
|
247
322
|
: error.message;
|
|
323
|
+
// Not axios's "Request failed with status code" wording: retry classification
|
|
324
|
+
// matches that text, and this change only explains the failure.
|
|
325
|
+
const message =
|
|
326
|
+
providerMessage.trim() || 'Daytona returned an empty response body';
|
|
248
327
|
const errorClasses: Record<number, typeof DaytonaError> = {
|
|
249
328
|
400: DaytonaValidationError,
|
|
250
329
|
401: DaytonaAuthenticationError,
|
package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts
CHANGED
|
@@ -6,7 +6,11 @@ import {
|
|
|
6
6
|
daytonaErrorClassForLog,
|
|
7
7
|
daytonaErrorCodeForLog,
|
|
8
8
|
daytonaErrorStatusForLog,
|
|
9
|
+
daytonaProviderDetailForRunError,
|
|
10
|
+
daytonaProviderMessageForRunError,
|
|
11
|
+
daytonaProviderTextForRunError,
|
|
9
12
|
daytonaSandboxStateForLog,
|
|
13
|
+
isDaytonaProviderError,
|
|
10
14
|
} from './daytona-launch-transport';
|
|
11
15
|
import type {
|
|
12
16
|
PlayRunnerExecutionConfig,
|
|
@@ -349,7 +353,9 @@ async function rejectAcquiredSandbox(
|
|
|
349
353
|
try {
|
|
350
354
|
await deleteSandbox();
|
|
351
355
|
} catch (error) {
|
|
352
|
-
const cleanupError =
|
|
356
|
+
const cleanupError = daytonaProviderTextForRunError(
|
|
357
|
+
error instanceof Error ? error.message : String(error),
|
|
358
|
+
);
|
|
353
359
|
throw new Error(
|
|
354
360
|
`${reason} Defensive deletion of Daytona sandbox ${sandbox.id} also failed: ${cleanupError}`,
|
|
355
361
|
{ cause: error },
|
|
@@ -672,6 +678,8 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
672
678
|
cancellationSignal?: AbortSignal;
|
|
673
679
|
}): Promise<DaytonaCreateResult> {
|
|
674
680
|
const errors: string[] = [];
|
|
681
|
+
// Customers read these; acquisition classification reads the raw `errors`.
|
|
682
|
+
const runErrors: string[] = [];
|
|
675
683
|
for (const [index, delayMs] of DAYTONA_CREATE_RETRY_DELAYS_MS.entries()) {
|
|
676
684
|
input.cancellationSignal?.throwIfAborted();
|
|
677
685
|
const attempt = input.nextProviderAttempt();
|
|
@@ -767,6 +775,11 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
767
775
|
errorClass: daytonaCreateErrorClass(error),
|
|
768
776
|
});
|
|
769
777
|
errors.push(message);
|
|
778
|
+
runErrors.push(
|
|
779
|
+
isDaytonaProviderError(error)
|
|
780
|
+
? `${daytonaProviderMessageForRunError(error) ?? ''}${daytonaProviderDetailForRunError(error)}`.trim()
|
|
781
|
+
: (daytonaProviderTextForRunError(message) ?? ''),
|
|
782
|
+
);
|
|
770
783
|
// Log-only: whether the create request returned before the deadline and
|
|
771
784
|
// what Daytona last reported, so an unknown outcome is attributable.
|
|
772
785
|
const launchOutcome =
|
|
@@ -878,7 +891,7 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
878
891
|
throw reconcilePending();
|
|
879
892
|
}
|
|
880
893
|
throw new Error(
|
|
881
|
-
`${message} Daytona did not return a cleanup-addressable sandbox identity; Modal fallback suppressed. The named ephemeral sandbox retains Daytona's auto-delete-on-stop backstop.`,
|
|
894
|
+
`${daytonaProviderTextForRunError(message)} Daytona did not return a cleanup-addressable sandbox identity; Modal fallback suppressed. The named ephemeral sandbox retains Daytona's auto-delete-on-stop backstop.`,
|
|
882
895
|
{ cause: error },
|
|
883
896
|
);
|
|
884
897
|
}
|
|
@@ -926,7 +939,7 @@ async function createRetriedOneShotDaytonaSandbox(input: {
|
|
|
926
939
|
...(reservation ? { sandboxCapacityLeaseId: reservation.leaseId } : {}),
|
|
927
940
|
};
|
|
928
941
|
}
|
|
929
|
-
const message = `Daytona sandbox create failed across ${errors.length} bounded attempts: ${
|
|
942
|
+
const message = `Daytona sandbox create failed across ${errors.length} bounded attempts: ${runErrors.join('; ')}`;
|
|
930
943
|
const fallbackReason =
|
|
931
944
|
resolveDaytonaSandboxAcquisitionUnavailableReason(errors);
|
|
932
945
|
if (fallbackReason) {
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
|
+
boundedDaytonaProviderTextForLog,
|
|
2
3
|
createDaytonaLaunchClient,
|
|
4
|
+
daytonaErrorCodeForLog,
|
|
5
|
+
daytonaErrorStatusForLog,
|
|
6
|
+
daytonaProviderDetailForRunError,
|
|
7
|
+
daytonaProviderMessageForRunError,
|
|
8
|
+
isDaytonaProviderError,
|
|
3
9
|
redactDaytonaErrorDetail,
|
|
10
|
+
withoutDaytonaProviderDetail,
|
|
4
11
|
} from './daytona-launch-transport';
|
|
5
12
|
import type { DaytonaClientOptions } from '../../daytona-runtime-config';
|
|
6
13
|
import {
|
|
@@ -1112,7 +1119,7 @@ export async function deleteDaytonaSandboxById(input: {
|
|
|
1112
1119
|
return outcome.kind === 'deleted' || outcome.kind === 'already_absent';
|
|
1113
1120
|
}
|
|
1114
1121
|
|
|
1115
|
-
function formatDaytonaError(error: unknown): string {
|
|
1122
|
+
export function formatDaytonaError(error: unknown): string {
|
|
1116
1123
|
if (!(error instanceof Error)) {
|
|
1117
1124
|
return String(error);
|
|
1118
1125
|
}
|
|
@@ -1122,11 +1129,20 @@ function formatDaytonaError(error: unknown): string {
|
|
|
1122
1129
|
error.cause && error.cause !== error
|
|
1123
1130
|
? ` cause=${formatDaytonaError(error.cause)}`
|
|
1124
1131
|
: '';
|
|
1132
|
+
const message = isDaytonaProviderError(error)
|
|
1133
|
+
? daytonaProviderMessageForRunError(error)
|
|
1134
|
+
: error.message;
|
|
1125
1135
|
const details =
|
|
1126
1136
|
error.name && error.name !== 'Error'
|
|
1127
|
-
? [error.name,
|
|
1128
|
-
:
|
|
1129
|
-
return `${details || 'Error'}${code}${cause}`;
|
|
1137
|
+
? [error.name, message].filter(Boolean).join(': ')
|
|
1138
|
+
: message || error.name || 'Error';
|
|
1139
|
+
return `${details || 'Error'}${code}${daytonaProviderDetailForRunError(error)}${cause}`;
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
/** A thrown error's message for a run error, keeping provider text bounded. */
|
|
1143
|
+
export function runErrorMessage(error: unknown): string {
|
|
1144
|
+
if (isDaytonaProviderError(error)) return formatDaytonaError(error);
|
|
1145
|
+
return error instanceof Error ? error.message : String(error);
|
|
1130
1146
|
}
|
|
1131
1147
|
|
|
1132
1148
|
function sleep(ms: number): Promise<void> {
|
|
@@ -1264,38 +1280,39 @@ export function retryableRuntimeInitializationFailureReason(
|
|
|
1264
1280
|
| 'daytona_infrastructure'
|
|
1265
1281
|
| null {
|
|
1266
1282
|
if (result.status !== 'failed') return null;
|
|
1267
|
-
|
|
1283
|
+
const error = withoutDaytonaProviderDetail(result.error);
|
|
1284
|
+
if (RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN.test(error)) {
|
|
1268
1285
|
return 'runtime_postgres_connect';
|
|
1269
1286
|
}
|
|
1270
|
-
if (RUNTIME_RECEIPT_GATEWAY_TRANSPORT_RETRY_PATTERN.test(
|
|
1287
|
+
if (RUNTIME_RECEIPT_GATEWAY_TRANSPORT_RETRY_PATTERN.test(error)) {
|
|
1271
1288
|
return 'runtime_receipt_gateway_transport';
|
|
1272
1289
|
}
|
|
1273
1290
|
const rowsProcessed = Number(result.stats?.rowsProcessed ?? 0);
|
|
1274
1291
|
if (
|
|
1275
1292
|
rowsProcessed === 0 &&
|
|
1276
1293
|
result.steps.length === 0 &&
|
|
1277
|
-
DAYTONA_RUNNER_STARTUP_TRANSPORT_RETRY_PATTERN.test(
|
|
1294
|
+
DAYTONA_RUNNER_STARTUP_TRANSPORT_RETRY_PATTERN.test(error)
|
|
1278
1295
|
) {
|
|
1279
1296
|
return 'daytona_runner_startup_transport';
|
|
1280
1297
|
}
|
|
1281
1298
|
if (
|
|
1282
1299
|
rowsProcessed === 0 &&
|
|
1283
1300
|
result.steps.length === 0 &&
|
|
1284
|
-
DAYTONA_PUBLIC_STATUS_TRANSIENT_RETRY_PATTERN.test(
|
|
1301
|
+
DAYTONA_PUBLIC_STATUS_TRANSIENT_RETRY_PATTERN.test(error)
|
|
1285
1302
|
) {
|
|
1286
1303
|
return 'daytona_public_status';
|
|
1287
1304
|
}
|
|
1288
1305
|
if (
|
|
1289
1306
|
rowsProcessed === 0 &&
|
|
1290
1307
|
result.steps.length === 0 &&
|
|
1291
|
-
RUNTIME_API_TRANSPORT_RETRY_PATTERN.test(
|
|
1308
|
+
RUNTIME_API_TRANSPORT_RETRY_PATTERN.test(error)
|
|
1292
1309
|
) {
|
|
1293
1310
|
return 'runtime_api_transport';
|
|
1294
1311
|
}
|
|
1295
1312
|
if (
|
|
1296
1313
|
rowsProcessed === 0 &&
|
|
1297
1314
|
result.steps.length === 0 &&
|
|
1298
|
-
DAYTONA_INFRASTRUCTURE_RETRY_PATTERN.test(
|
|
1315
|
+
DAYTONA_INFRASTRUCTURE_RETRY_PATTERN.test(error)
|
|
1299
1316
|
) {
|
|
1300
1317
|
return 'daytona_infrastructure';
|
|
1301
1318
|
}
|
|
@@ -1916,6 +1933,13 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1916
1933
|
runnerAttempt: Math.max(0, Math.floor(config.context.runAttempt ?? 0)),
|
|
1917
1934
|
error: formatDaytonaError(error),
|
|
1918
1935
|
operation: 'runner_backend',
|
|
1936
|
+
// Operator evidence: the run error withholds some of this from customers.
|
|
1937
|
+
providerStatus: daytonaErrorStatusForLog(error),
|
|
1938
|
+
providerCode: daytonaErrorCodeForLog(error),
|
|
1939
|
+
providerText:
|
|
1940
|
+
error instanceof Error
|
|
1941
|
+
? boundedDaytonaProviderTextForLog(error.message)
|
|
1942
|
+
: null,
|
|
1919
1943
|
elapsedMs: Date.now() - startedAt,
|
|
1920
1944
|
});
|
|
1921
1945
|
return withCleanup(
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts
CHANGED
|
@@ -92,7 +92,10 @@ import {
|
|
|
92
92
|
resolveRuntimeSandboxPlacementBackend,
|
|
93
93
|
type PlayRunnerBackend,
|
|
94
94
|
} from '../runner-backends';
|
|
95
|
-
import {
|
|
95
|
+
import {
|
|
96
|
+
retryableRuntimeInitializationFailureReason,
|
|
97
|
+
runErrorMessage,
|
|
98
|
+
} from '../runner-backends/backends/daytona';
|
|
96
99
|
import {
|
|
97
100
|
isRuntimeSandboxCanaryProduction,
|
|
98
101
|
resolveLaunchRuntimeSandboxPlacementPolicy,
|
|
@@ -2321,12 +2324,10 @@ async function terminalizeSandboxCapacityRejection(input: {
|
|
|
2321
2324
|
limit: input.error.limit,
|
|
2322
2325
|
});
|
|
2323
2326
|
const terminal = await input.deps.runWithClient((client) =>
|
|
2324
|
-
executePostgresSchedulerClaim(
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
input.options,
|
|
2329
|
-
),
|
|
2327
|
+
executePostgresSchedulerClaim(client, claim, async () => failedResult, {
|
|
2328
|
+
...input.options,
|
|
2329
|
+
terminalLogOutbox: 'failure_reason',
|
|
2330
|
+
}),
|
|
2330
2331
|
);
|
|
2331
2332
|
if (terminal.status === 'failed') {
|
|
2332
2333
|
const reporter = input.deps.createProgressReporter(
|
|
@@ -3261,7 +3262,9 @@ async function executeAssignedAbsurdAttempt(input: {
|
|
|
3261
3262
|
async () => runnerResult,
|
|
3262
3263
|
{
|
|
3263
3264
|
...options,
|
|
3264
|
-
terminalLogOutbox: input.fromDetachedRunner
|
|
3265
|
+
terminalLogOutbox: input.fromDetachedRunner
|
|
3266
|
+
? 'ctx_log_tail'
|
|
3267
|
+
: 'failure_reason',
|
|
3265
3268
|
sandboxCrashDiagnostic,
|
|
3266
3269
|
...(observedTerminalProgressEvents.length > 0 ||
|
|
3267
3270
|
failedTerminalProgressEvents.length > 0
|
|
@@ -5686,8 +5689,7 @@ async function salvageAbsurdFinalAttemptTerminal(input: {
|
|
|
5686
5689
|
);
|
|
5687
5690
|
}
|
|
5688
5691
|
if (!input.force && attempt < maxAttempts) return;
|
|
5689
|
-
const message =
|
|
5690
|
-
input.error instanceof Error ? input.error.message : String(input.error);
|
|
5692
|
+
const message = runErrorMessage(input.error);
|
|
5691
5693
|
const failedResult: PlayRunnerResult = {
|
|
5692
5694
|
status: 'failed',
|
|
5693
5695
|
error:
|
|
@@ -5757,10 +5759,11 @@ async function salvageAbsurdFinalAttemptTerminal(input: {
|
|
|
5757
5759
|
input.detachedRecovery
|
|
5758
5760
|
? {
|
|
5759
5761
|
...input.options,
|
|
5762
|
+
terminalLogOutbox: 'failure_reason',
|
|
5760
5763
|
preserveRunnerTerminal: true,
|
|
5761
5764
|
protectWaitingSuspensionFailure,
|
|
5762
5765
|
}
|
|
5763
|
-
: input.options,
|
|
5766
|
+
: { ...input.options, terminalLogOutbox: 'failure_reason' },
|
|
5764
5767
|
);
|
|
5765
5768
|
});
|
|
5766
5769
|
break;
|
|
@@ -137,6 +137,7 @@ import {
|
|
|
137
137
|
isCtxFetchHttpFailureDiagnostic,
|
|
138
138
|
parseCtxFetchHttpFailureDiagnostic,
|
|
139
139
|
readProvenanceTag,
|
|
140
|
+
tagLogEntry,
|
|
140
141
|
type CtxFetchHttpFailureDiagnostic,
|
|
141
142
|
} from '../log-provenance';
|
|
142
143
|
import {
|
|
@@ -473,7 +474,10 @@ export type PostgresSchedulerAttemptExecutor = (
|
|
|
473
474
|
|
|
474
475
|
type PostgresSchedulerCompletionLogMetadata = 'attach' | 'omit';
|
|
475
476
|
|
|
476
|
-
type PostgresSchedulerTerminalLogOutbox =
|
|
477
|
+
type PostgresSchedulerTerminalLogOutbox =
|
|
478
|
+
| 'omit'
|
|
479
|
+
| 'ctx_log_tail'
|
|
480
|
+
| 'failure_reason';
|
|
477
481
|
|
|
478
482
|
export type PostgresSchedulerSandboxCrashDiagnosticRequest = {
|
|
479
483
|
schemaVersion: 1;
|
|
@@ -506,7 +510,12 @@ type PostgresSchedulerClaimExecutionOptions = Pick<
|
|
|
506
510
|
* Detached runners push their complete result into Postgres before a worker
|
|
507
511
|
* wakes to finalize it. Put their authored log tail in the terminal outbox
|
|
508
512
|
* so a worker can exit immediately without losing that tail to a detached
|
|
509
|
-
* HTTP projection.
|
|
513
|
+
* HTTP projection. `failure_reason` is for a failure no runner delivered: a
|
|
514
|
+
* failed run gets one error line stating why, so a run that failed before
|
|
515
|
+
* Play code started has a reason in its logs. Never use it for a runner's own
|
|
516
|
+
* terminal. An empty `logs` there proves nothing, because the accept trigger
|
|
517
|
+
* moves a pushed tail out of the result, and the CLI shows the last log line
|
|
518
|
+
* as the customer's recent logs.
|
|
510
519
|
*/
|
|
511
520
|
terminalLogOutbox?: PostgresSchedulerTerminalLogOutbox;
|
|
512
521
|
/** Internal provider-read coordinates which must survive terminal-worker
|
|
@@ -15120,6 +15129,36 @@ function sanitizeTerminalLogBody(line: string): string {
|
|
|
15120
15129
|
return sanitizeLedgerLogLine(body, !runtimeDiagnostic);
|
|
15121
15130
|
}
|
|
15122
15131
|
|
|
15132
|
+
/**
|
|
15133
|
+
* One error line stating why a run failed. `message` must be the public failure
|
|
15134
|
+
* text. The line ID keeps it out of the envelope's position space, where a
|
|
15135
|
+
* pushed runner tail may already own position 0: a different line there would
|
|
15136
|
+
* abort the terminal write and strand the run.
|
|
15137
|
+
*/
|
|
15138
|
+
function terminalFailureReasonLogProjection(input: {
|
|
15139
|
+
attempt: number;
|
|
15140
|
+
message: string;
|
|
15141
|
+
}): {
|
|
15142
|
+
lines: string[];
|
|
15143
|
+
lineContexts: Array<PlayLogContext | null>;
|
|
15144
|
+
lineMetadata: Array<PlayLogMetadata | null>;
|
|
15145
|
+
lineIds: Array<string | null>;
|
|
15146
|
+
} {
|
|
15147
|
+
const occurredAt = Date.now();
|
|
15148
|
+
return {
|
|
15149
|
+
lines: [
|
|
15150
|
+
tagLogEntry(
|
|
15151
|
+
'diagnostic',
|
|
15152
|
+
'error',
|
|
15153
|
+
`[${new Date(occurredAt).toISOString()}] [runtime.failure] ${input.message}`,
|
|
15154
|
+
),
|
|
15155
|
+
],
|
|
15156
|
+
lineContexts: [null],
|
|
15157
|
+
lineMetadata: [{ level: 'error', occurredAt }],
|
|
15158
|
+
lineIds: [`runtime-failure:${input.attempt}`],
|
|
15159
|
+
};
|
|
15160
|
+
}
|
|
15161
|
+
|
|
15123
15162
|
function terminalLogProjectionForOutbox(
|
|
15124
15163
|
runnerResult: unknown,
|
|
15125
15164
|
mode: PostgresSchedulerTerminalLogOutbox | undefined,
|
|
@@ -15512,10 +15551,19 @@ export async function executePostgresSchedulerClaim(
|
|
|
15512
15551
|
checkpoint: null,
|
|
15513
15552
|
tableNamespace: null,
|
|
15514
15553
|
};
|
|
15515
|
-
const
|
|
15516
|
-
|
|
15517
|
-
|
|
15518
|
-
|
|
15554
|
+
const publicFailure = terminalFailureResult.errors?.[0]?.message;
|
|
15555
|
+
const terminalLogProjection: ReturnType<
|
|
15556
|
+
typeof terminalLogProjectionForOutbox
|
|
15557
|
+
> =
|
|
15558
|
+
options.terminalLogOutbox === 'failure_reason'
|
|
15559
|
+
? terminalFailureReasonLogProjection({
|
|
15560
|
+
attempt: claim.attempt,
|
|
15561
|
+
message: publicFailure?.trim() ? publicFailure : message,
|
|
15562
|
+
})
|
|
15563
|
+
: terminalLogProjectionForOutbox(
|
|
15564
|
+
terminalFailureResult,
|
|
15565
|
+
options.terminalLogOutbox,
|
|
15566
|
+
);
|
|
15519
15567
|
const failed = await failPostgresSchedulerAttempt(
|
|
15520
15568
|
client,
|
|
15521
15569
|
{
|