deepline 0.1.211 → 0.1.212
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +150 -11
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +23 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +1 -1
- package/dist/bundling-sources/sdk/src/client.ts +118 -3
- package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +253 -30
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +22 -20
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts +10 -4
- package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +3 -5
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +14 -2
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +4 -2
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +21 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +128 -6
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +37 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +19 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +63 -7
- package/dist/bundling-sources/shared_libs/play-runtime/secret-capability.ts +18 -4
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +29 -9
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +22 -1
- package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +79 -1
- package/dist/cli/index.js +637 -229
- package/dist/cli/index.mjs +637 -229
- package/dist/index.d.mts +76 -6
- package/dist/index.d.ts +76 -6
- package/dist/index.js +225 -43
- package/dist/index.mjs +225 -43
- package/dist/plays/bundle-play-file.mjs +65 -4
- package/package.json +1 -1
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
nextPlayRunLedgerTerminalSource,
|
|
8
8
|
normalizePlayRunLedgerTerminalSource,
|
|
9
9
|
} from './run-terminal-source';
|
|
10
|
+
import { MAX_LEDGER_LOG_LINES_PER_EVENT } from './ledger-safe-payload';
|
|
10
11
|
|
|
11
12
|
export type PlayRunLedgerStatus =
|
|
12
13
|
| 'queued'
|
|
@@ -84,6 +85,18 @@ export type PlayRunLedgerStepSnapshot = {
|
|
|
84
85
|
progress?: PlayRunLedgerStepProgress | null;
|
|
85
86
|
};
|
|
86
87
|
|
|
88
|
+
export type PlayRunLedgerDatasetSnapshot = {
|
|
89
|
+
datasetId: string;
|
|
90
|
+
path: string;
|
|
91
|
+
tableNamespace: string;
|
|
92
|
+
phase: 'registered' | 'available' | 'failed';
|
|
93
|
+
persistedRows: number;
|
|
94
|
+
succeededRows: number;
|
|
95
|
+
failedRows: number;
|
|
96
|
+
complete: boolean;
|
|
97
|
+
updatedAt: number;
|
|
98
|
+
};
|
|
99
|
+
|
|
87
100
|
export type PlayRunLedgerSnapshot = {
|
|
88
101
|
runId: string;
|
|
89
102
|
playName?: string | null;
|
|
@@ -96,6 +109,8 @@ export type PlayRunLedgerSnapshot = {
|
|
|
96
109
|
durationMs?: number | null;
|
|
97
110
|
orderedStepIds: string[];
|
|
98
111
|
stepsById: Record<string, PlayRunLedgerStepSnapshot>;
|
|
112
|
+
orderedDatasetIds: string[];
|
|
113
|
+
datasetsById: Record<string, PlayRunLedgerDatasetSnapshot>;
|
|
99
114
|
/**
|
|
100
115
|
* Bounded tail of the run's log lines (last {@link LOG_TAIL_LIMIT}).
|
|
101
116
|
* Full retention lives in the Run Log Stream (Convex `playRunLogChunks`);
|
|
@@ -217,10 +232,34 @@ export type PlayRunLedgerEvent =
|
|
|
217
232
|
* repeated identical lines while absorbing redundant re-sends.
|
|
218
233
|
*/
|
|
219
234
|
channelOffset?: number;
|
|
235
|
+
/** Scheduler attempt that owns this positional log channel. */
|
|
236
|
+
producerAttempt?: number;
|
|
237
|
+
/**
|
|
238
|
+
* Terminal transport replay normally matches ctx.log lines after
|
|
239
|
+
* removing timestamps. Detached-attempt recovery uses exact matching so
|
|
240
|
+
* the same message from a later attempt remains a distinct occurrence.
|
|
241
|
+
*/
|
|
242
|
+
transportDedupe?: 'semantic' | 'exact';
|
|
220
243
|
/** Set by ingestion when this append crossed the retention cap. */
|
|
221
244
|
logsTruncated?: boolean;
|
|
222
245
|
})
|
|
223
246
|
| (PlayRunLedgerBaseEvent & {
|
|
247
|
+
type: 'dataset.lifecycle';
|
|
248
|
+
datasetId: string;
|
|
249
|
+
path: string;
|
|
250
|
+
tableNamespace: string;
|
|
251
|
+
phase: 'registered' | 'available' | 'failed';
|
|
252
|
+
persistedRows?: number;
|
|
253
|
+
succeededRows?: number;
|
|
254
|
+
failedRows?: number;
|
|
255
|
+
complete?: boolean;
|
|
256
|
+
})
|
|
257
|
+
| (PlayRunLedgerBaseEvent & {
|
|
258
|
+
/**
|
|
259
|
+
* @deprecated Mixed-version deploy seam only. Owner: Play Runtime.
|
|
260
|
+
* Remove after 2026-07-19, once pre-dataset-lifecycle workers have
|
|
261
|
+
* exceeded the maximum run duration plus the seven-day drain window.
|
|
262
|
+
*/
|
|
224
263
|
type: 'sheet.summary.updated';
|
|
225
264
|
tableNamespace: string;
|
|
226
265
|
deltaCursor?: number;
|
|
@@ -241,6 +280,7 @@ export type PlayRunLedgerStatusPatch = {
|
|
|
241
280
|
* (see `slicePositionalLogLines`), forwarded onto the `log.appended` event.
|
|
242
281
|
*/
|
|
243
282
|
liveLogsChannelOffset?: number | null;
|
|
283
|
+
liveLogsProducerAttempt?: number | null;
|
|
244
284
|
liveNodeProgress?: unknown;
|
|
245
285
|
result?: unknown;
|
|
246
286
|
};
|
|
@@ -401,6 +441,8 @@ export function createEmptyPlayRunLedgerSnapshot(input: {
|
|
|
401
441
|
: null,
|
|
402
442
|
orderedStepIds: [],
|
|
403
443
|
stepsById: {},
|
|
444
|
+
orderedDatasetIds: [],
|
|
445
|
+
datasetsById: {},
|
|
404
446
|
logTail: [],
|
|
405
447
|
totalLogCount: 0,
|
|
406
448
|
logsTruncated: false,
|
|
@@ -447,6 +489,26 @@ export function normalizePlayRunLedgerSnapshot(
|
|
|
447
489
|
progress: rawProgress ? normalizeStepProgress(rawProgress) : null,
|
|
448
490
|
};
|
|
449
491
|
}
|
|
492
|
+
const rawDatasets = isRecord(value.datasetsById) ? value.datasetsById : {};
|
|
493
|
+
const datasetsById: Record<string, PlayRunLedgerDatasetSnapshot> = {};
|
|
494
|
+
for (const [datasetId, rawDataset] of Object.entries(rawDatasets)) {
|
|
495
|
+
if (!datasetId.trim() || !isRecord(rawDataset)) continue;
|
|
496
|
+
const phase =
|
|
497
|
+
rawDataset.phase === 'available' || rawDataset.phase === 'failed'
|
|
498
|
+
? rawDataset.phase
|
|
499
|
+
: 'registered';
|
|
500
|
+
datasetsById[datasetId] = {
|
|
501
|
+
datasetId,
|
|
502
|
+
path: optionalString(rawDataset.path) ?? `datasets.${datasetId}`,
|
|
503
|
+
tableNamespace: optionalString(rawDataset.tableNamespace) ?? datasetId,
|
|
504
|
+
phase,
|
|
505
|
+
persistedRows: Math.max(0, finiteNumber(rawDataset.persistedRows) ?? 0),
|
|
506
|
+
succeededRows: Math.max(0, finiteNumber(rawDataset.succeededRows) ?? 0),
|
|
507
|
+
failedRows: Math.max(0, finiteNumber(rawDataset.failedRows) ?? 0),
|
|
508
|
+
complete: rawDataset.complete === true,
|
|
509
|
+
updatedAt: finiteNumber(rawDataset.updatedAt) ?? 0,
|
|
510
|
+
};
|
|
511
|
+
}
|
|
450
512
|
const createdAt = finiteNumber(value.createdAt) ?? fallback.createdAt ?? null;
|
|
451
513
|
const startedAt = finiteNumber(value.startedAt) ?? fallback.startedAt ?? null;
|
|
452
514
|
const finishedAt =
|
|
@@ -484,6 +546,14 @@ export function normalizePlayRunLedgerSnapshot(
|
|
|
484
546
|
: finiteNumber(value.durationMs),
|
|
485
547
|
orderedStepIds: orderedStepIds.filter((stepId) => stepsById[stepId]),
|
|
486
548
|
stepsById,
|
|
549
|
+
orderedDatasetIds: (Array.isArray(value.orderedDatasetIds)
|
|
550
|
+
? value.orderedDatasetIds
|
|
551
|
+
: Object.keys(datasetsById)
|
|
552
|
+
).filter(
|
|
553
|
+
(datasetId): datasetId is string =>
|
|
554
|
+
typeof datasetId === 'string' && Boolean(datasetsById[datasetId]),
|
|
555
|
+
),
|
|
556
|
+
datasetsById,
|
|
487
557
|
logTail: logTail.slice(-LOG_TAIL_LIMIT),
|
|
488
558
|
// Snapshots persisted before totalLogCount existed only know the retained
|
|
489
559
|
// tail, so the best lower bound for the cumulative count is the tail size.
|
|
@@ -1269,6 +1339,34 @@ export function reducePlayRunLedgerEvent(
|
|
|
1269
1339
|
: null,
|
|
1270
1340
|
});
|
|
1271
1341
|
}
|
|
1342
|
+
case 'dataset.lifecycle': {
|
|
1343
|
+
const current = base.datasetsById[event.datasetId];
|
|
1344
|
+
const next: PlayRunLedgerDatasetSnapshot = {
|
|
1345
|
+
datasetId: event.datasetId,
|
|
1346
|
+
path: event.path,
|
|
1347
|
+
tableNamespace: event.tableNamespace,
|
|
1348
|
+
phase: event.phase,
|
|
1349
|
+
persistedRows: Math.max(
|
|
1350
|
+
current?.persistedRows ?? 0,
|
|
1351
|
+
event.persistedRows ?? 0,
|
|
1352
|
+
),
|
|
1353
|
+
succeededRows: Math.max(
|
|
1354
|
+
current?.succeededRows ?? 0,
|
|
1355
|
+
event.succeededRows ?? 0,
|
|
1356
|
+
),
|
|
1357
|
+
failedRows: Math.max(current?.failedRows ?? 0, event.failedRows ?? 0),
|
|
1358
|
+
complete: current?.complete === true || event.complete === true,
|
|
1359
|
+
updatedAt: occurredAt,
|
|
1360
|
+
};
|
|
1361
|
+
return withTiming({
|
|
1362
|
+
...base,
|
|
1363
|
+
orderedDatasetIds: current
|
|
1364
|
+
? base.orderedDatasetIds
|
|
1365
|
+
: [...base.orderedDatasetIds, event.datasetId],
|
|
1366
|
+
datasetsById: { ...base.datasetsById, [event.datasetId]: next },
|
|
1367
|
+
resultTableNamespace: base.resultTableNamespace ?? event.tableNamespace,
|
|
1368
|
+
});
|
|
1369
|
+
}
|
|
1272
1370
|
case 'sheet.summary.updated':
|
|
1273
1371
|
return withTiming({
|
|
1274
1372
|
...base,
|
|
@@ -1376,6 +1474,8 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
|
|
|
1376
1474
|
source?: PlayRunLedgerEventSource;
|
|
1377
1475
|
occurredAt?: number;
|
|
1378
1476
|
channelOffset?: number | null;
|
|
1477
|
+
producerAttempt?: number | null;
|
|
1478
|
+
transportDedupe?: 'semantic' | 'exact';
|
|
1379
1479
|
includeLogAppend?: boolean;
|
|
1380
1480
|
}): PlayRunLedgerEvent[] {
|
|
1381
1481
|
const source = input.source ?? 'worker';
|
|
@@ -1392,16 +1492,31 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
|
|
|
1392
1492
|
if (newLines.length === 0 || input.includeLogAppend === false) {
|
|
1393
1493
|
return [];
|
|
1394
1494
|
}
|
|
1395
|
-
|
|
1396
|
-
|
|
1495
|
+
const events: PlayRunLedgerEvent[] = [];
|
|
1496
|
+
for (
|
|
1497
|
+
let index = 0;
|
|
1498
|
+
index < newLines.length;
|
|
1499
|
+
index += MAX_LEDGER_LOG_LINES_PER_EVENT
|
|
1500
|
+
) {
|
|
1501
|
+
events.push({
|
|
1397
1502
|
type: 'log.appended',
|
|
1398
1503
|
runId: input.runId,
|
|
1399
1504
|
source,
|
|
1400
1505
|
occurredAt,
|
|
1401
|
-
lines: newLines,
|
|
1402
|
-
...(positional ? { channelOffset: input.channelOffset! } : {}),
|
|
1403
|
-
|
|
1404
|
-
|
|
1506
|
+
lines: newLines.slice(index, index + MAX_LEDGER_LOG_LINES_PER_EVENT),
|
|
1507
|
+
...(positional ? { channelOffset: input.channelOffset! + index } : {}),
|
|
1508
|
+
...(positional &&
|
|
1509
|
+
typeof input.producerAttempt === 'number' &&
|
|
1510
|
+
Number.isFinite(input.producerAttempt) &&
|
|
1511
|
+
input.producerAttempt >= 0
|
|
1512
|
+
? { producerAttempt: Math.trunc(input.producerAttempt) }
|
|
1513
|
+
: {}),
|
|
1514
|
+
...(input.transportDedupe
|
|
1515
|
+
? { transportDedupe: input.transportDedupe }
|
|
1516
|
+
: {}),
|
|
1517
|
+
});
|
|
1518
|
+
}
|
|
1519
|
+
return events;
|
|
1405
1520
|
}
|
|
1406
1521
|
|
|
1407
1522
|
/**
|
|
@@ -1419,6 +1534,8 @@ export function buildTerminalLogReplayEvents(input: {
|
|
|
1419
1534
|
source?: PlayRunLedgerEventSource;
|
|
1420
1535
|
occurredAt?: number;
|
|
1421
1536
|
liveLogTotalCount?: number;
|
|
1537
|
+
dedupeMode?: 'semantic' | 'exact';
|
|
1538
|
+
producerAttempt?: number;
|
|
1422
1539
|
}): PlayRunLedgerEvent[] {
|
|
1423
1540
|
const channelOffset = terminalLogReplayChannelOffset({
|
|
1424
1541
|
liveLogTotalCount: input.liveLogTotalCount,
|
|
@@ -1430,6 +1547,10 @@ export function buildTerminalLogReplayEvents(input: {
|
|
|
1430
1547
|
source: 'coordinator',
|
|
1431
1548
|
occurredAt: input.occurredAt,
|
|
1432
1549
|
channelOffset,
|
|
1550
|
+
producerAttempt: input.producerAttempt,
|
|
1551
|
+
transportDedupe:
|
|
1552
|
+
input.dedupeMode ??
|
|
1553
|
+
(typeof input.producerAttempt === 'number' ? 'exact' : undefined),
|
|
1433
1554
|
});
|
|
1434
1555
|
}
|
|
1435
1556
|
|
|
@@ -1466,6 +1587,7 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
|
|
|
1466
1587
|
source,
|
|
1467
1588
|
occurredAt: checkpointAt,
|
|
1468
1589
|
channelOffset: patch.liveLogsChannelOffset ?? null,
|
|
1590
|
+
producerAttempt: patch.liveLogsProducerAttempt ?? null,
|
|
1469
1591
|
})
|
|
1470
1592
|
: [];
|
|
1471
1593
|
const earliestStartedAt =
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
normalizePlayRunLedgerSnapshot,
|
|
5
5
|
summarizeRunRowOutcomes,
|
|
6
6
|
type PlayRunLedgerSnapshot,
|
|
7
|
+
type PlayRunLedgerDatasetSnapshot,
|
|
7
8
|
type PlayRunLedgerStepSnapshot,
|
|
8
9
|
type PlayRunRowOutcomeSummary,
|
|
9
10
|
} from './run-ledger';
|
|
@@ -76,6 +77,7 @@ export type PlayRunLiveSnapshot = {
|
|
|
76
77
|
logsTruncated?: boolean;
|
|
77
78
|
activeArtifactTableNamespace: string | null;
|
|
78
79
|
resultTableNamespace: string | null;
|
|
80
|
+
datasets?: PlayRunLedgerDatasetSnapshot[];
|
|
79
81
|
nodeStates: PlayRunStreamNodeState[];
|
|
80
82
|
activeNodeId: string | null;
|
|
81
83
|
/**
|
|
@@ -212,6 +214,11 @@ function buildSnapshotFromLedger(
|
|
|
212
214
|
...(snapshot.logsTruncated ? { logsTruncated: true } : {}),
|
|
213
215
|
activeArtifactTableNamespace: snapshot.activeArtifactTableNamespace ?? null,
|
|
214
216
|
resultTableNamespace: snapshot.resultTableNamespace ?? null,
|
|
217
|
+
datasets: snapshot.orderedDatasetIds
|
|
218
|
+
.map((datasetId) => snapshot.datasetsById[datasetId])
|
|
219
|
+
.filter((dataset): dataset is PlayRunLedgerDatasetSnapshot =>
|
|
220
|
+
Boolean(dataset),
|
|
221
|
+
),
|
|
215
222
|
nodeStates,
|
|
216
223
|
activeNodeId: snapshot.activeStepId ?? null,
|
|
217
224
|
...(rowOutcomes ? { rowOutcomes } : {}),
|
|
@@ -122,19 +122,34 @@ export async function inspectDetachedDaytonaRunner(input: {
|
|
|
122
122
|
cmdId: string;
|
|
123
123
|
outputPath: string;
|
|
124
124
|
exitCodePath: string;
|
|
125
|
-
}): Promise<{
|
|
125
|
+
}): Promise<{
|
|
126
|
+
exitCode: number | null;
|
|
127
|
+
result: PlayRunnerResult | null;
|
|
128
|
+
diagnosis: {
|
|
129
|
+
stage:
|
|
130
|
+
| 'salvaged'
|
|
131
|
+
| 'result_missing'
|
|
132
|
+
| 'command_inspection_failed'
|
|
133
|
+
| 'sandbox_lookup_failed';
|
|
134
|
+
detail: string | null;
|
|
135
|
+
};
|
|
136
|
+
}> {
|
|
126
137
|
try {
|
|
127
138
|
const { clientOptions } = loadDaytonaRequiredConfig();
|
|
128
139
|
const daytona = daytonaSdkClientFactory.createFull(clientOptions);
|
|
129
140
|
const sandbox = (await daytona.get(input.sandboxId)) as DaytonaSandbox;
|
|
130
141
|
let exitCode: number | null = null;
|
|
142
|
+
let commandInspectionError: string | null = null;
|
|
131
143
|
try {
|
|
132
144
|
const command = await sandbox.process.getSessionCommand(
|
|
133
145
|
input.sessionId,
|
|
134
146
|
input.cmdId,
|
|
135
147
|
);
|
|
136
|
-
exitCode =
|
|
148
|
+
exitCode =
|
|
149
|
+
typeof command?.exitCode === 'number' ? command.exitCode : null;
|
|
137
150
|
} catch (error) {
|
|
151
|
+
commandInspectionError =
|
|
152
|
+
error instanceof Error ? error.message : String(error);
|
|
138
153
|
console.warn('[play-runner.daytona.detached_inspect_command_failed]', {
|
|
139
154
|
sandboxId: input.sandboxId,
|
|
140
155
|
cmdId: input.cmdId,
|
|
@@ -148,7 +163,16 @@ export async function inspectDetachedDaytonaRunner(input: {
|
|
|
148
163
|
timeoutMs: 10_000,
|
|
149
164
|
});
|
|
150
165
|
if (!recovered.execution) {
|
|
151
|
-
return {
|
|
166
|
+
return {
|
|
167
|
+
exitCode,
|
|
168
|
+
result: null,
|
|
169
|
+
diagnosis: commandInspectionError
|
|
170
|
+
? {
|
|
171
|
+
stage: 'command_inspection_failed',
|
|
172
|
+
detail: commandInspectionError,
|
|
173
|
+
}
|
|
174
|
+
: { stage: 'result_missing', detail: null },
|
|
175
|
+
};
|
|
152
176
|
}
|
|
153
177
|
const result =
|
|
154
178
|
findPlayRunnerResult(parsePlayRunnerEvents(recovered.execution.result)) ??
|
|
@@ -156,14 +180,22 @@ export async function inspectDetachedDaytonaRunner(input: {
|
|
|
156
180
|
return {
|
|
157
181
|
exitCode: exitCode ?? recovered.execution.exitCode,
|
|
158
182
|
result,
|
|
183
|
+
diagnosis: result
|
|
184
|
+
? { stage: 'salvaged', detail: null }
|
|
185
|
+
: { stage: 'result_missing', detail: null },
|
|
159
186
|
};
|
|
160
187
|
} catch (error) {
|
|
188
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
161
189
|
console.warn('[play-runner.daytona.detached_inspect_failed]', {
|
|
162
190
|
sandboxId: input.sandboxId,
|
|
163
191
|
cmdId: input.cmdId,
|
|
164
|
-
error:
|
|
192
|
+
error: detail,
|
|
165
193
|
});
|
|
166
|
-
return {
|
|
194
|
+
return {
|
|
195
|
+
exitCode: null,
|
|
196
|
+
result: null,
|
|
197
|
+
diagnosis: { stage: 'sandbox_lookup_failed', detail },
|
|
198
|
+
};
|
|
167
199
|
}
|
|
168
200
|
}
|
|
169
201
|
|
|
@@ -9,16 +9,25 @@ import type {
|
|
|
9
9
|
PlayRunnerResult,
|
|
10
10
|
} from '@shared_libs/play-runtime/protocol';
|
|
11
11
|
|
|
12
|
-
export type PlayRunnerRuntimeResource =
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
12
|
+
export type PlayRunnerRuntimeResource = {
|
|
13
|
+
kind: 'daytona_sandbox';
|
|
14
|
+
sandboxId: string;
|
|
15
|
+
billingStartedAt: number;
|
|
16
|
+
billingEndedAt?: number | null;
|
|
17
|
+
terminalReason?: RuntimeResourceTerminalReason | null;
|
|
18
|
+
terminalAt?: number | null;
|
|
19
|
+
cpu?: number | null;
|
|
20
|
+
memoryGiB?: number | null;
|
|
21
|
+
diskGiB?: number | null;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export type RuntimeResourceTerminalReason =
|
|
25
|
+
| 'completed'
|
|
26
|
+
| 'runner_failed'
|
|
27
|
+
| 'sandbox_missing'
|
|
28
|
+
| 'sandbox_oom'
|
|
29
|
+
| 'cancelled'
|
|
30
|
+
| 'lease_expired';
|
|
22
31
|
|
|
23
32
|
export class RuntimeResourceFenceLostError extends Error {
|
|
24
33
|
constructor(message: string) {
|
|
@@ -127,6 +127,8 @@ type RuntimeApiContext = {
|
|
|
127
127
|
postgresSessionUnwrapKey?: string | null;
|
|
128
128
|
};
|
|
129
129
|
|
|
130
|
+
const RUNTIME_RUN_DATASET_CATALOG_TABLE = '_deepline_run_datasets';
|
|
131
|
+
|
|
130
132
|
type DirectRuntimeTestFaultState = {
|
|
131
133
|
headerValue: string;
|
|
132
134
|
counts: Partial<Record<RuntimeTestFaultName, number>>;
|
|
@@ -1572,8 +1574,17 @@ async function withRuntimePostgres<T>(
|
|
|
1572
1574
|
requestLocalPool = pool;
|
|
1573
1575
|
}
|
|
1574
1576
|
client = await connectRuntimePostgresPool(pool);
|
|
1577
|
+
if (session.executionRole) {
|
|
1578
|
+
await client.query(
|
|
1579
|
+
`SET ROLE ${quoteIdentifier(session.executionRole)}`,
|
|
1580
|
+
);
|
|
1581
|
+
}
|
|
1575
1582
|
break;
|
|
1576
1583
|
} catch (error) {
|
|
1584
|
+
if (client) {
|
|
1585
|
+
client.release();
|
|
1586
|
+
client = null;
|
|
1587
|
+
}
|
|
1577
1588
|
if (cachePool) {
|
|
1578
1589
|
const pool = postgresPools.get(session.postgresUrl);
|
|
1579
1590
|
postgresPools.delete(session.postgresUrl);
|
|
@@ -1604,7 +1615,13 @@ async function withRuntimePostgres<T>(
|
|
|
1604
1615
|
try {
|
|
1605
1616
|
return await fn(client);
|
|
1606
1617
|
} finally {
|
|
1607
|
-
|
|
1618
|
+
try {
|
|
1619
|
+
if (session.executionRole) {
|
|
1620
|
+
await client.query('RESET ROLE');
|
|
1621
|
+
}
|
|
1622
|
+
} finally {
|
|
1623
|
+
client.release();
|
|
1624
|
+
}
|
|
1608
1625
|
if (requestLocalPool) {
|
|
1609
1626
|
await Promise.resolve(requestLocalPool.end()).catch(() => {});
|
|
1610
1627
|
}
|
|
@@ -1833,6 +1850,38 @@ function columnSummaryTable(session: RuntimePostgresSession): string {
|
|
|
1833
1850
|
return fqRuntimeTable(session, session.postgres.columnSummaryTable);
|
|
1834
1851
|
}
|
|
1835
1852
|
|
|
1853
|
+
function runDatasetCatalogTable(session: RuntimePostgresSession): string {
|
|
1854
|
+
return fqRuntimeTable(session, RUNTIME_RUN_DATASET_CATALOG_TABLE);
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
async function registerRuntimeDataset(
|
|
1858
|
+
session: RuntimePostgresSession,
|
|
1859
|
+
input: { playName: string; tableNamespace: string; runId: string },
|
|
1860
|
+
): Promise<void> {
|
|
1861
|
+
const tableNamespace = normalizeTableNamespace(input.tableNamespace);
|
|
1862
|
+
const playName = normalizePlayNameForSheet(input.playName);
|
|
1863
|
+
const datasetId = createRuntimeDatasetId(playName, tableNamespace);
|
|
1864
|
+
await withRuntimePostgres(session, async (client) => {
|
|
1865
|
+
await client.query(
|
|
1866
|
+
`INSERT INTO ${runDatasetCatalogTable(session)} (
|
|
1867
|
+
run_id, dataset_id, play_name, table_namespace, public_path
|
|
1868
|
+
) VALUES ($1, $2, $3, $4, $5)
|
|
1869
|
+
ON CONFLICT (run_id, dataset_id) DO UPDATE SET
|
|
1870
|
+
play_name = EXCLUDED.play_name,
|
|
1871
|
+
table_namespace = EXCLUDED.table_namespace,
|
|
1872
|
+
public_path = EXCLUDED.public_path,
|
|
1873
|
+
updated_at = now()`,
|
|
1874
|
+
[
|
|
1875
|
+
input.runId,
|
|
1876
|
+
datasetId,
|
|
1877
|
+
playName,
|
|
1878
|
+
tableNamespace,
|
|
1879
|
+
`datasets.${tableNamespace}`,
|
|
1880
|
+
],
|
|
1881
|
+
);
|
|
1882
|
+
});
|
|
1883
|
+
}
|
|
1884
|
+
|
|
1836
1885
|
function workReceiptTable(session: RuntimePostgresSession): string {
|
|
1837
1886
|
return fqRuntimeTable(
|
|
1838
1887
|
session,
|
|
@@ -2022,8 +2071,11 @@ async function withRuntimeWorkReceiptClient<T>(
|
|
|
2022
2071
|
): Promise<T> {
|
|
2023
2072
|
// Receipt tables are part of customer Postgres bootstrap, so the hot path
|
|
2024
2073
|
// should not pay DDL on every fresh runtime isolate. Try the receipt query
|
|
2025
|
-
// first, then self-heal once on a missing relation/column.
|
|
2026
|
-
//
|
|
2074
|
+
// first, then self-heal once on a missing relation/column. Permission loss
|
|
2075
|
+
// during an operational receipt query must fail loud: silently repairing it
|
|
2076
|
+
// can let provider work continue after durable receipt persistence is gone.
|
|
2077
|
+
// Only the schema-migration branch below may repair ownership before its
|
|
2078
|
+
// retry. A genuinely missing schema or second failure also rethrows loudly.
|
|
2027
2079
|
const runWithSelfHeal = async (client: RuntimeQueryClient): Promise<T> => {
|
|
2028
2080
|
try {
|
|
2029
2081
|
return await operation(client);
|
|
@@ -2048,10 +2100,6 @@ async function withRuntimeWorkReceiptClient<T>(
|
|
|
2048
2100
|
playName: context.playName?.trim() || session.playName,
|
|
2049
2101
|
});
|
|
2050
2102
|
}
|
|
2051
|
-
} else if (isPostgresPermissionDeniedError(error)) {
|
|
2052
|
-
await repairRuntimeStorageGrants(context, {
|
|
2053
|
-
playName: context.playName?.trim() || session.playName,
|
|
2054
|
-
});
|
|
2055
2103
|
} else {
|
|
2056
2104
|
throw error;
|
|
2057
2105
|
}
|
|
@@ -5188,6 +5236,14 @@ export async function startRuntimeSheetDataset(
|
|
|
5188
5236
|
ms: Date.now() - sessionStartedAt,
|
|
5189
5237
|
rows: input.rows.length,
|
|
5190
5238
|
});
|
|
5239
|
+
// Register the exact run-scoped Dataset Handle before any row can become
|
|
5240
|
+
// durable. A crash after a later sheet write can therefore never leave
|
|
5241
|
+
// customer data discoverable only through static pipeline inference.
|
|
5242
|
+
await registerRuntimeDataset(session, {
|
|
5243
|
+
playName,
|
|
5244
|
+
tableNamespace: input.tableNamespace,
|
|
5245
|
+
runId: input.runId,
|
|
5246
|
+
});
|
|
5191
5247
|
const normalizeStartedAt = Date.now();
|
|
5192
5248
|
const uniqueRows = new Map<
|
|
5193
5249
|
string,
|
|
@@ -39,10 +39,24 @@ export function isSecretAuth(value: unknown): value is SecretAuth {
|
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
export function valueContainsSecret(value: unknown): boolean {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
42
|
+
const pending: unknown[] = [value];
|
|
43
|
+
const seen = new WeakSet<object>();
|
|
44
|
+
while (pending.length > 0) {
|
|
45
|
+
const candidate = pending.pop();
|
|
46
|
+
if (isSecretHandle(candidate) || isSecretAuth(candidate)) return true;
|
|
47
|
+
if (typeof candidate === 'string') {
|
|
48
|
+
if (SECRET_HANDLE_MARKER_RE.test(candidate)) return true;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
if (!candidate || typeof candidate !== 'object') continue;
|
|
52
|
+
if (seen.has(candidate)) continue;
|
|
53
|
+
seen.add(candidate);
|
|
54
|
+
if (Array.isArray(candidate)) {
|
|
55
|
+
for (const entry of candidate) pending.push(entry);
|
|
56
|
+
} else if (isRecord(candidate)) {
|
|
57
|
+
for (const entry of Object.values(candidate)) pending.push(entry);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
46
60
|
return false;
|
|
47
61
|
}
|
|
48
62
|
|
|
@@ -88,14 +88,15 @@ function decideToolExecuteHttpRetry(input: {
|
|
|
88
88
|
hasRetryAfterHeader?: boolean;
|
|
89
89
|
transientHttpRetrySafe?: boolean;
|
|
90
90
|
}): ToolExecuteHttpRetryDecision {
|
|
91
|
+
if (input.hardBillingFailure) {
|
|
92
|
+
return {
|
|
93
|
+
retryable: false,
|
|
94
|
+
attemptCap:
|
|
95
|
+
input.status === 429 ? TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS : 1,
|
|
96
|
+
reason: 'hard_billing_error',
|
|
97
|
+
};
|
|
98
|
+
}
|
|
91
99
|
if (input.status === 429) {
|
|
92
|
-
if (input.hardBillingFailure) {
|
|
93
|
-
return {
|
|
94
|
-
retryable: false,
|
|
95
|
-
attemptCap: TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS,
|
|
96
|
-
reason: 'hard_billing_error',
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
100
|
if (!input.hasRetryAfterHeader) {
|
|
100
101
|
return {
|
|
101
102
|
retryable: true,
|
|
@@ -178,6 +179,7 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
178
179
|
bodyText: string;
|
|
179
180
|
retryAfterHeader?: string | null;
|
|
180
181
|
transientHttpRetrySafe?: boolean;
|
|
182
|
+
providerLatencyMs?: number;
|
|
181
183
|
nowMs?: number;
|
|
182
184
|
}): ToolExecuteHttpFailureOutcome {
|
|
183
185
|
const transientHttpRetrySafe = input.transientHttpRetrySafe === true;
|
|
@@ -196,15 +198,33 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
196
198
|
maxAttempts: initialRetryDecision.attemptCap,
|
|
197
199
|
bodyText: input.bodyText,
|
|
198
200
|
});
|
|
201
|
+
if (
|
|
202
|
+
typeof input.providerLatencyMs === 'number' &&
|
|
203
|
+
Number.isFinite(input.providerLatencyMs) &&
|
|
204
|
+
input.providerLatencyMs >= 0
|
|
205
|
+
) {
|
|
206
|
+
error.message = `${error.message} (provider call ${Math.round(input.providerLatencyMs)}ms)`;
|
|
207
|
+
}
|
|
208
|
+
const hardBillingFailure = isHardBillingToolHttpError(error);
|
|
199
209
|
const retryDecision = decideToolExecuteHttpRetry({
|
|
200
210
|
status: input.status,
|
|
201
|
-
hardBillingFailure
|
|
211
|
+
hardBillingFailure,
|
|
202
212
|
hasRetryAfterHeader,
|
|
203
213
|
transientHttpRetrySafe,
|
|
204
214
|
});
|
|
205
215
|
const shouldRetry =
|
|
206
216
|
retryDecision.retryable && input.attempt < retryDecision.attemptCap;
|
|
207
|
-
|
|
217
|
+
// In-attempt retry safety and later-run repairability are different
|
|
218
|
+
// decisions. A non-idempotent 5xx must not be repeated automatically inside
|
|
219
|
+
// this execution attempt, but an explicit later `plays run` must be able to
|
|
220
|
+
// repair that failed work while completed calls remain reusable. Otherwise
|
|
221
|
+
// one provider gateway failure poisons the semantic call forever.
|
|
222
|
+
if (
|
|
223
|
+
!hardBillingFailure &&
|
|
224
|
+
(retryDecision.retryable || (input.status >= 500 && input.status < 600))
|
|
225
|
+
) {
|
|
226
|
+
error.receiptFailureKind = 'repairable';
|
|
227
|
+
}
|
|
208
228
|
const retryAfterMs = parseToolExecuteRetryAfterMs(
|
|
209
229
|
input.retryAfterHeader,
|
|
210
230
|
input.nowMs,
|
|
@@ -9,6 +9,7 @@ export type WorkReceiptStateStatus =
|
|
|
9
9
|
export type WorkReceiptStateInput = {
|
|
10
10
|
status: WorkReceiptStateStatus;
|
|
11
11
|
runId?: string | null;
|
|
12
|
+
error?: string | null;
|
|
12
13
|
failureKind?: 'terminal' | 'repairable' | null;
|
|
13
14
|
leaseId?: string | null;
|
|
14
15
|
leaseOwnerRunId?: string | null;
|
|
@@ -16,6 +17,25 @@ export type WorkReceiptStateInput = {
|
|
|
16
17
|
leaseExpiresAt?: string | null;
|
|
17
18
|
};
|
|
18
19
|
|
|
20
|
+
export function isLegacyRepairableWorkReceiptError(
|
|
21
|
+
error: string | null | undefined,
|
|
22
|
+
): boolean {
|
|
23
|
+
const message = error?.trim().toLowerCase() ?? '';
|
|
24
|
+
if (!message) return false;
|
|
25
|
+
if (
|
|
26
|
+
message.includes('billing cap') ||
|
|
27
|
+
message.includes('insufficient credits') ||
|
|
28
|
+
message.includes('monthly billing limit')
|
|
29
|
+
) {
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
return (
|
|
33
|
+
/(?:^|\D)5\d\d(?:\D|$)/.test(message) ||
|
|
34
|
+
message.includes('transport failed calling') ||
|
|
35
|
+
message.includes('runtime api call timed out')
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
|
|
19
39
|
export type WorkReceiptLeaseState =
|
|
20
40
|
| { kind: 'none' }
|
|
21
41
|
| {
|
|
@@ -254,7 +274,8 @@ export function decideWorkReceiptClaim(input: {
|
|
|
254
274
|
const claimantRunAttempt = normalizeAttempt(input.claimantRunAttempt);
|
|
255
275
|
const ownerAttempt = normalizeAttempt(input.receipt.leaseOwnerAttempt);
|
|
256
276
|
if (
|
|
257
|
-
input.receipt.failureKind === 'repairable'
|
|
277
|
+
(input.receipt.failureKind === 'repairable' ||
|
|
278
|
+
isLegacyRepairableWorkReceiptError(input.receipt.error)) &&
|
|
258
279
|
claimantRunId !== null &&
|
|
259
280
|
ownerRunId !== null &&
|
|
260
281
|
(claimantRunId !== ownerRunId || ownerAttempt < claimantRunAttempt)
|