deepline 0.1.211 → 0.1.212

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +150 -11
  2. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/customer-console.ts +23 -0
  3. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +1 -1
  4. package/dist/bundling-sources/sdk/src/client.ts +118 -3
  5. package/dist/bundling-sources/sdk/src/plays/bundle-play-file.ts +1 -1
  6. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  7. package/dist/bundling-sources/sdk/src/types.ts +47 -0
  8. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +253 -30
  9. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +22 -20
  10. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +12 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/dataset-id.ts +10 -4
  12. package/dist/bundling-sources/shared_libs/play-runtime/db-session.ts +9 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +3 -5
  14. package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +14 -2
  15. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +4 -2
  16. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +21 -1
  17. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +128 -6
  18. package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +7 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +37 -5
  20. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +19 -10
  21. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +63 -7
  22. package/dist/bundling-sources/shared_libs/play-runtime/secret-capability.ts +18 -4
  23. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +29 -9
  24. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +22 -1
  25. package/dist/bundling-sources/shared_libs/plays/bundling/index.ts +79 -1
  26. package/dist/cli/index.js +637 -229
  27. package/dist/cli/index.mjs +637 -229
  28. package/dist/index.d.mts +76 -6
  29. package/dist/index.d.ts +76 -6
  30. package/dist/index.js +225 -43
  31. package/dist/index.mjs +225 -43
  32. package/dist/plays/bundle-play-file.mjs +65 -4
  33. package/package.json +1 -1
@@ -7,6 +7,7 @@ import {
7
7
  nextPlayRunLedgerTerminalSource,
8
8
  normalizePlayRunLedgerTerminalSource,
9
9
  } from './run-terminal-source';
10
+ import { MAX_LEDGER_LOG_LINES_PER_EVENT } from './ledger-safe-payload';
10
11
 
11
12
  export type PlayRunLedgerStatus =
12
13
  | 'queued'
@@ -84,6 +85,18 @@ export type PlayRunLedgerStepSnapshot = {
84
85
  progress?: PlayRunLedgerStepProgress | null;
85
86
  };
86
87
 
88
+ export type PlayRunLedgerDatasetSnapshot = {
89
+ datasetId: string;
90
+ path: string;
91
+ tableNamespace: string;
92
+ phase: 'registered' | 'available' | 'failed';
93
+ persistedRows: number;
94
+ succeededRows: number;
95
+ failedRows: number;
96
+ complete: boolean;
97
+ updatedAt: number;
98
+ };
99
+
87
100
  export type PlayRunLedgerSnapshot = {
88
101
  runId: string;
89
102
  playName?: string | null;
@@ -96,6 +109,8 @@ export type PlayRunLedgerSnapshot = {
96
109
  durationMs?: number | null;
97
110
  orderedStepIds: string[];
98
111
  stepsById: Record<string, PlayRunLedgerStepSnapshot>;
112
+ orderedDatasetIds: string[];
113
+ datasetsById: Record<string, PlayRunLedgerDatasetSnapshot>;
99
114
  /**
100
115
  * Bounded tail of the run's log lines (last {@link LOG_TAIL_LIMIT}).
101
116
  * Full retention lives in the Run Log Stream (Convex `playRunLogChunks`);
@@ -217,10 +232,34 @@ export type PlayRunLedgerEvent =
217
232
  * repeated identical lines while absorbing redundant re-sends.
218
233
  */
219
234
  channelOffset?: number;
235
+ /** Scheduler attempt that owns this positional log channel. */
236
+ producerAttempt?: number;
237
+ /**
238
+ * Terminal transport replay normally matches ctx.log lines after
239
+ * removing timestamps. Detached-attempt recovery uses exact matching so
240
+ * the same message from a later attempt remains a distinct occurrence.
241
+ */
242
+ transportDedupe?: 'semantic' | 'exact';
220
243
  /** Set by ingestion when this append crossed the retention cap. */
221
244
  logsTruncated?: boolean;
222
245
  })
223
246
  | (PlayRunLedgerBaseEvent & {
247
+ type: 'dataset.lifecycle';
248
+ datasetId: string;
249
+ path: string;
250
+ tableNamespace: string;
251
+ phase: 'registered' | 'available' | 'failed';
252
+ persistedRows?: number;
253
+ succeededRows?: number;
254
+ failedRows?: number;
255
+ complete?: boolean;
256
+ })
257
+ | (PlayRunLedgerBaseEvent & {
258
+ /**
259
+ * @deprecated Mixed-version deploy seam only. Owner: Play Runtime.
260
+ * Remove after 2026-07-19, once pre-dataset-lifecycle workers have
261
+ * exceeded the maximum run duration plus the seven-day drain window.
262
+ */
224
263
  type: 'sheet.summary.updated';
225
264
  tableNamespace: string;
226
265
  deltaCursor?: number;
@@ -241,6 +280,7 @@ export type PlayRunLedgerStatusPatch = {
241
280
  * (see `slicePositionalLogLines`), forwarded onto the `log.appended` event.
242
281
  */
243
282
  liveLogsChannelOffset?: number | null;
283
+ liveLogsProducerAttempt?: number | null;
244
284
  liveNodeProgress?: unknown;
245
285
  result?: unknown;
246
286
  };
@@ -401,6 +441,8 @@ export function createEmptyPlayRunLedgerSnapshot(input: {
401
441
  : null,
402
442
  orderedStepIds: [],
403
443
  stepsById: {},
444
+ orderedDatasetIds: [],
445
+ datasetsById: {},
404
446
  logTail: [],
405
447
  totalLogCount: 0,
406
448
  logsTruncated: false,
@@ -447,6 +489,26 @@ export function normalizePlayRunLedgerSnapshot(
447
489
  progress: rawProgress ? normalizeStepProgress(rawProgress) : null,
448
490
  };
449
491
  }
492
+ const rawDatasets = isRecord(value.datasetsById) ? value.datasetsById : {};
493
+ const datasetsById: Record<string, PlayRunLedgerDatasetSnapshot> = {};
494
+ for (const [datasetId, rawDataset] of Object.entries(rawDatasets)) {
495
+ if (!datasetId.trim() || !isRecord(rawDataset)) continue;
496
+ const phase =
497
+ rawDataset.phase === 'available' || rawDataset.phase === 'failed'
498
+ ? rawDataset.phase
499
+ : 'registered';
500
+ datasetsById[datasetId] = {
501
+ datasetId,
502
+ path: optionalString(rawDataset.path) ?? `datasets.${datasetId}`,
503
+ tableNamespace: optionalString(rawDataset.tableNamespace) ?? datasetId,
504
+ phase,
505
+ persistedRows: Math.max(0, finiteNumber(rawDataset.persistedRows) ?? 0),
506
+ succeededRows: Math.max(0, finiteNumber(rawDataset.succeededRows) ?? 0),
507
+ failedRows: Math.max(0, finiteNumber(rawDataset.failedRows) ?? 0),
508
+ complete: rawDataset.complete === true,
509
+ updatedAt: finiteNumber(rawDataset.updatedAt) ?? 0,
510
+ };
511
+ }
450
512
  const createdAt = finiteNumber(value.createdAt) ?? fallback.createdAt ?? null;
451
513
  const startedAt = finiteNumber(value.startedAt) ?? fallback.startedAt ?? null;
452
514
  const finishedAt =
@@ -484,6 +546,14 @@ export function normalizePlayRunLedgerSnapshot(
484
546
  : finiteNumber(value.durationMs),
485
547
  orderedStepIds: orderedStepIds.filter((stepId) => stepsById[stepId]),
486
548
  stepsById,
549
+ orderedDatasetIds: (Array.isArray(value.orderedDatasetIds)
550
+ ? value.orderedDatasetIds
551
+ : Object.keys(datasetsById)
552
+ ).filter(
553
+ (datasetId): datasetId is string =>
554
+ typeof datasetId === 'string' && Boolean(datasetsById[datasetId]),
555
+ ),
556
+ datasetsById,
487
557
  logTail: logTail.slice(-LOG_TAIL_LIMIT),
488
558
  // Snapshots persisted before totalLogCount existed only know the retained
489
559
  // tail, so the best lower bound for the cumulative count is the tail size.
@@ -1269,6 +1339,34 @@ export function reducePlayRunLedgerEvent(
1269
1339
  : null,
1270
1340
  });
1271
1341
  }
1342
+ case 'dataset.lifecycle': {
1343
+ const current = base.datasetsById[event.datasetId];
1344
+ const next: PlayRunLedgerDatasetSnapshot = {
1345
+ datasetId: event.datasetId,
1346
+ path: event.path,
1347
+ tableNamespace: event.tableNamespace,
1348
+ phase: event.phase,
1349
+ persistedRows: Math.max(
1350
+ current?.persistedRows ?? 0,
1351
+ event.persistedRows ?? 0,
1352
+ ),
1353
+ succeededRows: Math.max(
1354
+ current?.succeededRows ?? 0,
1355
+ event.succeededRows ?? 0,
1356
+ ),
1357
+ failedRows: Math.max(current?.failedRows ?? 0, event.failedRows ?? 0),
1358
+ complete: current?.complete === true || event.complete === true,
1359
+ updatedAt: occurredAt,
1360
+ };
1361
+ return withTiming({
1362
+ ...base,
1363
+ orderedDatasetIds: current
1364
+ ? base.orderedDatasetIds
1365
+ : [...base.orderedDatasetIds, event.datasetId],
1366
+ datasetsById: { ...base.datasetsById, [event.datasetId]: next },
1367
+ resultTableNamespace: base.resultTableNamespace ?? event.tableNamespace,
1368
+ });
1369
+ }
1272
1370
  case 'sheet.summary.updated':
1273
1371
  return withTiming({
1274
1372
  ...base,
@@ -1376,6 +1474,8 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
1376
1474
  source?: PlayRunLedgerEventSource;
1377
1475
  occurredAt?: number;
1378
1476
  channelOffset?: number | null;
1477
+ producerAttempt?: number | null;
1478
+ transportDedupe?: 'semantic' | 'exact';
1379
1479
  includeLogAppend?: boolean;
1380
1480
  }): PlayRunLedgerEvent[] {
1381
1481
  const source = input.source ?? 'worker';
@@ -1392,16 +1492,31 @@ export function buildPlayRunLedgerEventsFromLogLines(input: {
1392
1492
  if (newLines.length === 0 || input.includeLogAppend === false) {
1393
1493
  return [];
1394
1494
  }
1395
- return [
1396
- {
1495
+ const events: PlayRunLedgerEvent[] = [];
1496
+ for (
1497
+ let index = 0;
1498
+ index < newLines.length;
1499
+ index += MAX_LEDGER_LOG_LINES_PER_EVENT
1500
+ ) {
1501
+ events.push({
1397
1502
  type: 'log.appended',
1398
1503
  runId: input.runId,
1399
1504
  source,
1400
1505
  occurredAt,
1401
- lines: newLines,
1402
- ...(positional ? { channelOffset: input.channelOffset! } : {}),
1403
- },
1404
- ];
1506
+ lines: newLines.slice(index, index + MAX_LEDGER_LOG_LINES_PER_EVENT),
1507
+ ...(positional ? { channelOffset: input.channelOffset! + index } : {}),
1508
+ ...(positional &&
1509
+ typeof input.producerAttempt === 'number' &&
1510
+ Number.isFinite(input.producerAttempt) &&
1511
+ input.producerAttempt >= 0
1512
+ ? { producerAttempt: Math.trunc(input.producerAttempt) }
1513
+ : {}),
1514
+ ...(input.transportDedupe
1515
+ ? { transportDedupe: input.transportDedupe }
1516
+ : {}),
1517
+ });
1518
+ }
1519
+ return events;
1405
1520
  }
1406
1521
 
1407
1522
  /**
@@ -1419,6 +1534,8 @@ export function buildTerminalLogReplayEvents(input: {
1419
1534
  source?: PlayRunLedgerEventSource;
1420
1535
  occurredAt?: number;
1421
1536
  liveLogTotalCount?: number;
1537
+ dedupeMode?: 'semantic' | 'exact';
1538
+ producerAttempt?: number;
1422
1539
  }): PlayRunLedgerEvent[] {
1423
1540
  const channelOffset = terminalLogReplayChannelOffset({
1424
1541
  liveLogTotalCount: input.liveLogTotalCount,
@@ -1430,6 +1547,10 @@ export function buildTerminalLogReplayEvents(input: {
1430
1547
  source: 'coordinator',
1431
1548
  occurredAt: input.occurredAt,
1432
1549
  channelOffset,
1550
+ producerAttempt: input.producerAttempt,
1551
+ transportDedupe:
1552
+ input.dedupeMode ??
1553
+ (typeof input.producerAttempt === 'number' ? 'exact' : undefined),
1433
1554
  });
1434
1555
  }
1435
1556
 
@@ -1466,6 +1587,7 @@ export function buildPlayRunLedgerEventsFromStatusPatch(input: {
1466
1587
  source,
1467
1588
  occurredAt: checkpointAt,
1468
1589
  channelOffset: patch.liveLogsChannelOffset ?? null,
1590
+ producerAttempt: patch.liveLogsProducerAttempt ?? null,
1469
1591
  })
1470
1592
  : [];
1471
1593
  const earliestStartedAt =
@@ -4,6 +4,7 @@ import {
4
4
  normalizePlayRunLedgerSnapshot,
5
5
  summarizeRunRowOutcomes,
6
6
  type PlayRunLedgerSnapshot,
7
+ type PlayRunLedgerDatasetSnapshot,
7
8
  type PlayRunLedgerStepSnapshot,
8
9
  type PlayRunRowOutcomeSummary,
9
10
  } from './run-ledger';
@@ -76,6 +77,7 @@ export type PlayRunLiveSnapshot = {
76
77
  logsTruncated?: boolean;
77
78
  activeArtifactTableNamespace: string | null;
78
79
  resultTableNamespace: string | null;
80
+ datasets?: PlayRunLedgerDatasetSnapshot[];
79
81
  nodeStates: PlayRunStreamNodeState[];
80
82
  activeNodeId: string | null;
81
83
  /**
@@ -212,6 +214,11 @@ function buildSnapshotFromLedger(
212
214
  ...(snapshot.logsTruncated ? { logsTruncated: true } : {}),
213
215
  activeArtifactTableNamespace: snapshot.activeArtifactTableNamespace ?? null,
214
216
  resultTableNamespace: snapshot.resultTableNamespace ?? null,
217
+ datasets: snapshot.orderedDatasetIds
218
+ .map((datasetId) => snapshot.datasetsById[datasetId])
219
+ .filter((dataset): dataset is PlayRunLedgerDatasetSnapshot =>
220
+ Boolean(dataset),
221
+ ),
215
222
  nodeStates,
216
223
  activeNodeId: snapshot.activeStepId ?? null,
217
224
  ...(rowOutcomes ? { rowOutcomes } : {}),
@@ -122,19 +122,34 @@ export async function inspectDetachedDaytonaRunner(input: {
122
122
  cmdId: string;
123
123
  outputPath: string;
124
124
  exitCodePath: string;
125
- }): Promise<{ exitCode: number | null; result: PlayRunnerResult | null }> {
125
+ }): Promise<{
126
+ exitCode: number | null;
127
+ result: PlayRunnerResult | null;
128
+ diagnosis: {
129
+ stage:
130
+ | 'salvaged'
131
+ | 'result_missing'
132
+ | 'command_inspection_failed'
133
+ | 'sandbox_lookup_failed';
134
+ detail: string | null;
135
+ };
136
+ }> {
126
137
  try {
127
138
  const { clientOptions } = loadDaytonaRequiredConfig();
128
139
  const daytona = daytonaSdkClientFactory.createFull(clientOptions);
129
140
  const sandbox = (await daytona.get(input.sandboxId)) as DaytonaSandbox;
130
141
  let exitCode: number | null = null;
142
+ let commandInspectionError: string | null = null;
131
143
  try {
132
144
  const command = await sandbox.process.getSessionCommand(
133
145
  input.sessionId,
134
146
  input.cmdId,
135
147
  );
136
- exitCode = typeof command?.exitCode === 'number' ? command.exitCode : null;
148
+ exitCode =
149
+ typeof command?.exitCode === 'number' ? command.exitCode : null;
137
150
  } catch (error) {
151
+ commandInspectionError =
152
+ error instanceof Error ? error.message : String(error);
138
153
  console.warn('[play-runner.daytona.detached_inspect_command_failed]', {
139
154
  sandboxId: input.sandboxId,
140
155
  cmdId: input.cmdId,
@@ -148,7 +163,16 @@ export async function inspectDetachedDaytonaRunner(input: {
148
163
  timeoutMs: 10_000,
149
164
  });
150
165
  if (!recovered.execution) {
151
- return { exitCode, result: null };
166
+ return {
167
+ exitCode,
168
+ result: null,
169
+ diagnosis: commandInspectionError
170
+ ? {
171
+ stage: 'command_inspection_failed',
172
+ detail: commandInspectionError,
173
+ }
174
+ : { stage: 'result_missing', detail: null },
175
+ };
152
176
  }
153
177
  const result =
154
178
  findPlayRunnerResult(parsePlayRunnerEvents(recovered.execution.result)) ??
@@ -156,14 +180,22 @@ export async function inspectDetachedDaytonaRunner(input: {
156
180
  return {
157
181
  exitCode: exitCode ?? recovered.execution.exitCode,
158
182
  result,
183
+ diagnosis: result
184
+ ? { stage: 'salvaged', detail: null }
185
+ : { stage: 'result_missing', detail: null },
159
186
  };
160
187
  } catch (error) {
188
+ const detail = error instanceof Error ? error.message : String(error);
161
189
  console.warn('[play-runner.daytona.detached_inspect_failed]', {
162
190
  sandboxId: input.sandboxId,
163
191
  cmdId: input.cmdId,
164
- error: error instanceof Error ? error.message : String(error),
192
+ error: detail,
165
193
  });
166
- return { exitCode: null, result: null };
194
+ return {
195
+ exitCode: null,
196
+ result: null,
197
+ diagnosis: { stage: 'sandbox_lookup_failed', detail },
198
+ };
167
199
  }
168
200
  }
169
201
 
@@ -9,16 +9,25 @@ import type {
9
9
  PlayRunnerResult,
10
10
  } from '@shared_libs/play-runtime/protocol';
11
11
 
12
- export type PlayRunnerRuntimeResource =
13
- | {
14
- kind: 'daytona_sandbox';
15
- sandboxId: string;
16
- billingStartedAt: number;
17
- billingEndedAt?: number | null;
18
- cpu?: number | null;
19
- memoryGiB?: number | null;
20
- diskGiB?: number | null;
21
- };
12
+ export type PlayRunnerRuntimeResource = {
13
+ kind: 'daytona_sandbox';
14
+ sandboxId: string;
15
+ billingStartedAt: number;
16
+ billingEndedAt?: number | null;
17
+ terminalReason?: RuntimeResourceTerminalReason | null;
18
+ terminalAt?: number | null;
19
+ cpu?: number | null;
20
+ memoryGiB?: number | null;
21
+ diskGiB?: number | null;
22
+ };
23
+
24
+ export type RuntimeResourceTerminalReason =
25
+ | 'completed'
26
+ | 'runner_failed'
27
+ | 'sandbox_missing'
28
+ | 'sandbox_oom'
29
+ | 'cancelled'
30
+ | 'lease_expired';
22
31
 
23
32
  export class RuntimeResourceFenceLostError extends Error {
24
33
  constructor(message: string) {
@@ -127,6 +127,8 @@ type RuntimeApiContext = {
127
127
  postgresSessionUnwrapKey?: string | null;
128
128
  };
129
129
 
130
+ const RUNTIME_RUN_DATASET_CATALOG_TABLE = '_deepline_run_datasets';
131
+
130
132
  type DirectRuntimeTestFaultState = {
131
133
  headerValue: string;
132
134
  counts: Partial<Record<RuntimeTestFaultName, number>>;
@@ -1572,8 +1574,17 @@ async function withRuntimePostgres<T>(
1572
1574
  requestLocalPool = pool;
1573
1575
  }
1574
1576
  client = await connectRuntimePostgresPool(pool);
1577
+ if (session.executionRole) {
1578
+ await client.query(
1579
+ `SET ROLE ${quoteIdentifier(session.executionRole)}`,
1580
+ );
1581
+ }
1575
1582
  break;
1576
1583
  } catch (error) {
1584
+ if (client) {
1585
+ client.release();
1586
+ client = null;
1587
+ }
1577
1588
  if (cachePool) {
1578
1589
  const pool = postgresPools.get(session.postgresUrl);
1579
1590
  postgresPools.delete(session.postgresUrl);
@@ -1604,7 +1615,13 @@ async function withRuntimePostgres<T>(
1604
1615
  try {
1605
1616
  return await fn(client);
1606
1617
  } finally {
1607
- client.release();
1618
+ try {
1619
+ if (session.executionRole) {
1620
+ await client.query('RESET ROLE');
1621
+ }
1622
+ } finally {
1623
+ client.release();
1624
+ }
1608
1625
  if (requestLocalPool) {
1609
1626
  await Promise.resolve(requestLocalPool.end()).catch(() => {});
1610
1627
  }
@@ -1833,6 +1850,38 @@ function columnSummaryTable(session: RuntimePostgresSession): string {
1833
1850
  return fqRuntimeTable(session, session.postgres.columnSummaryTable);
1834
1851
  }
1835
1852
 
1853
+ function runDatasetCatalogTable(session: RuntimePostgresSession): string {
1854
+ return fqRuntimeTable(session, RUNTIME_RUN_DATASET_CATALOG_TABLE);
1855
+ }
1856
+
1857
+ async function registerRuntimeDataset(
1858
+ session: RuntimePostgresSession,
1859
+ input: { playName: string; tableNamespace: string; runId: string },
1860
+ ): Promise<void> {
1861
+ const tableNamespace = normalizeTableNamespace(input.tableNamespace);
1862
+ const playName = normalizePlayNameForSheet(input.playName);
1863
+ const datasetId = createRuntimeDatasetId(playName, tableNamespace);
1864
+ await withRuntimePostgres(session, async (client) => {
1865
+ await client.query(
1866
+ `INSERT INTO ${runDatasetCatalogTable(session)} (
1867
+ run_id, dataset_id, play_name, table_namespace, public_path
1868
+ ) VALUES ($1, $2, $3, $4, $5)
1869
+ ON CONFLICT (run_id, dataset_id) DO UPDATE SET
1870
+ play_name = EXCLUDED.play_name,
1871
+ table_namespace = EXCLUDED.table_namespace,
1872
+ public_path = EXCLUDED.public_path,
1873
+ updated_at = now()`,
1874
+ [
1875
+ input.runId,
1876
+ datasetId,
1877
+ playName,
1878
+ tableNamespace,
1879
+ `datasets.${tableNamespace}`,
1880
+ ],
1881
+ );
1882
+ });
1883
+ }
1884
+
1836
1885
  function workReceiptTable(session: RuntimePostgresSession): string {
1837
1886
  return fqRuntimeTable(
1838
1887
  session,
@@ -2022,8 +2071,11 @@ async function withRuntimeWorkReceiptClient<T>(
2022
2071
  ): Promise<T> {
2023
2072
  // Receipt tables are part of customer Postgres bootstrap, so the hot path
2024
2073
  // should not pay DDL on every fresh runtime isolate. Try the receipt query
2025
- // first, then self-heal once on a missing relation/column. A genuinely
2026
- // missing schema, permission error, or second failure still rethrows loudly.
2074
+ // first, then self-heal once on a missing relation/column. Permission loss
2075
+ // during an operational receipt query must fail loud: silently repairing it
2076
+ // can let provider work continue after durable receipt persistence is gone.
2077
+ // Only the schema-migration branch below may repair ownership before its
2078
+ // retry. A genuinely missing schema or second failure also rethrows loudly.
2027
2079
  const runWithSelfHeal = async (client: RuntimeQueryClient): Promise<T> => {
2028
2080
  try {
2029
2081
  return await operation(client);
@@ -2048,10 +2100,6 @@ async function withRuntimeWorkReceiptClient<T>(
2048
2100
  playName: context.playName?.trim() || session.playName,
2049
2101
  });
2050
2102
  }
2051
- } else if (isPostgresPermissionDeniedError(error)) {
2052
- await repairRuntimeStorageGrants(context, {
2053
- playName: context.playName?.trim() || session.playName,
2054
- });
2055
2103
  } else {
2056
2104
  throw error;
2057
2105
  }
@@ -5188,6 +5236,14 @@ export async function startRuntimeSheetDataset(
5188
5236
  ms: Date.now() - sessionStartedAt,
5189
5237
  rows: input.rows.length,
5190
5238
  });
5239
+ // Register the exact run-scoped Dataset Handle before any row can become
5240
+ // durable. A crash after a later sheet write can therefore never leave
5241
+ // customer data discoverable only through static pipeline inference.
5242
+ await registerRuntimeDataset(session, {
5243
+ playName,
5244
+ tableNamespace: input.tableNamespace,
5245
+ runId: input.runId,
5246
+ });
5191
5247
  const normalizeStartedAt = Date.now();
5192
5248
  const uniqueRows = new Map<
5193
5249
  string,
@@ -39,10 +39,24 @@ export function isSecretAuth(value: unknown): value is SecretAuth {
39
39
  }
40
40
 
41
41
  export function valueContainsSecret(value: unknown): boolean {
42
- if (isSecretHandle(value) || isSecretAuth(value)) return true;
43
- if (typeof value === 'string') return SECRET_HANDLE_MARKER_RE.test(value);
44
- if (Array.isArray(value)) return value.some(valueContainsSecret);
45
- if (isRecord(value)) return Object.values(value).some(valueContainsSecret);
42
+ const pending: unknown[] = [value];
43
+ const seen = new WeakSet<object>();
44
+ while (pending.length > 0) {
45
+ const candidate = pending.pop();
46
+ if (isSecretHandle(candidate) || isSecretAuth(candidate)) return true;
47
+ if (typeof candidate === 'string') {
48
+ if (SECRET_HANDLE_MARKER_RE.test(candidate)) return true;
49
+ continue;
50
+ }
51
+ if (!candidate || typeof candidate !== 'object') continue;
52
+ if (seen.has(candidate)) continue;
53
+ seen.add(candidate);
54
+ if (Array.isArray(candidate)) {
55
+ for (const entry of candidate) pending.push(entry);
56
+ } else if (isRecord(candidate)) {
57
+ for (const entry of Object.values(candidate)) pending.push(entry);
58
+ }
59
+ }
46
60
  return false;
47
61
  }
48
62
 
@@ -88,14 +88,15 @@ function decideToolExecuteHttpRetry(input: {
88
88
  hasRetryAfterHeader?: boolean;
89
89
  transientHttpRetrySafe?: boolean;
90
90
  }): ToolExecuteHttpRetryDecision {
91
+ if (input.hardBillingFailure) {
92
+ return {
93
+ retryable: false,
94
+ attemptCap:
95
+ input.status === 429 ? TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS : 1,
96
+ reason: 'hard_billing_error',
97
+ };
98
+ }
91
99
  if (input.status === 429) {
92
- if (input.hardBillingFailure) {
93
- return {
94
- retryable: false,
95
- attemptCap: TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS,
96
- reason: 'hard_billing_error',
97
- };
98
- }
99
100
  if (!input.hasRetryAfterHeader) {
100
101
  return {
101
102
  retryable: true,
@@ -178,6 +179,7 @@ export function classifyToolExecuteHttpFailure(input: {
178
179
  bodyText: string;
179
180
  retryAfterHeader?: string | null;
180
181
  transientHttpRetrySafe?: boolean;
182
+ providerLatencyMs?: number;
181
183
  nowMs?: number;
182
184
  }): ToolExecuteHttpFailureOutcome {
183
185
  const transientHttpRetrySafe = input.transientHttpRetrySafe === true;
@@ -196,15 +198,33 @@ export function classifyToolExecuteHttpFailure(input: {
196
198
  maxAttempts: initialRetryDecision.attemptCap,
197
199
  bodyText: input.bodyText,
198
200
  });
201
+ if (
202
+ typeof input.providerLatencyMs === 'number' &&
203
+ Number.isFinite(input.providerLatencyMs) &&
204
+ input.providerLatencyMs >= 0
205
+ ) {
206
+ error.message = `${error.message} (provider call ${Math.round(input.providerLatencyMs)}ms)`;
207
+ }
208
+ const hardBillingFailure = isHardBillingToolHttpError(error);
199
209
  const retryDecision = decideToolExecuteHttpRetry({
200
210
  status: input.status,
201
- hardBillingFailure: isHardBillingToolHttpError(error),
211
+ hardBillingFailure,
202
212
  hasRetryAfterHeader,
203
213
  transientHttpRetrySafe,
204
214
  });
205
215
  const shouldRetry =
206
216
  retryDecision.retryable && input.attempt < retryDecision.attemptCap;
207
- if (retryDecision.retryable) error.receiptFailureKind = 'repairable';
217
+ // In-attempt retry safety and later-run repairability are different
218
+ // decisions. A non-idempotent 5xx must not be repeated automatically inside
219
+ // this execution attempt, but an explicit later `plays run` must be able to
220
+ // repair that failed work while completed calls remain reusable. Otherwise
221
+ // one provider gateway failure poisons the semantic call forever.
222
+ if (
223
+ !hardBillingFailure &&
224
+ (retryDecision.retryable || (input.status >= 500 && input.status < 600))
225
+ ) {
226
+ error.receiptFailureKind = 'repairable';
227
+ }
208
228
  const retryAfterMs = parseToolExecuteRetryAfterMs(
209
229
  input.retryAfterHeader,
210
230
  input.nowMs,
@@ -9,6 +9,7 @@ export type WorkReceiptStateStatus =
9
9
  export type WorkReceiptStateInput = {
10
10
  status: WorkReceiptStateStatus;
11
11
  runId?: string | null;
12
+ error?: string | null;
12
13
  failureKind?: 'terminal' | 'repairable' | null;
13
14
  leaseId?: string | null;
14
15
  leaseOwnerRunId?: string | null;
@@ -16,6 +17,25 @@ export type WorkReceiptStateInput = {
16
17
  leaseExpiresAt?: string | null;
17
18
  };
18
19
 
20
+ export function isLegacyRepairableWorkReceiptError(
21
+ error: string | null | undefined,
22
+ ): boolean {
23
+ const message = error?.trim().toLowerCase() ?? '';
24
+ if (!message) return false;
25
+ if (
26
+ message.includes('billing cap') ||
27
+ message.includes('insufficient credits') ||
28
+ message.includes('monthly billing limit')
29
+ ) {
30
+ return false;
31
+ }
32
+ return (
33
+ /(?:^|\D)5\d\d(?:\D|$)/.test(message) ||
34
+ message.includes('transport failed calling') ||
35
+ message.includes('runtime api call timed out')
36
+ );
37
+ }
38
+
19
39
  export type WorkReceiptLeaseState =
20
40
  | { kind: 'none' }
21
41
  | {
@@ -254,7 +274,8 @@ export function decideWorkReceiptClaim(input: {
254
274
  const claimantRunAttempt = normalizeAttempt(input.claimantRunAttempt);
255
275
  const ownerAttempt = normalizeAttempt(input.receipt.leaseOwnerAttempt);
256
276
  if (
257
- input.receipt.failureKind === 'repairable' &&
277
+ (input.receipt.failureKind === 'repairable' ||
278
+ isLegacyRepairableWorkReceiptError(input.receipt.error)) &&
258
279
  claimantRunId !== null &&
259
280
  ownerRunId !== null &&
260
281
  (claimantRunId !== ownerRunId || ownerAttempt < claimantRunAttempt)