deepline 0.3.246 → 0.3.248

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +177 -13
  2. package/dist/bundling-sources/sdk/src/index.ts +8 -0
  3. package/dist/bundling-sources/sdk/src/plays/input-example.ts +40 -0
  4. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  5. package/dist/bundling-sources/sdk/src/runs/overview.ts +457 -0
  6. package/dist/bundling-sources/sdk/src/runs/receipts.ts +23 -0
  7. package/dist/bundling-sources/sdk/src/tool-calls.ts +21 -1
  8. package/dist/bundling-sources/sdk/src/types.ts +16 -1
  9. package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +37 -1
  10. package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +1 -0
  11. package/dist/bundling-sources/shared_libs/play-data-plane/column-names.ts +17 -3
  12. package/dist/bundling-sources/shared_libs/play-data-plane/sheet-contract.ts +40 -8
  13. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +26 -1
  14. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +150 -19
  15. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +2 -1
  16. package/dist/bundling-sources/shared_libs/play-runtime/daytona-runtime-config.ts +6 -3
  17. package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts +26 -21
  18. package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts +23 -21
  19. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +17 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/log-context.ts +105 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +8 -1
  22. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +9 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +40 -4
  24. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +50 -5
  25. package/dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts +3 -1
  26. package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +28 -1
  27. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +14 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +56 -16
  29. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-control-transport.ts +184 -0
  30. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts +327 -0
  31. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +87 -30
  32. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +24 -13
  33. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-start-admission.ts +23 -6
  34. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +76 -33
  35. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +90 -35
  36. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +16 -0
  37. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +62 -21
  38. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +368 -66
  39. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +22 -4
  40. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +106 -81
  41. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +1347 -0
  42. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-queue-health.ts +16 -19
  43. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +14 -6
  44. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +16 -0
  45. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +40 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +3 -2
  47. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +5 -2
  48. package/dist/bundling-sources/shared_libs/plays/dataset-summary.ts +11 -7
  49. package/dist/bundling-sources/shared_libs/plays/static-pipeline-storage-codec.ts +5 -2
  50. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +11 -2
  51. package/dist/cli/index.js +3504 -2368
  52. package/dist/cli/index.mjs +3188 -2052
  53. package/dist/{compiler-manifest-n8ZtPv2h.d.mts → compiler-manifest-DavmLa8y.d.mts} +1 -0
  54. package/dist/{compiler-manifest-n8ZtPv2h.d.ts → compiler-manifest-DavmLa8y.d.ts} +1 -0
  55. package/dist/index.d.mts +97 -5
  56. package/dist/index.d.ts +97 -5
  57. package/dist/index.js +135 -15
  58. package/dist/index.mjs +135 -15
  59. package/dist/install-integrity.json +9 -2
  60. package/dist/plays/bundle-play-file.d.mts +2 -2
  61. package/dist/plays/bundle-play-file.d.ts +2 -2
  62. package/dist/plays/bundle-play-file.mjs +1 -1
  63. package/dist/release.d.mts +1 -1
  64. package/dist/release.d.ts +1 -1
  65. package/dist/release.js +1 -1
  66. package/dist/release.mjs +1 -1
  67. package/package.json +1 -1
@@ -25,6 +25,7 @@ import type {
25
25
  PlayRunnerRuntimeTiming,
26
26
  } from '../protocol';
27
27
  import { PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA } from '../runtime-scheduler-topology';
28
+ import { RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER } from './runtime-event-search';
28
29
  import type { PlayRunnerRuntimeResource } from '../runner-backends/types';
29
30
  import {
30
31
  RuntimeSandboxOperationState,
@@ -76,6 +77,10 @@ import {
76
77
  type DocflowObservationUpsert,
77
78
  } from '../docflow-observation';
78
79
  import { normalizePlayDocflowNodeIoState } from '../docflow-node-io';
80
+ import {
81
+ normalizePlayLogContext,
82
+ type PlayLogContext,
83
+ } from '../log-context';
79
84
  import {
80
85
  workReceiptFailureKindCodeForWrite,
81
86
  workReceiptFailureKindFromCode,
@@ -299,7 +304,10 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
299
304
  // v46 adds a bounded provider-scope Retry-After hold with shared adaptive-rate
300
305
  // state. The scope owns learning and dispatch across runs and tool rules;
301
306
  // existing deadline rows expand additively and initialize on their first seed.
302
- export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 46;
307
+ // v47 attaches the append-only runtime event projection to durable outbox and
308
+ // receipt transitions; no existing runtime wire field changes.
309
+ // v48 expands optional compact-log columns and readers before writer enablement.
310
+ export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 48;
303
311
  const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
304
312
  'scheduler_schema_migrations';
305
313
  export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
@@ -885,6 +893,8 @@ const OUTBOX_RETRY_MAX_MS = 15 * 60 * 1_000;
885
893
  // diagnostic window, then delete in small, independently committed batches.
886
894
  export const POSTGRES_SCHEDULER_OUTBOX_DELIVERED_RETENTION_MS =
887
895
  30 * 24 * 60 * 60 * 1_000;
896
+ export const POSTGRES_SCHEDULER_OUTBOX_LOG_BODY_RETENTION_MS =
897
+ 7 * 24 * 60 * 60 * 1_000;
888
898
  export const POSTGRES_SCHEDULER_OUTBOX_DELIVERED_RETENTION_BATCH_LIMIT = 1_000;
889
899
  const RUN_LEDGER_DISPATCH_TOKEN_TIMEOUT_MS = 15_000;
890
900
  const COMPUTE_BILLING_DISPATCH_TOKEN_TIMEOUT_MS = 15_000;
@@ -1101,6 +1111,8 @@ export type PostgresSchedulerSchemaShapeRequirements = Readonly<{
1101
1111
  rateRequesterFairShare?: boolean;
1102
1112
  /** v45 added durable Retry-After holds shared across rules and runs. */
1103
1113
  rateScopeRetryAfter?: boolean;
1114
+ /** v48 expands compact log storage; prior readers ignore these columns. */
1115
+ runtimeLogStorage?: boolean;
1104
1116
  }>;
1105
1117
 
1106
1118
  export async function hasCurrentPostgresSchedulerSchemaShape(
@@ -1718,6 +1730,22 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
1718
1730
  AND index_row.indisvalid
1719
1731
  )
1720
1732
  )
1733
+ AND (
1734
+ $38::boolean = false
1735
+ OR (
1736
+ EXISTS (
1737
+ SELECT 1 FROM information_schema.columns
1738
+ WHERE table_schema = $1 AND table_name = 'runs'
1739
+ AND column_name = 'run_log_store' AND is_nullable = 'YES'
1740
+ )
1741
+ AND (
1742
+ SELECT count(*) FROM information_schema.columns
1743
+ WHERE table_schema = $1 AND table_name = 'attempts'
1744
+ AND column_name IN ('runner_result_digest', 'runner_log_payload_json')
1745
+ AND is_nullable = 'YES'
1746
+ ) = 2
1747
+ )
1748
+ )
1721
1749
  AS ready
1722
1750
  `,
1723
1751
  [
@@ -1758,6 +1786,7 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
1758
1786
  `${quotedSchema}.${quoteIdent('rate_requesters')}`,
1759
1787
  requirements.rateScopeRetryAfter !== false,
1760
1788
  `${quotedSchema}.${quoteIdent('rate_scope_holds')}`,
1789
+ requirements.runtimeLogStorage !== false,
1761
1790
  ],
1762
1791
  );
1763
1792
  if (currentShape.rows[0]?.ready !== true) return false;
@@ -1980,6 +2009,19 @@ export async function ensurePostgresSchedulerOnlineAdditiveExpansion(
1980
2009
  options,
1981
2010
  )} (state, updated_at)
1982
2011
  `);
2012
+ await ensurePostgresSchedulerRunLogStorageExpansion(client, options);
2013
+ }
2014
+
2015
+ /** Deployment-only expansion; absent values keep legacy storage authoritative. */
2016
+ async function ensurePostgresSchedulerRunLogStorageExpansion(
2017
+ client: PostgresSchedulerQueryClient,
2018
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
2019
+ ): Promise<void> {
2020
+ await client.query(`ALTER TABLE ${tableName('runs', options)}
2021
+ ADD COLUMN IF NOT EXISTS run_log_store text`);
2022
+ await client.query(`ALTER TABLE ${tableName('attempts', options)}
2023
+ ADD COLUMN IF NOT EXISTS runner_result_digest text,
2024
+ ADD COLUMN IF NOT EXISTS runner_log_payload_json jsonb`);
1983
2025
  }
1984
2026
 
1985
2027
  /** Read the highest deployment-owned scheduler schema marker. */
@@ -2546,6 +2588,7 @@ export async function ensurePostgresSchedulerSchema(
2546
2588
  ALTER TABLE ${tableName('attempts', options)}
2547
2589
  ADD COLUMN IF NOT EXISTS runner_terminal_received_at timestamptz
2548
2590
  `);
2591
+ await ensurePostgresSchedulerRunLogStorageExpansion(client, options);
2549
2592
  // Authoritative provenance for terminal reporting. Do not infer this from
2550
2593
  // the completed/cleanup state shape: direct ingress and compatibility wake
2551
2594
  // can both produce those rows.
@@ -4949,7 +4992,14 @@ async function submitPostgresSchedulerRunTransaction(
4949
4992
  'graphHash', $5::text,
4950
4993
  'runtimeBackend', $6::text,
4951
4994
  'schedulerBackend', launch_payload.launch_json ->> 'schedulerBackend',
4952
- 'schedulerSchema', launch_payload.launch_json ->> 'schedulerSchema',
4995
+ 'schedulerSchema', COALESCE(launch_payload.launch_json ->> 'schedulerSchema', '${schemaName(options)}'),
4996
+ 'runLogStore', CASE WHEN EXISTS (
4997
+ SELECT 1 FROM pg_proc AS proc
4998
+ JOIN pg_namespace AS namespace ON namespace.oid = proc.pronamespace
4999
+ WHERE namespace.nspname = '${schemaName(options)}'
5000
+ AND proc.proname = 'search_outbox_event'
5001
+ AND obj_description(proc.oid, 'pg_proc') = '${RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER}'
5002
+ ) THEN 'planet_scale' END,
4953
5003
  'executionProfile', launch_payload.launch_json ->> 'executionProfile',
4954
5004
  'runtimeReleaseId', launch_payload.launch_json ->> 'runtimeReleaseId',
4955
5005
  'runtimeDeployVersion', launch_payload.launch_json ->> 'runtimeDeployVersion',
@@ -6023,7 +6073,14 @@ export async function firePostgresSchedulerCronSchedule(
6023
6073
  'graphHash', launch_payload.launch_json ->> 'graphHash',
6024
6074
  'runtimeBackend', launch_payload.launch_json ->> 'runtimeBackend',
6025
6075
  'schedulerBackend', launch_payload.launch_json ->> 'schedulerBackend',
6026
- 'schedulerSchema', launch_payload.launch_json ->> 'schedulerSchema',
6076
+ 'schedulerSchema', COALESCE(launch_payload.launch_json ->> 'schedulerSchema', '${schemaName(options)}'),
6077
+ 'runLogStore', CASE WHEN EXISTS (
6078
+ SELECT 1 FROM pg_proc AS proc
6079
+ JOIN pg_namespace AS namespace ON namespace.oid = proc.pronamespace
6080
+ WHERE namespace.nspname = '${schemaName(options)}'
6081
+ AND proc.proname = 'search_outbox_event'
6082
+ AND obj_description(proc.oid, 'pg_proc') = '${RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER}'
6083
+ ) THEN 'planet_scale' END,
6027
6084
  'executionProfile', launch_payload.launch_json ->> 'executionProfile',
6028
6085
  'runtimeReleaseId', launch_payload.launch_json ->> 'runtimeReleaseId',
6029
6086
  'runtimeDeployVersion', launch_payload.launch_json ->> 'runtimeDeployVersion',
@@ -8472,6 +8529,36 @@ export type PostgresSchedulerAttemptAuthority = Pick<
8472
8529
  (typeof POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS)[number]
8473
8530
  >;
8474
8531
 
8532
+ type PostgresSchedulerAuthorityRow = PostgresSchedulerAttemptAuthority & {
8533
+ launch_required_fields_valid: boolean;
8534
+ };
8535
+
8536
+ // Keep JSON types intact: text columns would coerce malformed launch scalars.
8537
+ const POSTGRES_SCHEDULER_AUTHORITY_COLUMNS = [
8538
+ ...POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS,
8539
+ 'playId',
8540
+ 'executorToken',
8541
+ ]
8542
+ .map((field) => `${quoteIdent(field)} jsonb`)
8543
+ .join(', ');
8544
+ const POSTGRES_SCHEDULER_AUTHORITY_SELECT = `${POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS.map((field) => `launch.${quoteIdent(field)}`).join(', ')},
8545
+ ${['runId', 'playId', 'playName', 'executorToken', 'baseUrl']
8546
+ .map((field) => `jsonb_typeof(launch.${quoteIdent(field)}) = 'string'`)
8547
+ .join(' AND ')} AS launch_required_fields_valid`;
8548
+
8549
+ function parsePostgresSchedulerAuthorityRow(
8550
+ row: PostgresSchedulerAuthorityRow | undefined,
8551
+ ): PostgresSchedulerAttemptAuthority | null {
8552
+ if (!row) return null;
8553
+ const { launch_required_fields_valid, ...authority } = row;
8554
+ if (launch_required_fields_valid !== true) {
8555
+ throw new Error(
8556
+ 'Postgres scheduler launch_json is missing required fields.',
8557
+ );
8558
+ }
8559
+ return authority;
8560
+ }
8561
+
8475
8562
  /**
8476
8563
  * Prove the same attempt lease as the full execution reader, returning only
8477
8564
  * the token issuer's authority. Input, graph, and artifact bytes never cross
@@ -8484,29 +8571,13 @@ export async function readPostgresSchedulerAttemptAuthorityForLease(
8484
8571
  input: { runId: string; attempt: number; leaseToken: string },
8485
8572
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
8486
8573
  ): Promise<PostgresSchedulerAttemptAuthority | null> {
8487
- const fields = POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS;
8488
- // Keep JSON types intact. A text record column would coerce malformed launch
8489
- // scalars into strings, bypassing the full reader's required-field checks.
8490
- const columns = [...fields, 'playId', 'executorToken']
8491
- .map((field) => `${quoteIdent(field)} jsonb`)
8492
- .join(', ');
8493
- const result = await client.query<
8494
- PostgresSchedulerAttemptAuthority & {
8495
- launch_required_fields_valid: boolean;
8496
- }
8497
- >(
8574
+ const result = await client.query<PostgresSchedulerAuthorityRow>(
8498
8575
  `
8499
- SELECT ${fields.map((field) => `launch.${quoteIdent(field)}`).join(', ')},
8500
- ${['runId', 'playId', 'playName', 'executorToken', 'baseUrl']
8501
- .map(
8502
- (field) =>
8503
- `jsonb_typeof(launch.${quoteIdent(field)}) = 'string'`,
8504
- )
8505
- .join(' AND ')} AS launch_required_fields_valid
8576
+ SELECT ${POSTGRES_SCHEDULER_AUTHORITY_SELECT}
8506
8577
  FROM ${tableName('runs', options)} AS runs
8507
8578
  JOIN ${tableName('attempts', options)} AS attempts
8508
8579
  ON attempts.run_id = runs.run_id
8509
- CROSS JOIN LATERAL jsonb_to_record(runs.launch_json) AS launch(${columns})
8580
+ CROSS JOIN LATERAL jsonb_to_record(runs.launch_json) AS launch(${POSTGRES_SCHEDULER_AUTHORITY_COLUMNS})
8510
8581
  WHERE runs.run_id = $1
8511
8582
  AND runs.status IN ('queued', 'running', 'waiting')
8512
8583
  AND attempts.attempt = $2
@@ -8516,15 +8587,7 @@ export async function readPostgresSchedulerAttemptAuthorityForLease(
8516
8587
  `,
8517
8588
  [input.runId, input.attempt, input.leaseToken],
8518
8589
  );
8519
- const row = result.rows[0];
8520
- if (!row) return null;
8521
- const { launch_required_fields_valid, ...authority } = row;
8522
- if (launch_required_fields_valid !== true) {
8523
- throw new Error(
8524
- 'Postgres scheduler launch_json is missing required fields.',
8525
- );
8526
- }
8527
- return authority;
8590
+ return parsePostgresSchedulerAuthorityRow(result.rows[0]);
8528
8591
  }
8529
8592
 
8530
8593
  /**
@@ -9817,10 +9880,11 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
9817
9880
  const ledgerTerminalResult = terminalRunResultForLedger(terminalResult, {
9818
9881
  content: 'full',
9819
9882
  });
9820
- const terminalLogLines = terminalLogLinesForOutbox(
9883
+ const terminalLogProjection = terminalLogProjectionForOutbox(
9821
9884
  input.result,
9822
9885
  input.terminalLogOutbox,
9823
9886
  );
9887
+ const terminalLogLines = terminalLogProjection.lines;
9824
9888
  const runtimeTiming = extractRunnerRuntimeTiming(input.result);
9825
9889
  const terminalWorkCounterSegment = terminalWorkCounterSegmentFromRunnerResult(
9826
9890
  input.result,
@@ -9910,7 +9974,13 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
9910
9974
  AND target.attempt = $2
9911
9975
  AND target.status IN ('running', 'waiting')
9912
9976
  AND target.lease_token = $11
9913
- AND (target.runner_result_json IS NULL OR target.runner_result_json = $3::jsonb)
9977
+ AND (
9978
+ target.runner_result_json IS NULL
9979
+ OR target.runner_result_json = $3::jsonb
9980
+ OR target.runner_result_digest = encode(
9981
+ sha256(convert_to(($3::jsonb)::text, 'UTF8')), 'hex'
9982
+ )
9983
+ )
9914
9984
  AND run.run_id = target.run_id
9915
9985
  AND NOT EXISTS (
9916
9986
  SELECT 1
@@ -10070,9 +10140,10 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
10070
10140
  jsonb_build_object(
10071
10141
  'output', $6::jsonb,
10072
10142
  'terminalLogLines', $7::jsonb,
10143
+ 'terminalLogContexts', $17::jsonb,
10073
10144
  'terminalProgressEvents', $8::jsonb,
10074
10145
  'resultSummary', $16::jsonb,
10075
- 'terminalWorkCounterSegment', $17::jsonb
10146
+ 'terminalWorkCounterSegment', $18::jsonb
10076
10147
  ),
10077
10148
  ${causalOutboxCreatedAt('run_terminal.run_id', options)}
10078
10149
  FROM run_terminal
@@ -10107,6 +10178,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
10107
10178
  input.engineWake.wakeChannel,
10108
10179
  input.allowCappedCompletion === true,
10109
10180
  stringifyPostgresJson(resultSummary),
10181
+ JSON.stringify(terminalLogProjection.lineContexts),
10110
10182
  stringifyPostgresJson(terminalWorkCounterSegment ?? null),
10111
10183
  ],
10112
10184
  );
@@ -10618,6 +10690,36 @@ export async function claimPostgresSchedulerComputeBillingJobs(
10618
10690
  }));
10619
10691
  }
10620
10692
 
10693
+ /**
10694
+ * Issue terminal compute-billing authority under the same settlement lease as
10695
+ * the full worker reader, without hydrating execution input or retained graphs.
10696
+ * Historical launches keep their absent-descriptor fallback in the issuer.
10697
+ */
10698
+ export async function readComputeBillingDispatchAuthorityForLease(
10699
+ client: PostgresSchedulerQueryClient,
10700
+ input: { runId: string; leaseOwner: string; attemptCount?: number },
10701
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
10702
+ ): Promise<PostgresSchedulerAttemptAuthority | null> {
10703
+ const result = await client.query<PostgresSchedulerAuthorityRow>(
10704
+ `
10705
+ SELECT ${POSTGRES_SCHEDULER_AUTHORITY_SELECT}
10706
+ FROM ${tableName('compute_billing_jobs_v2', options)} AS job
10707
+ JOIN ${tableName('runs', options)} AS runs ON runs.run_id = job.run_id
10708
+ CROSS JOIN LATERAL jsonb_to_record(runs.launch_json)
10709
+ AS launch(${POSTGRES_SCHEDULER_AUTHORITY_COLUMNS})
10710
+ WHERE job.run_id = $1
10711
+ AND job.status = 'settling'
10712
+ AND job.lease_owner = $2
10713
+ AND job.lease_expires_at > now()
10714
+ AND ($3::integer IS NULL OR job.attempt_count = $3)
10715
+ AND runs.status IN ('completed', 'failed', 'cancelled')
10716
+ LIMIT 1
10717
+ `,
10718
+ [input.runId, input.leaseOwner, input.attemptCount ?? null],
10719
+ );
10720
+ return parsePostgresSchedulerAuthorityRow(result.rows[0]);
10721
+ }
10722
+
10621
10723
  export async function readComputeBillingDispatchLaunchForLease(
10622
10724
  client: PostgresSchedulerQueryClient,
10623
10725
  input: { runId: string; leaseOwner: string; attemptCount?: number },
@@ -12075,7 +12177,8 @@ export async function isPostgresSchedulerRunnerProgressAttemptCurrent(
12075
12177
  * Bumps `heartbeat_at` on the runner's attempt row (the same liveness column
12076
12178
  * the worker's own state transitions maintain — no new schema) and returns the
12077
12179
  * play-level run status so the gateway can put cancellation on the heartbeat
12078
- * channel. Returns null when the run does not exist.
12180
+ * channel. Returns null when the run is missing or this attempt is no longer
12181
+ * current while the run remains active.
12079
12182
  */
12080
12183
  export async function heartbeatPostgresSchedulerDetachedRunner(
12081
12184
  client: PostgresSchedulerQueryClient,
@@ -12125,7 +12228,26 @@ export async function heartbeatPostgresSchedulerDetachedRunner(
12125
12228
  `,
12126
12229
  [input.runId, input.attempt],
12127
12230
  );
12128
- return heartbeat.rows[0]?.status ?? null;
12231
+ const status = heartbeat.rows[0]?.status;
12232
+ if (status) return status;
12233
+
12234
+ // A terminal writer can commit while the UPDATE waits for its attempt lock.
12235
+ // PostgreSQL then rechecks the changed attempt and skips it, but the UNION
12236
+ // still reads the statement's earlier snapshot. Read terminal state with a
12237
+ // fresh snapshot after that wait; never acknowledge a replacement's active
12238
+ // state without renewing this exact attempt through the fence above.
12239
+ const terminal = await client.query<{
12240
+ status: 'completed' | 'failed' | 'cancelled';
12241
+ }>(
12242
+ `
12243
+ SELECT status
12244
+ FROM ${tableName('runs', options)}
12245
+ WHERE run_id = $1
12246
+ AND status IN ('completed', 'failed', 'cancelled')
12247
+ `,
12248
+ [input.runId],
12249
+ );
12250
+ return terminal.rows[0]?.status ?? null;
12129
12251
  }
12130
12252
 
12131
12253
  export async function claimPostgresSchedulerSignal(
@@ -12214,6 +12336,7 @@ export async function completePostgresSchedulerAttempt(
12214
12336
  rowOutcomes?: unknown;
12215
12337
  runtimeTiming?: PlayRunnerRuntimeTiming | null;
12216
12338
  terminalLogLines?: readonly string[] | null;
12339
+ terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
12217
12340
  terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
12218
12341
  },
12219
12342
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
@@ -12223,10 +12346,25 @@ export async function completePostgresSchedulerAttempt(
12223
12346
  content: 'full',
12224
12347
  });
12225
12348
  const resultSummary = terminalRowOutcomesSummary(input.rowOutcomes);
12226
- const terminalLogLines = (input.terminalLogLines ?? [])
12227
- .map((line) => line.trim())
12228
- .filter(Boolean)
12349
+ if (
12350
+ input.terminalLogContexts != null &&
12351
+ input.terminalLogContexts.length !== (input.terminalLogLines ?? []).length
12352
+ ) {
12353
+ throw new TypeError(
12354
+ 'Terminal log contexts must align with terminal log lines.',
12355
+ );
12356
+ }
12357
+ const terminalLogEntries = (input.terminalLogLines ?? [])
12358
+ .map((line, index) => ({
12359
+ line: line.trim(),
12360
+ context: input.terminalLogContexts?.[index] ?? null,
12361
+ }))
12362
+ .filter((entry) => entry.line.length > 0)
12229
12363
  .slice(-LOG_TAIL_LIMIT);
12364
+ const terminalLogLines = terminalLogEntries.map((entry) => entry.line);
12365
+ const terminalLogContexts = terminalLogEntries.map((entry) =>
12366
+ normalizePlayLogContext(entry.context),
12367
+ );
12230
12368
  const completed = await client.query<{ run_id: string }>(
12231
12369
  `
12232
12370
  WITH completed AS (
@@ -12402,8 +12540,9 @@ export async function completePostgresSchedulerAttempt(
12402
12540
  jsonb_build_object(
12403
12541
  'output', $9::jsonb,
12404
12542
  'terminalLogLines', $8::jsonb,
12543
+ 'terminalLogContexts', $11::jsonb,
12405
12544
  'resultSummary', $10::jsonb,
12406
- 'terminalWorkCounterSegment', $11::jsonb
12545
+ 'terminalWorkCounterSegment', $12::jsonb
12407
12546
  ),
12408
12547
  ${causalOutboxCreatedAt('run_terminal.run_id', options)}
12409
12548
  FROM run_terminal
@@ -12425,6 +12564,7 @@ export async function completePostgresSchedulerAttempt(
12425
12564
  JSON.stringify(terminalLogLines),
12426
12565
  stringifyPostgresJson(ledgerTerminalResult ?? null),
12427
12566
  stringifyPostgresJson(resultSummary),
12567
+ JSON.stringify(terminalLogContexts),
12428
12568
  stringifyPostgresJson(input.terminalWorkCounterSegment ?? null),
12429
12569
  ],
12430
12570
  );
@@ -12443,6 +12583,7 @@ export async function failPostgresSchedulerAttempt(
12443
12583
  error: string;
12444
12584
  result?: unknown;
12445
12585
  terminalLogLines?: readonly string[] | null;
12586
+ terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
12446
12587
  /** Refuse to overwrite an authoritative detached runner terminal that won
12447
12588
  * after claim. Resumable `suspended` receipts may be superseded by the
12448
12589
  * hard ceiling in the same fenced statement. */
@@ -12478,10 +12619,25 @@ export async function failPostgresSchedulerAttempt(
12478
12619
  );
12479
12620
  // Keep the recovery tail separate from result: large failures are represented
12480
12621
  // by a result reference, but their customer guidance must still be logged.
12481
- const terminalLogLines = (input.terminalLogLines ?? [])
12482
- .map((line) => line.trim())
12483
- .filter(Boolean)
12622
+ if (
12623
+ input.terminalLogContexts != null &&
12624
+ input.terminalLogContexts.length !== (input.terminalLogLines ?? []).length
12625
+ ) {
12626
+ throw new TypeError(
12627
+ 'Terminal log contexts must align with terminal log lines.',
12628
+ );
12629
+ }
12630
+ const terminalLogEntries = (input.terminalLogLines ?? [])
12631
+ .map((line, index) => ({
12632
+ line: line.trim(),
12633
+ context: input.terminalLogContexts?.[index] ?? null,
12634
+ }))
12635
+ .filter((entry) => entry.line.length > 0)
12484
12636
  .slice(-LOG_TAIL_LIMIT);
12637
+ const terminalLogLines = terminalLogEntries.map((entry) => entry.line);
12638
+ const terminalLogContexts = terminalLogEntries.map((entry) =>
12639
+ normalizePlayLogContext(entry.context),
12640
+ );
12485
12641
  const failed = await client.query<{ run_id: string }>(
12486
12642
  `
12487
12643
  WITH failed AS (
@@ -12717,8 +12873,9 @@ export async function failPostgresSchedulerAttempt(
12717
12873
  'error', $4::jsonb ->> 'message',
12718
12874
  'result', $8::jsonb,
12719
12875
  'terminalLogLines', $12::jsonb,
12876
+ 'terminalLogContexts', $14::jsonb,
12720
12877
  'resultSummary', $13::jsonb,
12721
- 'terminalWorkCounterSegment', $14::jsonb
12878
+ 'terminalWorkCounterSegment', $15::jsonb
12722
12879
  ),
12723
12880
  ${causalOutboxCreatedAt('run_terminal.run_id', options)}
12724
12881
  FROM run_terminal
@@ -12743,6 +12900,7 @@ export async function failPostgresSchedulerAttempt(
12743
12900
  input.preserveRunnerTerminal === true,
12744
12901
  JSON.stringify(terminalLogLines),
12745
12902
  stringifyPostgresJson(resultSummary),
12903
+ JSON.stringify(terminalLogContexts),
12746
12904
  stringifyPostgresJson(terminalWorkCounterSegment ?? null),
12747
12905
  ],
12748
12906
  );
@@ -13431,37 +13589,62 @@ function boundedRunLogTailMetadata(logs: string[]): {
13431
13589
  return tail.length > 0 ? { tail, totalLogCount: logs.length } : null;
13432
13590
  }
13433
13591
 
13434
- function terminalLogLinesForOutbox(
13592
+ function terminalLogProjectionForOutbox(
13435
13593
  runnerResult: unknown,
13436
13594
  mode: PostgresSchedulerTerminalLogOutbox | undefined,
13437
- ): string[] {
13595
+ ): { lines: string[]; lineContexts: Array<PlayLogContext | null> } {
13438
13596
  if (
13439
13597
  mode !== 'ctx_log_tail' ||
13440
13598
  !runnerResult ||
13441
13599
  typeof runnerResult !== 'object'
13442
13600
  ) {
13443
- return [];
13601
+ return { lines: [], lineContexts: [] };
13444
13602
  }
13445
- const logs = (runnerResult as { logs?: unknown }).logs;
13603
+ const result = runnerResult as {
13604
+ logs?: unknown;
13605
+ logContexts?: unknown;
13606
+ };
13607
+ const logs = result.logs;
13446
13608
  if (!Array.isArray(logs)) {
13447
- return [];
13609
+ return { lines: [], lineContexts: [] };
13610
+ }
13611
+ if (
13612
+ result.logContexts !== undefined &&
13613
+ (!Array.isArray(result.logContexts) ||
13614
+ result.logContexts.length !== logs.length)
13615
+ ) {
13616
+ throw new TypeError('Terminal log contexts must align with runner logs.');
13448
13617
  }
13449
- return logs
13450
- .filter((line): line is string => {
13451
- if (typeof line !== 'string') return false;
13618
+ const logContexts = Array.isArray(result.logContexts)
13619
+ ? result.logContexts
13620
+ : undefined;
13621
+ const entries = logs
13622
+ .map((line, index) => {
13623
+ if (typeof line !== 'string') return null;
13452
13624
  const normalized = readProvenanceTag(line).line.trim();
13453
13625
  // Retain bounded terminal measurement summaries alongside timestamped
13454
13626
  // customer/recovery lines after a detached runner has exited.
13455
- return (
13627
+ const retained =
13456
13628
  CTX_LOG_TIMESTAMP_PREFIX_RE.test(normalized) ||
13457
13629
  normalized.startsWith('[runtime.failure_diagnostic] ') ||
13458
13630
  normalized.startsWith('[runtime.phase_summary] ') ||
13459
13631
  normalized.startsWith('[runtime.logs] ') ||
13460
- normalized.startsWith('[runtime.log_transport] ')
13461
- );
13632
+ normalized.startsWith('[runtime.log_transport] ');
13633
+ if (!retained) return null;
13634
+ return {
13635
+ line: line.trim(),
13636
+ context: normalizePlayLogContext(logContexts?.[index]),
13637
+ };
13462
13638
  })
13463
- .map((line) => line.trim())
13639
+ .filter(
13640
+ (entry): entry is { line: string; context: PlayLogContext | null } =>
13641
+ entry !== null,
13642
+ )
13464
13643
  .slice(-LOG_TAIL_LIMIT);
13644
+ return {
13645
+ lines: entries.map((entry) => entry.line),
13646
+ lineContexts: entries.map((entry) => entry.context),
13647
+ };
13465
13648
  }
13466
13649
 
13467
13650
  export function attachRunnerWorkProgressMetadata(
@@ -13694,6 +13877,10 @@ export async function executePostgresSchedulerClaim(
13694
13877
  { logMetadata: options.completionLogMetadata ?? 'attach' },
13695
13878
  )
13696
13879
  : output;
13880
+ const terminalLogProjection = terminalLogProjectionForOutbox(
13881
+ output,
13882
+ options.terminalLogOutbox,
13883
+ );
13697
13884
  const completed = await completePostgresSchedulerAttempt(
13698
13885
  client,
13699
13886
  {
@@ -13703,14 +13890,12 @@ export async function executePostgresSchedulerClaim(
13703
13890
  output: terminalOutput,
13704
13891
  rowOutcomes: (output as { rowOutcomes?: unknown } | null)?.rowOutcomes,
13705
13892
  runtimeTiming: extractRunnerRuntimeTiming(output),
13893
+ terminalLogLines: terminalLogProjection.lines,
13894
+ terminalLogContexts: terminalLogProjection.lineContexts,
13706
13895
  terminalWorkCounterSegment: terminalWorkCounterSegmentFromRunnerResult(
13707
13896
  output,
13708
13897
  claim.attempt,
13709
13898
  ),
13710
- terminalLogLines: terminalLogLinesForOutbox(
13711
- output,
13712
- options.terminalLogOutbox,
13713
- ),
13714
13899
  },
13715
13900
  options,
13716
13901
  );
@@ -13754,6 +13939,10 @@ export async function executePostgresSchedulerClaim(
13754
13939
  checkpoint: null,
13755
13940
  tableNamespace: null,
13756
13941
  };
13942
+ const terminalLogProjection = terminalLogProjectionForOutbox(
13943
+ terminalFailureResult,
13944
+ options.terminalLogOutbox,
13945
+ );
13757
13946
  const failed = await failPostgresSchedulerAttempt(
13758
13947
  client,
13759
13948
  {
@@ -13762,10 +13951,8 @@ export async function executePostgresSchedulerClaim(
13762
13951
  leaseToken: claim.leaseToken,
13763
13952
  error: message,
13764
13953
  result: terminalFailureResult,
13765
- terminalLogLines: terminalLogLinesForOutbox(
13766
- terminalFailureResult,
13767
- options.terminalLogOutbox,
13768
- ),
13954
+ terminalLogLines: terminalLogProjection.lines,
13955
+ terminalLogContexts: terminalLogProjection.lineContexts,
13769
13956
  sandboxCrashDiagnostic: options.sandboxCrashDiagnostic,
13770
13957
  },
13771
13958
  options,
@@ -17032,9 +17219,101 @@ export async function listPostgresSchedulerDeliveredOutboxRetentionCandidates(
17032
17219
  */
17033
17220
  export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
17034
17221
  client: PostgresSchedulerQueryClient,
17035
- input: { olderThan: Date; limit?: number },
17222
+ input: {
17223
+ olderThan: Date;
17224
+ logBodiesOlderThan?: Date;
17225
+ limit?: number;
17226
+ },
17036
17227
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
17037
17228
  ): Promise<PostgresSchedulerDeliveredOutboxRow[]> {
17229
+ if (input.logBodiesOlderThan) {
17230
+ const logCutoff = input.logBodiesOlderThan.getTime();
17231
+ if (
17232
+ !Number.isFinite(logCutoff) ||
17233
+ logCutoff > Date.now() - 60 * 60 * 1_000
17234
+ ) {
17235
+ throw new TypeError(
17236
+ 'Delivered outbox log-body cleanup cutoff must be at least one hour old.',
17237
+ );
17238
+ }
17239
+ const limit = normalizeOutboxRetentionLimit(input.limit);
17240
+ await client.query(
17241
+ `WITH candidates AS (
17242
+ SELECT event_id
17243
+ FROM ${tableName('outbox', options)} AS outbox
17244
+ WHERE delivered_at IS NOT NULL
17245
+ AND failed_at IS NULL
17246
+ AND delivered_at < $1::timestamptz
17247
+ AND (
17248
+ (event_type = 'run.events'
17249
+ AND jsonb_typeof(payload_json->'events') = 'array'
17250
+ AND EXISTS (
17251
+ SELECT 1
17252
+ FROM jsonb_array_elements(
17253
+ CASE WHEN jsonb_typeof(payload_json->'events') = 'array'
17254
+ THEN payload_json->'events' ELSE '[]'::jsonb END
17255
+ ) AS item
17256
+ WHERE item->>'type' = 'log.appended'
17257
+ AND (item ? 'lines' OR item ? 'lineContexts')
17258
+ ))
17259
+ OR (event_type IN ('run.completed', 'run.failed')
17260
+ AND (
17261
+ payload_json ? 'terminalLogLines'
17262
+ OR (jsonb_typeof(payload_json->'terminalProgressEvents') = 'array'
17263
+ AND EXISTS (
17264
+ SELECT 1
17265
+ FROM jsonb_array_elements(
17266
+ CASE WHEN jsonb_typeof(payload_json->'terminalProgressEvents') = 'array'
17267
+ THEN payload_json->'terminalProgressEvents' ELSE '[]'::jsonb END
17268
+ ) AS item
17269
+ WHERE item->>'type' = 'log.appended'
17270
+ AND (item ? 'lines' OR item ? 'lineContexts')
17271
+ ))
17272
+ ))
17273
+ )
17274
+ ORDER BY delivered_at ASC, event_id ASC
17275
+ LIMIT $2::integer
17276
+ FOR UPDATE SKIP LOCKED
17277
+ )
17278
+ UPDATE ${tableName('outbox', options)} AS outbox
17279
+ SET payload_json = CASE
17280
+ WHEN outbox.event_type = 'run.events' THEN jsonb_set(
17281
+ outbox.payload_json,
17282
+ '{events}',
17283
+ (SELECT jsonb_agg(
17284
+ CASE WHEN item.value->>'type' = 'log.appended'
17285
+ THEN item.value - 'lines' - 'lineContexts'
17286
+ ELSE item.value
17287
+ END ORDER BY item.ordinality
17288
+ )
17289
+ FROM jsonb_array_elements(
17290
+ CASE WHEN jsonb_typeof(outbox.payload_json->'events') = 'array'
17291
+ THEN outbox.payload_json->'events' ELSE '[]'::jsonb END
17292
+ )
17293
+ WITH ORDINALITY AS item(value, ordinality)),
17294
+ false
17295
+ )
17296
+ WHEN jsonb_typeof(outbox.payload_json->'terminalProgressEvents') = 'array'
17297
+ THEN jsonb_set(
17298
+ outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts',
17299
+ '{terminalProgressEvents}',
17300
+ coalesce((SELECT jsonb_agg(
17301
+ CASE WHEN item.value->>'type' = 'log.appended'
17302
+ THEN item.value - 'lines' - 'lineContexts'
17303
+ ELSE item.value
17304
+ END ORDER BY item.ordinality
17305
+ )
17306
+ FROM jsonb_array_elements(outbox.payload_json->'terminalProgressEvents')
17307
+ WITH ORDINALITY AS item(value, ordinality)), '[]'::jsonb),
17308
+ false
17309
+ )
17310
+ ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts'
17311
+ END
17312
+ FROM candidates
17313
+ WHERE outbox.event_id = candidates.event_id`,
17314
+ [input.logBodiesOlderThan.toISOString(), limit],
17315
+ );
17316
+ }
17038
17317
  const result = await client.query<{
17039
17318
  event_id: string;
17040
17319
  run_id: string;
@@ -17317,9 +17596,11 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
17317
17596
  const occurredAt = event.createdAt.getTime();
17318
17597
  if (event.eventType === 'run.events') {
17319
17598
  const events = payloadField(event.payload, 'events');
17599
+ const logsStored =
17600
+ payloadField(event.payload, 'logBodyStore') === 'planet_scale';
17320
17601
  if (
17321
17602
  !Array.isArray(events) ||
17322
- events.length === 0 ||
17603
+ (events.length === 0 && !logsStored) ||
17323
17604
  events.some(
17324
17605
  (value) =>
17325
17606
  !recordPayload(value) ||
@@ -17364,6 +17645,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
17364
17645
  event.runId,
17365
17646
  );
17366
17647
  const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
17648
+ const lineContexts = terminalLogContextsPayloadField(event.payload, lines);
17367
17649
  const resultSummary = payloadField(event.payload, 'resultSummary');
17368
17650
  return [
17369
17651
  ...progressEvents.map(normalizeOutboxIngressPreview),
@@ -17376,6 +17658,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
17376
17658
  occurredAt,
17377
17659
  source: 'worker',
17378
17660
  lines,
17661
+ ...(lineContexts ? { lineContexts } : {}),
17379
17662
  } as PlayRunLedgerEvent,
17380
17663
  ]),
17381
17664
  {
@@ -17404,6 +17687,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
17404
17687
  event.runId,
17405
17688
  );
17406
17689
  const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
17690
+ const lineContexts = terminalLogContextsPayloadField(event.payload, lines);
17407
17691
  const result = terminalRunResultForLedger(
17408
17692
  payloadField(event.payload, 'result'),
17409
17693
  );
@@ -17419,6 +17703,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
17419
17703
  occurredAt,
17420
17704
  source: 'worker',
17421
17705
  lines,
17706
+ ...(lineContexts ? { lineContexts } : {}),
17422
17707
  } as PlayRunLedgerEvent,
17423
17708
  ]),
17424
17709
  {
@@ -18236,6 +18521,9 @@ async function projectPostgresSchedulerCreatedEvent(
18236
18521
  runtimeBackend: value('runtimeBackend'),
18237
18522
  schedulerBackend: value('schedulerBackend'),
18238
18523
  schedulerSchema: value('schedulerSchema'),
18524
+ ...(value('runLogStore') === 'planet_scale'
18525
+ ? { runLogStore: 'planet_scale' as const }
18526
+ : {}),
18239
18527
  executionProfile: value('executionProfile'),
18240
18528
  runtimeReleaseId: value('runtimeReleaseId'),
18241
18529
  runtimeDeployVersion: value('runtimeDeployVersion'),
@@ -18666,6 +18954,20 @@ function stringArrayPayloadField(payload: unknown, key: string): string[] {
18666
18954
  : [];
18667
18955
  }
18668
18956
 
18957
+ function terminalLogContextsPayloadField(
18958
+ payload: unknown,
18959
+ lines: readonly string[],
18960
+ ): Array<PlayLogContext | null> | null {
18961
+ const value = payloadField(payload, 'terminalLogContexts');
18962
+ if (value === undefined) return null;
18963
+ if (!Array.isArray(value) || value.length !== lines.length) {
18964
+ throw new TypeError(
18965
+ 'Terminal log contexts must align with terminal log lines.',
18966
+ );
18967
+ }
18968
+ return value.map(normalizePlayLogContext);
18969
+ }
18970
+
18669
18971
  function ledgerEventsPayloadField(
18670
18972
  payload: unknown,
18671
18973
  key: string,