deepline 0.3.128 → 0.3.130

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -6,6 +6,9 @@ import {
   acquirePgNotifyHub,
   resolveWorkflowRuntimePostgresUrl,
   withRuntimeSchedulerClient,
+  withRuntimeSchedulerTransaction,
+  RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
+  RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
   type PgNotifyHub,
   type PgNotifyHubState,
   type RuntimeSchedulerPoolOptions,
@@ -20,6 +23,7 @@ import { PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA } from '../runtime-scheduler-topo
 import type { PlayRunnerRuntimeResource } from '../runner-backends/types';
 import {
   RuntimeSandboxOperationState,
+  type RuntimeSandboxProvider,
   runtimeSandboxProviderForResourceKind,
 } from './runtime-sandbox-operation';
 import {
@@ -53,6 +57,10 @@ import type {
   RuntimeStepReceipt,
   SkipRuntimeStepReceiptInput,
 } from '../ctx-types';
+import {
+  isRuntimeReceiptValueReference,
+  type RuntimeReceiptValueReference,
+} from '../runtime-receipt-value';
 import { PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS } from '../lease-policy';
 import { RUNTIME_RELIABILITY_POLICY } from '../runtime-reliability-policy';
 import { COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID } from '../durable-receipt-execution';
@@ -217,12 +225,46 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
  * interrupted deployment cannot claim a newer version without its required
  * objects.
  */
-// v38 adds bounded health indexes for the runtime-owned outbox, sandbox
-// cleanup, and compute-billing queues. The indexes are additive: v37 cron
-// scheduling, admission, and outbox projection remain valid while a deploy
-// expands the shared schema, then the marker records the completed health
-// capability for deployment tooling.
-export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 38;
+// v34 adds the delivered-outbox retention index. It is additive: old outbox
+// rows and their durable delivery identities remain readable during the
+// rolling window.
+// v35 adds the admission-priority outbox index and wake trigger. They remain
+// additive: the projector still leases every event from durable storage.
+// v36 adds durable rate-permit reservations, so provider cancellation can
+// distinguish an unused permit from one that may have reached the provider.
+// Additive: a v34 reader ignores the table.
+// v37 reserves immutable Integration Operation bodies before object upload.
+// An old gateway ignores the nullable column; a current writer refuses to
+// publish a reference without its fenced pending owner.
+// v38 adds the durable Gradient2 provider-concurrency checkpoint. Older
+// runners keep their declared cap; current runners only require the additive
+// fields after the deploy-owned schema marker has advanced.
+// v39 adds durable deduplication for buffered observations and durable
+// server-authored 429 attribution. A lost acquire response cannot replay
+// learned capacity, and a runner cannot forge occupancy/RTT feedback.
+// v40 combines Foundation's additive objects with main's blocked-outbox
+// oldest-first health index. Deployment must apply both branches' objects.
+// v41 adds the runtime-owned cron schedule cursor and its due-time index from
+// main's durable trigger admission (#5779). That branch numbered the same
+// objects v37, which this lineage had already spent on immutable Integration
+// Operation bodies, so the merged schema carries them here instead. The table
+// is additive on both sides: v40 admission and outbox projection stay valid
+// while a deploy expands the shared schema, and the marker then records the
+// completed runtime-cron capability for deployment tooling.
+// v42 absorbs main's bounded health indexes for the runtime-owned outbox,
+// sandbox cleanup, and compute-billing queues (#6086 lineage). That branch
+// numbered them v38, which this lineage had already spent, so the objects are
+// carried here instead; all nine are `CREATE INDEX IF NOT EXISTS` and additive,
+// so v41 admission and outbox projection stay valid while a deploy expands the
+// shared schema, and the marker then records the completed health capability.
+// v43 adds per-run fair-share tracking for provider rate buckets: the
+// `rate_requesters` table and a nullable `requester_id` on concurrency leases.
+// Both are additive; only an acquire that carries a run identity reads them, so
+// v42 runtime paths stay valid while a deploy expands the shared schema.
+// v44 moves the v39-v43 shared-schema expansion onto the online, statement-
+// scoped migration path. The reader contract is unchanged; the marker ensures
+// deployment retries do not re-enter the old cross-table DDL transaction.
+export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 44;
 const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
   'scheduler_schema_migrations';
 export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
@@ -514,6 +556,8 @@ type ReceiptRow = {
   k: string;
   status: string;
   output_json: unknown | null;
+  /** Nullable during the inline/reference reader expansion. */
+  output_ref?: unknown | null;
   error: string | null;
   error_payload: unknown | null;
   failure_kind: number | string | null;
@@ -1004,8 +1048,18 @@ export type PostgresSchedulerSchemaShapeRequirements = Readonly<{
   blockedOutboxIndex?: boolean;
   admissionPriorityIndex?: boolean;
   terminalComputeBillingOutboxTrigger?: boolean;
+  /** v35 added the table; a v34 reader does not require it. */
+  rateReservations?: boolean;
+  /** v36 added the pending immutable-value owner; a v35 reader ignores it. */
+  integrationOperationValueReservation?: boolean;
+  /** v37 added the durable Gradient2 checkpoint; a v36 reader ignores it. */
+  gradient2ConcurrencyCheckpoint?: boolean;
+  /** v38 makes buffered Gradient2 observations idempotent; v37 ignores it. */
+  gradient2ObservationDeduplication?: boolean;
   outboxWakeTrigger?: boolean;
   cronScheduleSchema?: boolean;
+  /** v43 added per-run fair-share tracking for rate buckets; v42 ignores it. */
+  rateRequesterFairShare?: boolean;
 }>;
 
 export async function hasCurrentPostgresSchedulerSchemaShape(
@@ -1077,7 +1131,51 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
             AND column_default IS NOT NULL
         )
         AND (
-          $24::boolean = false
+          $27::boolean = false
+          OR (
+          SELECT count(DISTINCT column_name)
+          FROM information_schema.columns
+          WHERE table_schema = $1
+            AND table_name = 'rate_bucket'
+            AND column_name IN (
+              'learned_concurrency_limit',
+              'concurrency_short_rtt_ms',
+              'concurrency_long_rtt_ms',
+              'concurrency_minimum_rtt_ms',
+              'concurrency_successful_samples',
+              'concurrency_window_limit',
+              'concurrency_has_observed_throttle'
+            )
+          ) = 7
+        )
+        AND (
+          $28::boolean = false
+          OR (
+            to_regclass($29) IS NOT NULL
+            AND EXISTS (
+              SELECT 1
+              FROM information_schema.columns
+              WHERE table_schema = $1
+                AND table_name = 'rate_concurrency_observers'
+                AND column_name = 'expires_at_ms'
+                AND data_type = 'bigint'
+                AND is_nullable = 'NO'
+            )
+            AND (
+              SELECT count(DISTINCT column_name)
+              FROM information_schema.columns
+              WHERE table_schema = $1
+                AND table_name = 'rate_concurrency_leases'
+                AND column_name IN (
+                  'in_flight_at_start',
+                  'target_requests_per_second',
+                  'minimum_rtt_ms'
+                )
+            ) = 3
+          )
+        )
+        AND (
+          $31::boolean = false
           OR EXISTS (
             SELECT 1
             FROM pg_index AS index_row
@@ -1088,8 +1186,8 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
             WHERE index_namespace.nspname = $1
               AND index_class.relname = 'runtime_scheduler_outbox_admission_pending_idx'
               AND index_row.indrelid = to_regclass($8)
-              AND index_row.indisready
-              AND index_row.indisvalid
+            AND index_row.indisready
+            AND index_row.indisvalid
           )
         )
         AND (
@@ -1103,7 +1201,7 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
           )
         )
         AND (
-          $23::boolean = false
+          $30::boolean = false
           OR EXISTS (
             SELECT 1
             FROM pg_trigger
@@ -1254,10 +1352,24 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
         AND to_regclass($20) IS NOT NULL
         AND to_regclass($21) IS NOT NULL
         AND to_regclass($22) IS NOT NULL
+        AND ($24::boolean = false OR to_regclass($23) IS NOT NULL)
+        AND (
+          $34::boolean = false
+          OR (
+            to_regclass($35) IS NOT NULL
+            AND EXISTS (
+              SELECT 1
+              FROM information_schema.columns
+              WHERE table_schema = $1
+                AND table_name = 'rate_concurrency_leases'
+                AND column_name = 'requester_id'
+            )
+          )
+        )
         AND (
-          $25::boolean = false
+          $32::boolean = false
           OR (
-            to_regclass($26) IS NOT NULL
+            to_regclass($33) IS NOT NULL
             AND (
               SELECT count(DISTINCT column_name)
               FROM information_schema.columns
@@ -1291,7 +1403,7 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
                 ON index_namespace.oid = index_class.relnamespace
               WHERE index_namespace.nspname = $1
                 AND index_class.relname = 'runtime_scheduler_cron_schedules_due_idx'
-                AND index_row.indrelid = to_regclass($26)
+                AND index_row.indrelid = to_regclass($33)
                 AND index_row.indisready
                 AND index_row.indisvalid
             )
@@ -1315,6 +1427,7 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
             AND constraint_row.contype = 'p'
         ) = 'operation_key'
         AND (
+          $26::boolean = false OR (
           SELECT count(DISTINCT column_name)
           FROM information_schema.columns
           WHERE table_schema = $1
@@ -1323,9 +1436,35 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
               'operation_request_digest',
               'legacy_request_fingerprint',
               'canonical_outcome_version',
-              'canonical_outcome_json'
+              'canonical_outcome_json',
+              'canonical_outcome_ref',
+              'canonical_outcome_pending_ref'
             )
-        ) = 4
+          ) = 6
+        )
+        AND (
+          $26::boolean = false OR (
+            to_regclass($25) IS NOT NULL
+            AND (
+              SELECT count(DISTINCT column_name)
+                FROM information_schema.columns
+               WHERE table_schema = $1
+                 AND table_name = 'runtime_integration_operation_value_retirements'
+                 AND column_name IN (
+                   'org_id',
+                   'object_key',
+                   'object_digest',
+                   'operation_key',
+                   'reference_json',
+                   'state',
+                   'claim_token',
+                   'created_at',
+                   'updated_at',
+                   'deletion_confirmed_at'
+                 )
+            ) = 10
+          )
+        )
         AND (
           SELECT count(DISTINCT column_name)
           FROM information_schema.columns
@@ -1538,10 +1677,19 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
       `${quotedSchema}.${quoteIdent('runtime_gateway_reader_fences')}`,
       `${quotedSchema}.${quoteIdent('runtime_sandbox_operations')}`,
       `${quotedSchema}.${quoteIdent('outbox_repair_operations')}`,
+      `${quotedSchema}.${quoteIdent('rate_reservations')}`,
+      requirements.rateReservations !== false,
+      `${quotedSchema}.${quoteIdent('runtime_integration_operation_value_retirements')}`,
+      requirements.integrationOperationValueReservation !== false,
+      requirements.gradient2ConcurrencyCheckpoint !== false,
+      requirements.gradient2ObservationDeduplication !== false,
+      `${quotedSchema}.${quoteIdent('rate_concurrency_observers')}`,
       requirements.outboxWakeTrigger !== false,
       requirements.admissionPriorityIndex !== false,
       requirements.cronScheduleSchema !== false,
       `${quotedSchema}.${quoteIdent(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE)}`,
+      requirements.rateRequesterFairShare !== false,
+      `${quotedSchema}.${quoteIdent('rate_requesters')}`,
     ],
   );
   if (currentShape.rows[0]?.ready !== true) return false;
@@ -1697,6 +1845,89 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
   return runtimeAdminIndexes.rows[0]?.ready === true;
 }
 
+/**
+ * Expand the shared scheduler schema without retaining locks across tables.
+ *
+ * The deployment caller gives every statement its own short transaction with
+ * transaction-local lock and statement timeouts. Never wrap the whole helper
+ * in one transaction: old workers acquire the same live-table locks in a
+ * different order and can deadlock the release. The helper does not write the
+ * schema marker; callers record it only after the complete expansion and all
+ * concurrent indexes are verified.
+ */
+export async function ensurePostgresSchedulerOnlineAdditiveExpansion(
+  client: PostgresSchedulerQueryClient,
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<void> {
+  const schema = schemaName(options);
+  await client.query(`
+    ALTER TABLE ${tableName('receipts', options)}
+      ADD COLUMN IF NOT EXISTS output_ref jsonb
+  `);
+  await client.query(`
+    ALTER TABLE ${tableName('integration_invocation_receipts', options)}
+      ADD COLUMN IF NOT EXISTS canonical_outcome_ref jsonb,
+      ADD COLUMN IF NOT EXISTS canonical_outcome_pending_ref jsonb
+  `);
+  await client.query(`
+    CREATE TABLE IF NOT EXISTS ${tableName(
+      'runtime_integration_operation_value_retirements',
+      options,
+    )} (
+      org_id text NOT NULL,
+      object_key text NOT NULL,
+      object_digest text NOT NULL,
+      operation_key text NOT NULL,
+      reference_json jsonb NOT NULL,
+      state text NOT NULL,
+      claim_token text,
+      created_at timestamptz NOT NULL DEFAULT now(),
+      updated_at timestamptz NOT NULL DEFAULT now(),
+      deletion_confirmed_at timestamptz,
+      PRIMARY KEY (org_id, object_key, object_digest),
+      CHECK (state IN ('pending', 'deleting', 'deleted', 'cancelled')),
+      CHECK ((state = 'deleting') = (claim_token IS NOT NULL))
+    )
+  `);
+  await client.query(`
+    CREATE INDEX IF NOT EXISTS ${quoteIdent(
+      `${schema}_integration_operation_value_retirements_pending_idx`,
+    )}
+      ON ${tableName(
+        'runtime_integration_operation_value_retirements',
+        options,
+      )} (state, updated_at)
+  `);
+}
+
+/** Read the highest deployment-owned scheduler schema marker. */
+export async function readPostgresSchedulerSchemaVersion(
+  client: PostgresSchedulerQueryClient,
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<number> {
+  const result = await client.query<{ version: number | null }>(
+    `SELECT max(version) AS version FROM ${tableName(
+      POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE,
+      options,
+    )}`,
+  );
+  return Number(result.rows[0]?.version ?? 0);
+}
+
+/** Record the deployment-owned marker after an online expansion is complete. */
+export async function recordPostgresSchedulerSchemaVersion(
+  client: PostgresSchedulerQueryClient,
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<void> {
+  await client.query(
+    `INSERT INTO ${tableName(
+      POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE,
+      options,
+    )} (version) VALUES ($1) ON CONFLICT (version) DO NOTHING`,
+    [POSTGRES_SCHEDULER_SCHEMA_VERSION],
+  );
+}
+
 /**
  * Deployment completion check. Migration bookkeeping and the full target
  * shape belong to deployment tooling, never request or worker admission.
@@ -2622,6 +2853,13 @@ export async function ensurePostgresSchedulerSchema(
       updated_at timestamptz NOT NULL DEFAULT now(),
       PRIMARY KEY (org_id, k)
     )
+  `);
+    // Readers first: older rows keep `output_json`; a referenced body is
+    // published only after it was acknowledged by object storage. The nullable
+    // expansion keeps mixed scheduler-schema lanes readable during rollout.
+    await client.query(`
+    ALTER TABLE ${tableName('receipts', options)}
+      ADD COLUMN IF NOT EXISTS output_ref jsonb
   `);
     await client.query(`
     CREATE TABLE IF NOT EXISTS ${tableName(
@@ -2653,11 +2891,45 @@ export async function ensurePostgresSchedulerSchema(
       ADD COLUMN IF NOT EXISTS operation_request_digest text,
       ADD COLUMN IF NOT EXISTS legacy_request_fingerprint text,
       ADD COLUMN IF NOT EXISTS canonical_outcome_version smallint,
-      ADD COLUMN IF NOT EXISTS canonical_outcome_json jsonb
+      ADD COLUMN IF NOT EXISTS canonical_outcome_json jsonb,
+      ADD COLUMN IF NOT EXISTS canonical_outcome_ref jsonb,
+      ADD COLUMN IF NOT EXISTS canonical_outcome_pending_ref jsonb
   `);
     await client.query(`
     ALTER TABLE ${tableName('integration_invocation_receipts', options)}
       ADD COLUMN IF NOT EXISTS parent_run_id text NOT NULL DEFAULT ''
+  `);
+    // A reference stays live while it is attached to an operation row. A
+    // later recovery detaches failed or incomplete output into this small,
+    // fenced journal; the gateway can then delete the immutable object and
+    // confirm absence before the journal says it is gone. This is deliberately
+    // a journal rather than an object TTL: completed operation outcomes remain
+    // the replay authority until the operation itself is explicitly replaced.
+    await client.query(`
+    CREATE TABLE IF NOT EXISTS ${tableName(
+      'runtime_integration_operation_value_retirements',
+      options,
+    )} (
+      org_id text NOT NULL,
+      object_key text NOT NULL,
+      object_digest text NOT NULL,
+      operation_key text NOT NULL,
+      reference_json jsonb NOT NULL,
+      state text NOT NULL CHECK (state IN ('pending', 'deleting', 'deleted', 'cancelled')),
+      claim_token text,
+      created_at timestamptz NOT NULL DEFAULT now(),
+      updated_at timestamptz NOT NULL DEFAULT now(),
+      deletion_confirmed_at timestamptz,
+      PRIMARY KEY (org_id, object_key, object_digest),
+      CHECK ((state = 'deleting') = (claim_token IS NOT NULL))
+    )
+  `);
+    await client.query(`
+    CREATE INDEX IF NOT EXISTS ${quoteIdent(
+      `${schema}_integration_operation_value_retirements_pending_idx`,
+    )}
+      ON ${tableName('runtime_integration_operation_value_retirements', options)}
+        (state, updated_at)
   `);
     // Gateway reader capability is a shared-storage compatibility fact. It is
     // intentionally separate from runtime release admission: worker lanes do
@@ -3961,6 +4233,72 @@ export async function ensureAbsurdSchema(
   }
 }
 
+/**
+ * The runtime is allowed to use the shared Absurd schema only after the
+ * deployment migration has installed its exact engine and Deepline
+ * compatibility contract. This is deliberately read-only: workers and
+ * serverless submitters must never take the global Absurd bootstrap lock or
+ * run DDL while serving a customer request.
+ */
+export class AbsurdSchemaNotReadyError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'AbsurdSchemaNotReadyError';
+  }
+}
+
+export async function assertAbsurdSchemaReady(
+  client: PostgresSchedulerQueryClient,
+): Promise<void> {
+  let result: {
+    rows: Array<{
+      version: string | null;
+      compatibilityVersion: string | null;
+    }>;
+  };
+  try {
+    result = await client.query<{
+      version: string | null;
+      compatibilityVersion: string | null;
+    }>(`
+      SELECT
+        absurd.get_schema_version() AS "version",
+        absurd.deepline_no_scheduler_org_cap_compatibility_version()
+          AS "compatibilityVersion"
+    `);
+  } catch (error) {
+    const code =
+      typeof error === 'object' && error !== null && 'code' in error
+        ? (error as { code?: unknown }).code
+        : undefined;
+    if (
+      code === '42883' ||
+      (error instanceof Error &&
+        /does not exist|undefined function/i.test(error.message))
+    ) {
+      throw new AbsurdSchemaNotReadyError(
+        'Absurd scheduler schema is absent or missing the Deepline compatibility marker; run the runtime scheduler migration before activating a lane.',
+      );
+    }
+    throw error;
+  }
+
+  const state = result.rows[0];
+  if (
+    state?.version !== ABSURD_SCHEMA_VERSION ||
+    state.compatibilityVersion !==
+      ABSURD_NO_SCHEDULER_ORG_CAP_COMPATIBILITY_VERSION
+  ) {
+    throw new AbsurdSchemaNotReadyError(
+      'Absurd scheduler schema is incompatible: ' +
+        `required_version=${ABSURD_SCHEMA_VERSION} observed_version=${state?.version ?? 'missing'} ` +
+        `required_compatibility=${ABSURD_NO_SCHEDULER_ORG_CAP_COMPATIBILITY_VERSION} ` +
+        `observed_compatibility=${state?.compatibilityVersion ?? 'missing'}. ` +
+        'Run the runtime scheduler migration before activating a lane.',
+    );
+  }
+}
+
 export async function submitPostgresSchedulerRun(
   client: PostgresSchedulerQueryClient,
   input: PlaySchedulerSubmitInput,
@@ -7834,6 +8172,65 @@ export async function recordPostgresSchedulerManagedSandboxOperationLive(
   return updated.rows.length === 1;
 }
 
+/**
+ * Answers one question a rejected promotion cannot answer on its own: did this
+ * attempt lose its fence, or did it merely re-drive a promotion it already
+ * completed?
+ *
+ * `recordPostgresSchedulerManagedSandboxOperationLive` is a one-shot
+ * `reserved -> live` compare-and-set, so it returns false both when the fence
+ * is gone and when this same fenced attempt already promoted this same sandbox.
+ * A provider retry inside one attempt legitimately re-reports an earlier
+ * sandbox (for example to stamp its billing end when it is retired), and
+ * reading that rejection as a lost fence terminates a run whose attempt is
+ * still fully owned.
+ *
+ * This proves the benign case from durable state rather than assuming it: it
+ * returns true only when the attempt row still carries this lease token in a
+ * live status AND the operation is already live for this exact sandbox. Any
+ * other shape leaves the caller's fence loss loud and unchanged.
+ */
+export async function readPostgresSchedulerManagedSandboxOperationAlreadyLive(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    runId: string;
+    attempt: number;
+    leaseToken: string;
+    operationKey: string;
+    provider: RuntimeSandboxProvider;
+    sandboxId: string;
+  },
+  options?: Pick<PostgresSchedulerOptions, 'schema'>,
+): Promise<boolean> {
+  const found = await client.query<{ operation_key: string }>(
+    `
+      SELECT operation.operation_key
+      FROM ${tableName('runtime_sandbox_operations', options)} AS operation
+      JOIN ${tableName('attempts', options)} AS attempt
+        ON attempt.run_id = operation.run_id
+       AND attempt.attempt = operation.attempt
+      WHERE operation.operation_key = $4
+        AND operation.run_id = $1
+        AND operation.attempt = $2
+        AND operation.provider = $5
+        AND operation.state = $6
+        AND operation.sandbox_id = $7
+        AND attempt.lease_token = $3
+        AND attempt.status IN ('claimed', 'running', 'waiting')
+    `,
+    [
+      input.runId,
+      input.attempt,
+      input.leaseToken,
+      input.operationKey,
+      input.provider,
+      RuntimeSandboxOperationState.Live,
+      input.sandboxId,
+    ],
+  );
+  return found.rows.length === 1;
+}
+
 /**
  * A create call that timed out may settle only after its caller has marked the
  * attempt terminal. Preserve the provider fact under that terminal attempt,
@@ -8661,6 +9058,10 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
     attempt: number;
     leaseToken: string;
     result: unknown;
+    /** Progress facts folded before direct completion. They travel with the
+     * terminal outbox so the ledger sees progress before terminal state in one
+     * ordered projection. */
+    terminalProgressEvents?: readonly unknown[];
     engineWake: {
       queueName: string;
       eventName: string;
@@ -8715,7 +9116,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
             OR run.status = 'running'
           )
           AND (
-            $14::boolean
+            $15::boolean
             OR run.launch_json #>> '{contractSnapshot,billingLimit,maxCreditsPerRun}' IS NULL
           )
         FOR UPDATE
@@ -8779,7 +9180,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
         WHERE target.run_id = $1
           AND target.attempt = $2
           AND target.status IN ('running', 'waiting')
-          AND target.lease_token = $10
+          AND target.lease_token = $11
           AND (target.runner_result_json IS NULL OR target.runner_result_json = $3::jsonb)
           AND run.run_id = target.run_id
           AND NOT EXISTS (
@@ -8939,21 +9340,22 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
           'run.completed',
           jsonb_build_object(
             'output', $6::jsonb,
-            'terminalLogLines', $7::jsonb
+            'terminalLogLines', $7::jsonb,
+            'terminalProgressEvents', $8::jsonb
           ),
           ${causalOutboxCreatedAt('run_terminal.run_id', options)}
         FROM run_terminal
         ON CONFLICT (event_id) DO NOTHING
       ),
       notify_terminal AS (
-        SELECT run_id, pg_notify($8, $9) FROM run_terminal
+        SELECT run_id, pg_notify($9, $10) FROM run_terminal
       ),
       engine_wake AS (
-        SELECT run_id, absurd.emit_event($11, $12, 'null'::jsonb)
+        SELECT run_id, absurd.emit_event($12, $13, 'null'::jsonb)
         FROM notify_terminal
       ),
       notify_engine AS (
-        SELECT run_id, pg_notify($13, $11) FROM engine_wake
+        SELECT run_id, pg_notify($14, $12) FROM engine_wake
       )
       SELECT run_id FROM notify_engine
     `,
@@ -8965,6 +9367,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
       stringifyPostgresJson(runtimeTiming ?? null),
       stringifyPostgresJson(ledgerTerminalResult ?? null),
       JSON.stringify(terminalLogLines),
+      JSON.stringify(input.terminalProgressEvents ?? []),
       postgresSchedulerNotifyChannel('terminal', options),
       JSON.stringify({ runId: input.runId, status: 'completed' }),
       input.leaseToken,
@@ -11081,6 +11484,7 @@ export async function failPostgresSchedulerAttempt(
     leaseToken: string;
     error: string;
     result?: unknown;
+    terminalLogLines?: readonly string[] | null;
     /** Refuse to overwrite a detached runner terminal that won after claim. */
     preserveRunnerTerminal?: boolean;
     sandboxCrashDiagnostic?: PostgresSchedulerSandboxCrashDiagnosticRequest;
@@ -11105,6 +11509,12 @@ export async function failPostgresSchedulerAttempt(
     terminalResult === undefined ? (input.result ?? null) : terminalResult,
     terminalResult === undefined ? {} : { content: 'full' },
   );
+  // Keep the recovery tail separate from result: large failures are represented
+  // by a result reference, but their customer guidance must still be logged.
+  const terminalLogLines = (input.terminalLogLines ?? [])
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .slice(-LOG_TAIL_LIMIT);
   const failed = await client.query<{ run_id: string }>(
     `
       WITH failed AS (
@@ -11337,7 +11747,8 @@ export async function failPostgresSchedulerAttempt(
           'run.failed',
           jsonb_build_object(
             'error', $4::jsonb ->> 'message',
-            'result', $8::jsonb
+            'result', $8::jsonb,
+            'terminalLogLines', $12::jsonb
           ),
           ${causalOutboxCreatedAt('run_terminal.run_id', options)}
         FROM run_terminal
@@ -11360,6 +11771,7 @@ export async function failPostgresSchedulerAttempt(
       runtimeTerminalReasonFromError(input.error),
       stringifyPostgresJson(input.sandboxCrashDiagnostic ?? null),
       input.preserveRunnerTerminal === true,
+      JSON.stringify(terminalLogLines),
     ],
   );
   return failed.rows.length > 0;
@@ -11424,7 +11836,21 @@ export async function cancelPostgresSchedulerRun(
         WHERE target.run_id = $1
           AND target.status IN ('queued', 'running', 'waiting')
           ${strandedEligibility}
-        RETURNING target.run_id, target.launch_json
+        RETURNING target.run_id, target.org_id, target.launch_json
+      ),
+      released_receipts AS (
+        -- Cancellation is a terminal, and a terminal releases everything the
+        -- run claimed. A receipt left 'running' behind a dead lease matches no
+        -- non-forced claim status set, so a successor would park on it for the
+        -- whole receipt wait budget instead of executing the call itself.
+        UPDATE ${tableName('receipts', options)} AS receipt
+        SET ${RELEASE_RECEIPT_TO_PENDING_SET_CLAUSE}
+        FROM cancelled_run
+        WHERE receipt.org_id = cancelled_run.org_id
+          AND receipt.status IN ('queued', 'running')
+          AND COALESCE(receipt.lease_owner_run_id, receipt.run_id)
+              = cancelled_run.run_id
+        RETURNING receipt.k
       ),
       cancelled_attempts AS (
         UPDATE ${tableName('attempts', options)}
@@ -11878,6 +12304,33 @@ function extractRunnerRuntimeTiming(
 
 const CTX_LOG_TIMESTAMP_PREFIX_RE = /^\[\d{4}-\d{2}-\d{2}T[\d:.]+Z\]\s+/;
 const RUN_LOG_TAIL_METADATA_MAX_BYTES = 128 * 1024;
+// The runner's terminal resource snapshot is a small aggregate (phase timings,
+// process CPU/RSS, per-map admission state). It rides in result metadata so
+// terminal diagnostics never depend on a bounded log tail.
+const RUNTIME_RESOURCES_METADATA_MAX_BYTES = 32 * 1024;
+
+function boundedRuntimeResourcesMetadata(
+  runnerResult: unknown,
+): Record<string, unknown> | null {
+  const stats = (runnerResult as { stats?: unknown }).stats;
+  if (!stats || typeof stats !== 'object' || Array.isArray(stats)) return null;
+  const runtimeResources = (stats as { runtimeResources?: unknown })
+    .runtimeResources;
+  if (
+    !runtimeResources ||
+    typeof runtimeResources !== 'object' ||
+    Array.isArray(runtimeResources)
+  ) {
+    return null;
+  }
+  if (
+    jsonByteLengthUpTo(runtimeResources, RUNTIME_RESOURCES_METADATA_MAX_BYTES)
+      .exceeded
+  ) {
+    return null;
+  }
+  return runtimeResources as Record<string, unknown>;
+}
 
 function boundedRunLogTailMetadata(logs: string[]): {
   tail: string[];
@@ -11915,11 +12368,18 @@ function terminalLogLinesForOutbox(
     return [];
   }
   return logs
-    .filter(
-      (line): line is string =>
-        typeof line === 'string' &&
-        CTX_LOG_TIMESTAMP_PREFIX_RE.test(readProvenanceTag(line).line.trim()),
-    )
+    .filter((line): line is string => {
+      if (typeof line !== 'string') return false;
+      const normalized = readProvenanceTag(line).line.trim();
+      // Retain bounded terminal measurement summaries alongside timestamped
+      // customer/recovery lines after a detached runner has exited.
+      return (
+        CTX_LOG_TIMESTAMP_PREFIX_RE.test(normalized) ||
+        normalized.startsWith('[runtime.phase_summary] ') ||
+        normalized.startsWith('[runtime.logs] ') ||
+        normalized.startsWith('[runtime.log_transport] ')
+      );
+    })
     .map((line) => line.trim())
     .slice(-LOG_TAIL_LIMIT);
 }
@@ -11961,6 +12421,9 @@ function attachRunnerWorkProgressMetadata(
     options.logMetadata === 'omit' || metadata?.runLogTail || logs.length === 0
       ? null
       : boundedRunLogTailMetadata(logs);
+  const runtimeResources = metadata?.runtimeResources
+    ? null
+    : boundedRuntimeResourcesMetadata(runnerResult);
   const outputWarnings = Array.isArray(
     (runnerResult as { outputWarnings?: unknown }).outputWarnings,
   )
@@ -12002,6 +12465,7 @@ function attachRunnerWorkProgressMetadata(
   if (
     (total === null || skipped === null || metadata?.workProgress) &&
     !runLogTail &&
+    !runtimeResources &&
     mergedOutputWarnings.length === 0
   ) {
     return output;
@@ -12034,6 +12498,7 @@ function attachRunnerWorkProgressMetadata(
       ...(metadata ?? {}),
       ...(workProgress ? { workProgress } : {}),
       ...(runLogTail ? { runLogTail } : {}),
+      ...(runtimeResources ? { runtimeResources } : {}),
       ...(mergedOutputWarnings.length > 0
         ? { outputWarnings: mergedOutputWarnings }
         : {}),
@@ -12185,6 +12650,10 @@ export async function executePostgresSchedulerClaim(
         leaseToken: claim.leaseToken,
         error: message,
         result: terminalFailureResult,
+        terminalLogLines: terminalLogLinesForOutbox(
+          terminalFailureResult,
+          options.terminalLogOutbox,
+        ),
         sandboxCrashDiagnostic: options.sandboxCrashDiagnostic,
       },
       options,
@@ -12495,10 +12964,21 @@ function postgresSchedulerReceiptLeaseId(input: {
 
 function mapReceiptRow(row: ReceiptRow): RuntimeStepReceipt {
   const leaseExpiresAt = row.lease_expires_at;
+  const outputRef = isRuntimeReceiptValueReference(row.output_ref)
+    ? row.output_ref
+    : null;
   return {
     key: row.k,
     status: row.status as RuntimeStepReceipt['status'],
     ...(row.output_json === null ? {} : { output: row.output_json }),
+    ...(outputRef
+      ? {
+          outputRef,
+          ...(outputRef.inlineFields
+            ? { inlineFields: outputRef.inlineFields }
+            : {}),
+        }
+      : {}),
     ...(row.error ? { error: row.error } : {}),
     ...(row.error_payload &&
     typeof row.error_payload === 'object' &&
@@ -12529,7 +13009,7 @@ export async function getPostgresSchedulerReceipt(
 ): Promise<RuntimeStepReceipt | null> {
   const result = await client.query<ReceiptRow>(
     `
-      SELECT k, status, output_json, error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at
+      SELECT k, status, output_json, output_ref, error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at
       FROM ${tableName('receipts', options)}
       WHERE org_id = $1
         AND k = $2
@@ -12555,6 +13035,7 @@ export async function getPostgresSchedulerReceipts(
       SELECT receipt.k,
              receipt.status,
              receipt.output_json,
+             receipt.output_ref,
              receipt.error,
              receipt.error_payload,
              receipt.failure_kind,
@@ -12778,7 +13259,7 @@ export async function claimPostgresSchedulerReceipt(
         -- Completed/skipped receipts are immutable cache hits unless a caller
         -- explicitly forces refresh. Read them before the UPSERT so popular
         -- prebuilt keys do not take a conflicting-row lock or create WAL.
-        SELECT k, status, output_json, error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at
+        SELECT k, status, output_json, output_ref, error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at
           FROM ${tableName('receipts', options)}
          WHERE org_id = $9
            AND k = $1
@@ -12822,6 +13303,7 @@ export async function claimPostgresSchedulerReceipt(
               ELSE now() + ($6::double precision * interval '1 millisecond')
             END,
             output_json = NULL,
+            output_ref = NULL,
             error = NULL,
             error_payload = NULL,
             failure_kind = 0,
@@ -12855,13 +13337,13 @@ export async function claimPostgresSchedulerReceipt(
               AND COALESCE(${tableName('receipts', options)}.lease_owner_attempt, 0) < $5::integer
             )
           )
-        RETURNING k, status, output_json, error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at
+        RETURNING k, status, output_json, output_ref, error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at
       )
-      SELECT k, status, output_json, error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at,
+      SELECT k, status, output_json, output_ref, error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at,
              'existing'::text AS claim_state
         FROM terminal
       UNION ALL
-      SELECT k, status, output_json, error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at,
+      SELECT k, status, output_json, output_ref, error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at,
              'claimed'::text AS claim_state
         FROM claimed
     `,
@@ -12939,6 +13421,7 @@ export async function claimPostgresSchedulerReceipts(
         SELECT receipt.k,
                receipt.status,
                receipt.output_json,
+               receipt.output_ref,
                receipt.error,
                receipt.error_payload,
                receipt.failure_kind,
@@ -13004,6 +13487,7 @@ export async function claimPostgresSchedulerReceipts(
               ELSE now() + ($6::double precision * interval '1 millisecond')
             END,
             output_json = NULL,
+            output_ref = NULL,
             error = NULL,
             error_payload = NULL,
             failure_kind = 0,
@@ -13022,12 +13506,13 @@ export async function claimPostgresSchedulerReceipts(
               AND ${tableName('receipts', options)}.lease_expires_at IS NULL
             OR ${tableName('receipts', options)}.lease_expires_at <= now()
           )
-        RETURNING k, status, output_json, error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at
+        RETURNING k, status, output_json, output_ref, error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at
       ),
       latest AS (
         SELECT receipt.k,
                receipt.status,
                receipt.output_json,
+               receipt.output_ref,
                receipt.error,
                receipt.error_payload,
                receipt.failure_kind,
@@ -13064,6 +13549,7 @@ export async function claimPostgresSchedulerReceipts(
         SELECT terminal.k,
                terminal.status,
                terminal.output_json,
+               terminal.output_ref,
                terminal.error,
                terminal.error_payload,
                terminal.failure_kind,
@@ -13080,6 +13566,7 @@ export async function claimPostgresSchedulerReceipts(
         SELECT claimed.k,
                claimed.status,
                claimed.output_json,
+               claimed.output_ref,
                claimed.error,
                claimed.error_payload,
                claimed.failure_kind,
@@ -13097,6 +13584,7 @@ export async function claimPostgresSchedulerReceipts(
         SELECT latest.k,
                latest.status,
                latest.output_json,
+               latest.output_ref,
                latest.error,
                latest.error_payload,
                latest.failure_kind,
@@ -13116,6 +13604,7 @@ export async function claimPostgresSchedulerReceipts(
       SELECT unioned.k,
              unioned.status,
              unioned.output_json,
+             unioned.output_ref,
              unioned.error,
              unioned.error_payload,
              unioned.failure_kind,
@@ -13156,22 +13645,88 @@ export async function claimPostgresSchedulerReceipts(
   );
 }
 
+/**
+ * A storage completion whose value is raw or already canonicalized by the
+ * gateway. Transport callers retain the public `output` requirement; the
+ * gateway intentionally supplies only `outputJson` or `outputRef` after it has
+ * released that duplicate structured graph.
+ */
+type RuntimeStepReceiptCompletionValue = {
+  output?: unknown;
+  outputJson?: string | null;
+  outputRef?: RuntimeReceiptValueReference | null;
+};
+
+type RuntimeStepReceiptCompletionInput = Omit<
+  CompleteRuntimeStepReceiptInput,
+  'output'
+> &
+  RuntimeStepReceiptCompletionValue;
+
+type RuntimeStepReceiptCompletionBatchInput = Omit<
+  CompleteRuntimeStepReceiptsInput['receipts'][number],
+  'output'
+> &
+  RuntimeStepReceiptCompletionValue;
+
+/**
+ * Canonical `jsonb` text for one receipt completion.
+ *
+ * The receipt gateway already redacted and serialized the output once, and
+ * hands that exact text through as `outputJson`, so the same value is not
+ * walked a second time here. Callers that pass only a value get the identical
+ * bytes from `stringifyPostgresJson`.
+ */
+function receiptOutputJson(
+  input: RuntimeStepReceiptCompletionValue,
+): string | null {
+  if (input.outputRef) {
+    if (input.outputJson !== null && input.outputJson !== undefined) {
+      throw new Error(
+        'Receipt completion cannot contain inline output and outputRef.',
+      );
+    }
+    return null;
+  }
+  return (
+    input.outputJson ??
+    stringifyRequiredReceiptJson(input.output ?? null, 'Receipt output')
+  );
+}
+
+function receiptOutputRefJson(
+  input: RuntimeStepReceiptCompletionValue,
+): string | null {
+  return input.outputRef
+    ? stringifyRequiredReceiptJson(input.outputRef, 'Receipt output reference')
+    : null;
+}
+
+function stringifyRequiredReceiptJson(value: unknown, field: string): string {
+  const serialized = stringifyPostgresJson(value);
+  if (serialized === undefined) {
+    throw new TypeError(`${field} must be JSON-serializable.`);
+  }
+  return serialized;
+}
+
 export async function completePostgresSchedulerReceipt(
   client: PostgresSchedulerQueryClient,
-  input: CompleteRuntimeStepReceiptInput & { returnOutput?: boolean },
+  input: RuntimeStepReceiptCompletionInput & { returnOutput?: boolean },
   options: PostgresSchedulerReceiptOptions,
 ): Promise<RuntimeStepReceipt | null> {
   if (input.leaseId === COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID) {
     const inserted = await client.query<ReceiptRow>(
       `
         INSERT INTO ${tableName('receipts', options)} AS receipt (
-          org_id, k, status, output_json, error, error_payload, failure_kind, run_id,
+          org_id, k, status, output_json, output_ref, error, error_payload, failure_kind, run_id,
           lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at
-        ) VALUES ($1, $2, 'completed', $3::jsonb, NULL, NULL, 0, $4,
+        ) VALUES ($1, $2, 'completed', $3::jsonb, $4::jsonb, NULL, NULL, 0, $5,
                   NULL, NULL, NULL, NULL)
         ON CONFLICT (org_id, k) DO UPDATE
         SET status = EXCLUDED.status,
             output_json = EXCLUDED.output_json,
+            output_ref = EXCLUDED.output_ref,
             error = NULL,
             error_payload = NULL,
             failure_kind = 0,
@@ -13183,25 +13738,33 @@ export async function completePostgresSchedulerReceipt(
             updated_at = now()
         WHERE receipt.status NOT IN ('completed', 'skipped')
         RETURNING k, status,
-                  CASE WHEN $5::boolean THEN output_json ELSE NULL::jsonb END AS output_json,
+                  CASE WHEN $6::boolean THEN output_json ELSE NULL::jsonb END AS output_json,
+                  output_ref,
                   error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id,
                   lease_owner_attempt, lease_expires_at
       `,
       [
         receiptOrgId(options),
         input.key,
-        stringifyPostgresJson(input.output ?? null),
+        receiptOutputJson(input),
+        receiptOutputRefJson(input),
         input.runId,
         input.returnOutput !== false,
       ],
     );
-    if (inserted.rows[0]) return mapReceiptRow(inserted.rows[0]);
+    if (inserted.rows[0]) {
+      return {
+        ...mapReceiptRow(inserted.rows[0]),
+        completionDisposition: 'persisted',
+      };
+    }
     // A separate statement gets a fresh READ COMMITTED snapshot after a
     // concurrent insert wins and commits.
     const winner = await client.query<ReceiptRow>(
       `
         SELECT k, status,
                CASE WHEN $3::boolean THEN output_json ELSE NULL::jsonb END AS output_json,
+               output_ref,
                error, failure_kind, run_id, lease_id, lease_owner_run_id,
                lease_owner_attempt, lease_expires_at
           FROM ${tableName('receipts', options)}
@@ -13212,7 +13775,12 @@ export async function completePostgresSchedulerReceipt(
       `,
       [receiptOrgId(options), input.key, input.returnOutput !== false],
     );
-    return winner.rows[0] ? mapReceiptRow(winner.rows[0]) : null;
+    return winner.rows[0]
+      ? {
+          ...mapReceiptRow(winner.rows[0]),
+          completionDisposition: 'existing',
+        }
+      : null;
   }
   const runAttempt = normalizePostgresSchedulerReceiptRunAttempt(
     input.runAttempt,
@@ -13223,6 +13791,7 @@ export async function completePostgresSchedulerReceipt(
       UPDATE ${tableName('receipts', options)}
       SET status = 'completed',
           output_json = $3::jsonb,
+          output_ref = $7::jsonb,
           error = NULL,
           error_payload = NULL,
           failure_kind = 0,
@@ -13243,51 +13812,226 @@ export async function completePostgresSchedulerReceipt(
           )
         )
       RETURNING k, status,
-                CASE WHEN $7::boolean THEN output_json ELSE NULL::jsonb END AS output_json,
+                CASE WHEN $8::boolean THEN output_json ELSE NULL::jsonb END AS output_json,
+                output_ref,
                 error, failure_kind, run_id, lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at
     `,
     [
       input.key,
       input.runId,
-      stringifyPostgresJson(input.output ?? null),
+      receiptOutputJson(input),
       runAttempt,
       leaseId,
       receiptOrgId(options),
+      receiptOutputRefJson(input),
       input.returnOutput !== false,
     ],
   );
-  return result.rows[0] ? mapReceiptRow(result.rows[0]) : null;
+  return result.rows[0]
+    ? {
+        ...mapReceiptRow(result.rows[0]),
+        completionDisposition: 'persisted',
+      }
+    : null;
+}
+
+/**
+ * Publish a batch of immutable completed receipts (receipt-cache inserts) in
+ * one INSERT ... ON CONFLICT statement, with a second statement only for the
+ * keys a concurrent publisher already won. Semantics match the scalar
+ * `completePostgresSchedulerReceipt` cache-insert path receipt by receipt: a
+ * key this batch persists reports `persisted`; a key already completed or
+ * skipped by someone else (including an earlier duplicate of the same key in
+ * this batch) reports the winner's row as `existing`; a key that is neither
+ * reports null.
+ */
+async function completePostgresSchedulerCacheInsertReceipts(
+  client: PostgresSchedulerQueryClient,
+  input: {
+    receipts: ReadonlyArray<RuntimeStepReceiptCompletionBatchInput>;
+    returnOutput?: boolean;
+  },
+  options: PostgresSchedulerReceiptOptions,
+): Promise<Array<RuntimeStepReceipt | null>> {
+  if (input.receipts.length === 0) return [];
+  const returnOutput = input.returnOutput !== false;
+  const orgId = receiptOrgId(options);
+  // ON CONFLICT DO UPDATE rejects a statement that touches the same row
+  // twice, so a key repeated inside one batch is inserted once; its later
+  // duplicates resolve through the winner read exactly as the scalar path
+  // resolved a second publisher.
+  const firstOccurrence = new Map<string, number>();
+  input.receipts.forEach((receipt, index) => {
+    if (!firstOccurrence.has(receipt.key))
+      firstOccurrence.set(receipt.key, index);
+  });
+  const insertIndexes = [...firstOccurrence.values()];
+  const inserted = await client.query<ReceiptRow>(
+    `
+      INSERT INTO ${tableName('receipts', options)} AS receipt (
+        org_id, k, status, output_json, output_ref, error, error_payload, failure_kind, run_id,
+        lease_id, lease_owner_run_id, lease_owner_attempt, lease_expires_at
+      )
+      SELECT $1, key_values.k, 'completed', output_values.output_json::jsonb,
+             ref_values.output_ref::jsonb, NULL, NULL, 0,
+             run_values.run_id, NULL, NULL, NULL, NULL
+        FROM unnest($2::text[]) WITH ORDINALITY AS key_values(k, ord)
+        JOIN unnest($3::text[]) WITH ORDINALITY AS output_values(output_json, ord)
+          ON output_values.ord = key_values.ord
+        JOIN unnest($4::text[]) WITH ORDINALITY AS ref_values(output_ref, ord)
+          ON ref_values.ord = key_values.ord
+        JOIN unnest($5::text[]) WITH ORDINALITY AS run_values(run_id, ord)
+          ON run_values.ord = key_values.ord
+      ON CONFLICT (org_id, k) DO UPDATE
+      SET status = EXCLUDED.status,
+          output_json = EXCLUDED.output_json,
+          output_ref = EXCLUDED.output_ref,
+          error = NULL,
+          error_payload = NULL,
+          failure_kind = 0,
+          run_id = EXCLUDED.run_id,
+          lease_id = NULL,
+          lease_owner_run_id = NULL,
+          lease_owner_attempt = NULL,
+          lease_expires_at = NULL,
+          updated_at = now()
+      WHERE receipt.status NOT IN ('completed', 'skipped')
+      RETURNING k, status,
+                CASE WHEN $6::boolean THEN output_json ELSE NULL::jsonb END AS output_json,
+                output_ref,
+                error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id,
+                lease_owner_attempt, lease_expires_at
+    `,
+    [
+      orgId,
+      insertIndexes.map((index) => input.receipts[index]!.key),
+      insertIndexes.map((index) => receiptOutputJson(input.receipts[index]!)),
+      insertIndexes.map((index) =>
+        receiptOutputRefJson(input.receipts[index]!),
+      ),
+      insertIndexes.map((index) => input.receipts[index]!.runId),
+      returnOutput,
+    ],
+  );
+  const persistedByKey = new Map(inserted.rows.map((row) => [row.k, row]));
+  const results: Array<RuntimeStepReceipt | null> = new Array(
+    input.receipts.length,
+  ).fill(null);
+  const unresolvedKeys = new Set<string>();
+  input.receipts.forEach((receipt, index) => {
+    const row = persistedByKey.get(receipt.key);
+    if (row && firstOccurrence.get(receipt.key) === index) {
+      results[index] = {
+        ...mapReceiptRow(row),
+        completionDisposition: 'persisted',
+      };
+    } else {
+      unresolvedKeys.add(receipt.key);
+    }
+  });
+  if (unresolvedKeys.size === 0) return results;
+  // A separate statement gets a fresh READ COMMITTED snapshot after a
+  // concurrent publisher wins and commits.
+  const winners = await client.query<ReceiptRow>(
+    `
+      SELECT k, status,
+             CASE WHEN $3::boolean THEN output_json ELSE NULL::jsonb END AS output_json,
+             output_ref,
+             error, error_payload, failure_kind, run_id, lease_id, lease_owner_run_id,
+             lease_owner_attempt, lease_expires_at
+        FROM ${tableName('receipts', options)}
+       WHERE org_id = $1
+         AND k = ANY($2::text[])
+         AND status IN ('completed', 'skipped')
+    `,
+    [orgId, [...unresolvedKeys], returnOutput],
+  );
+  const winnerByKey = new Map(winners.rows.map((row) => [row.k, row]));
+  let alreadyWon = 0;
+  let duplicatedInBatch = 0;
+  let unresolved = 0;
+  input.receipts.forEach((receipt, index) => {
+    if (results[index]) return;
+    const row = winnerByKey.get(receipt.key);
+    results[index] = row
+      ? { ...mapReceiptRow(row), completionDisposition: 'existing' }
+      : null;
+    if (!row) unresolved += 1;
+    else if (firstOccurrence.get(receipt.key) === index) alreadyWon += 1;
+    else duplicatedInBatch += 1;
+  });
+  // A losing publisher is the intended path, but every one costs the client
+  // a full-output read-back. Count them per batch so a read-back rate above
+  // the duplicate-call rate is visible in the gateway log without tracing.
+  console.info('[postgres-scheduler.receipt_publication_batch]', {
+    receipts: input.receipts.length,
+    persisted:
+      input.receipts.length - alreadyWon - duplicatedInBatch - unresolved,
+    alreadyWon,
+    duplicatedInBatch,
+    unresolved,
+  });
+  return results;
 }
 
 export async function completePostgresSchedulerReceipts(
   client: PostgresSchedulerQueryClient,
-  input: CompleteRuntimeStepReceiptsInput & { returnOutput?: boolean },
+  input: {
+    receipts: ReadonlyArray<RuntimeStepReceiptCompletionBatchInput>;
+    returnOutput?: boolean;
+  },
   options: PostgresSchedulerReceiptOptions,
 ): Promise<Array<RuntimeStepReceipt | null>> {
   if (input.receipts.length === 0) return [];
-  if (
-    input.receipts.some(
-      (receipt) => receipt.leaseId === COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
-    )
-  ) {
+  const cacheInsertIndexes: number[] = [];
+  const leasedIndexes: number[] = [];
+  input.receipts.forEach((receipt, index) => {
+    (receipt.leaseId === COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID
+      ? cacheInsertIndexes
+      : leasedIndexes
+    ).push(index);
+  });
+  if (cacheInsertIndexes.length > 0) {
     // Immutable cache publication is a different operation from leased Work
-    // Receipt completion. Route it through the scalar implementation so mixed
-    // batches cannot silently fall back to the legacy UPDATE-only semantics.
-    return await Promise.all(
-      input.receipts.map((receipt) =>
-        completePostgresSchedulerReceipt(
-          client,
-          { ...receipt, returnOutput: input.returnOutput },
-          options,
-        ),
-      ),
+    // Receipt completion, so a mixed batch runs as two set statements, never
+    // as one receipt per query. A runner batch is whatever settled during one
+    // gateway round trip, so under load it mixes both kinds every time; the
+    // per-receipt fallback this replaces cost 229 serialized queries on one
+    // connection for a single 1k-fixture completion request.
+    const results: Array<RuntimeStepReceipt | null> = new Array(
+      input.receipts.length,
+    ).fill(null);
+    const published = await completePostgresSchedulerCacheInsertReceipts(
+      client,
+      {
+        receipts: cacheInsertIndexes.map((index) => input.receipts[index]!),
+        returnOutput: input.returnOutput,
+      },
+      options,
     );
+    cacheInsertIndexes.forEach((index, position) => {
+      results[index] = published[position]!;
+    });
+    if (leasedIndexes.length > 0) {
+      const completed = await completePostgresSchedulerReceipts(
+        client,
+        {
+          receipts: leasedIndexes.map((index) => input.receipts[index]!),
+          returnOutput: input.returnOutput,
+        },
+        options,
+      );
+      leasedIndexes.forEach((index, position) => {
+        results[index] = completed[position]!;
+      });
+    }
+    return results;
   }
   const normalized = input.receipts.map((receipt) => ({
     ...receipt,
     runAttempt: normalizePostgresSchedulerReceiptRunAttempt(receipt.runAttempt),
     leaseId: receipt.leaseId?.trim() || null,
-    output: stringifyPostgresJson(receipt.output ?? null),
+    output: receiptOutputJson(receipt),
   }));
   const result = await client.query<ReceiptRow & { ord: number | string }>(
     `
@@ -13297,6 +14041,7 @@ export async function completePostgresSchedulerReceipts(
                attempt_values.run_attempt,
                lease_values.lease_id,
                output_values.output_json,
+               ref_values.output_ref,
                key_values.ord
           FROM unnest($1::text[]) WITH ORDINALITY AS key_values(k, ord)
           JOIN unnest($2::text[]) WITH ORDINALITY AS run_values(run_id, ord)
@@ -13307,11 +14052,14 @@ export async function completePostgresSchedulerReceipts(
             ON lease_values.ord = key_values.ord
           JOIN unnest($5::text[]) WITH ORDINALITY AS output_values(output_json, ord)
             ON output_values.ord = key_values.ord
+          JOIN unnest($6::text[]) WITH ORDINALITY AS ref_values(output_ref, ord)
+            ON ref_values.ord = key_values.ord
       ),
       completed AS (
         UPDATE ${tableName('receipts', options)} AS target
            SET status = 'completed',
                output_json = inputs.output_json::jsonb,
+               output_ref = inputs.output_ref::jsonb,
                error = NULL,
                error_payload = NULL,
                failure_kind = 0,
@@ -13320,7 +14068,7 @@ export async function completePostgresSchedulerReceipts(
                updated_at = now()
           FROM inputs
          WHERE target.k = inputs.k
-           AND target.org_id = $6
+           AND target.org_id = $7
            AND target.run_id = inputs.run_id
            AND target.status = 'running'
            AND COALESCE(target.lease_owner_run_id, target.run_id) = inputs.run_id
@@ -13334,7 +14082,8 @@ export async function completePostgresSchedulerReceipts(
            )
         RETURNING target.k,
                   target.status,
-                  CASE WHEN $7::boolean THEN target.output_json ELSE NULL::jsonb END AS output_json,
+                  CASE WHEN $8::boolean THEN target.output_json ELSE NULL::jsonb END AS output_json,
+                  target.output_ref,
                   target.error,
                   target.error_payload,
                   target.failure_kind,
@@ -13348,6 +14097,7 @@ export async function completePostgresSchedulerReceipts(
       SELECT completed.k,
              completed.status,
              completed.output_json,
+             completed.output_ref,
              completed.error,
              completed.error_payload,
              completed.failure_kind,
@@ -13367,11 +14117,19 @@ export async function completePostgresSchedulerReceipts(
       normalized.map((receipt) => receipt.runAttempt),
       normalized.map((receipt) => receipt.leaseId),
       normalized.map((receipt) => receipt.output),
+      normalized.map((receipt) => receiptOutputRefJson(receipt)),
       receiptOrgId(options),
       input.returnOutput !== false,
     ],
   );
-  return result.rows.map((row) => (row.k ? mapReceiptRow(row) : null));
+  return result.rows.map((row) =>
+    row.k
+      ? {
+          ...mapReceiptRow(row),
+          completionDisposition: 'persisted',
+        }
+      : null,
+  );
 }
 
 export async function failPostgresSchedulerReceipt(
@@ -13388,6 +14146,7 @@ export async function failPostgresSchedulerReceipt(
       UPDATE ${tableName('receipts', options)}
       SET status = 'failed',
           output_json = NULL,
+          output_ref = NULL,
           error = $3,
           error_payload = $8::jsonb,
           failure_kind = $6::smallint,
@@ -13467,6 +14226,7 @@ export async function failPostgresSchedulerReceipts(
         UPDATE ${tableName('receipts', options)} AS target
            SET status = 'failed',
                output_json = NULL,
+               output_ref = NULL,
                error = inputs.error,
                error_payload = inputs.error_payload,
                failure_kind = inputs.failure_kind,
@@ -13546,6 +14306,7 @@ export async function releasePostgresSchedulerReceipt(
       -- return to that claimable state for a later run to retry it.
       SET status = 'pending',
           output_json = NULL,
+          output_ref = NULL,
           error = NULL,
           error_payload = NULL,
           failure_kind = 0,
@@ -13572,6 +14333,22 @@ export async function releasePostgresSchedulerReceipt(
   return result.rows[0] ? mapReceiptRow(result.rows[0]) : null;
 }
 
+/**
+ * The write that returns a claimed receipt to the free pool. Every release
+ * path shares it so a successor's non-forced claim — whose status set is
+ * `('pending','failed')` and whose lease guard requires no live lease — always
+ * matches what a dead owner left behind.
+ */
+const RELEASE_RECEIPT_TO_PENDING_SET_CLAUSE = `status = 'pending',
+          output_json = NULL,
+          output_ref = NULL,
+          error = NULL,
+          error_payload = NULL,
+          failure_kind = 0,
+          lease_id = NULL,
+          lease_expires_at = NULL,
+          updated_at = now()`;
+
 export async function releasePostgresSchedulerReceiptsForAttempt(
   client: PostgresSchedulerQueryClient,
   input: { runId: string; runAttempt: number },
@@ -13583,14 +14360,7 @@ export async function releasePostgresSchedulerReceiptsForAttempt(
   const released = await client.query<{ k: string }>(
     `
       UPDATE ${tableName('receipts', options)}
-      SET status = 'pending',
-          output_json = NULL,
-          error = NULL,
-          error_payload = NULL,
-          failure_kind = 0,
-          lease_id = NULL,
-          lease_expires_at = NULL,
-          updated_at = now()
+      SET ${RELEASE_RECEIPT_TO_PENDING_SET_CLAUSE}
       WHERE org_id = $1
         AND run_id = $2
         AND status IN ('queued', 'running')
@@ -13843,6 +14613,7 @@ export async function skipPostgresSchedulerReceipt(
           lease_owner_attempt = $4,
           lease_expires_at = NULL,
           output_json = $3::jsonb,
+          output_ref = NULL,
           error = NULL,
           error_payload = NULL,
           failure_kind = 0,
@@ -14945,8 +15716,14 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
     ];
   }
   if (event.eventType === 'run.completed') {
+    const progressEvents = ledgerEventsPayloadField(
+      event.payload,
+      'terminalProgressEvents',
+      event.runId,
+    );
     const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
     return [
+      ...progressEvents,
       ...(lines.length === 0
         ? []
         : [
@@ -14970,10 +15747,28 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
     ];
   }
   if (event.eventType === 'run.failed') {
+    const progressEvents = ledgerEventsPayloadField(
+      event.payload,
+      'terminalProgressEvents',
+      event.runId,
+    );
+    const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
     const result = terminalRunResultForLedger(
       payloadField(event.payload, 'result'),
     );
     return [
+      ...progressEvents,
+      ...(lines.length === 0
+        ? []
+        : [
+            {
+              type: 'log.appended',
+              runId: event.runId,
+              occurredAt,
+              source: 'worker',
+              lines,
+            } as PlayRunLedgerEvent,
+          ]),
       {
         type: 'run.failed',
         runId: event.runId,
@@ -15930,6 +16725,26 @@ function stringArrayPayloadField(payload: unknown, key: string): string[] {
     : [];
 }
 
+function ledgerEventsPayloadField(
+  payload: unknown,
+  key: string,
+  runId: string,
+): PlayRunLedgerIngressEvent[] {
+  const value = payloadField(payload, key);
+  if (!Array.isArray(value)) return [];
+  return value.filter((candidate): candidate is PlayRunLedgerIngressEvent => {
+    const record = recordPayload(candidate);
+    return (
+      record !== null &&
+      record.runId === runId &&
+      typeof record.occurredAt === 'number' &&
+      Number.isFinite(record.occurredAt) &&
+      typeof record.source === 'string' &&
+      (typeof record.type === 'string' || typeof record.key === 'string')
+    );
+  });
+}
+
 function numberPayloadField(payload: unknown, key: string): number | null {
   const value = payloadField(payload, key);
   return typeof value === 'number' && Number.isFinite(value) ? value : null;
@@ -16162,15 +16977,27 @@ export function postgresSchedulerRunHandle(
     },
     cancel: async () => {
       const poolOptions = resolve();
-      await withRuntimeSchedulerClient(poolOptions, (client) =>
-        cancelPostgresSchedulerRun(client, { runId }, options),
+      await withRuntimeSchedulerTransaction(
+        poolOptions,
+        {
+          lockTimeoutMs: RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
+          statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
+        },
+        (client) => cancelPostgresSchedulerRun(client, { runId }, options),
+        { runId, phase: 'cancel_run', priority: 'critical' },
       );
     },
     signal: async (payload) => {
       if (payload.kind === 'cancel') {
         const poolOptions = resolve();
-        await withRuntimeSchedulerClient(poolOptions, (client) =>
-          cancelPostgresSchedulerRun(client, { runId }, options),
+        await withRuntimeSchedulerTransaction(
+          poolOptions,
+          {
+            lockTimeoutMs: RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
+            statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
+          },
+          (client) => cancelPostgresSchedulerRun(client, { runId }, options),
+          { runId, phase: 'cancel_run', priority: 'critical' },
         );
         return;
       }