deepline 0.3.113 → 0.3.115

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -34,11 +34,24 @@ const RECOVERY_FAILURE_RETRY_MAX_MS = 60_000;
 export const NOTIFY_UNAVAILABLE_SAFETY_WAKE_MS = 1_000;
 const MAX_TIMEOUT_MS = 2_147_483_647;
 /**
- * Two short claim/deliver waves match each projector's bounded scheduler pool.
- * The claim query uses SKIP LOCKED, so the wave can advance separate runs
- * without splitting the ordered lifecycle of any one run.
+ * One projector can keep this many independent Run Ledger deliveries in
+ * flight after it has found durable work. Each delivery owns only short
+ * Postgres checkouts: claim, token-context read, and acknowledgement. It does
+ * not hold a scheduler connection across either gateway HTTP request. This is
+ * therefore a network/projection concurrency budget, deliberately independent
+ * of the projector's much smaller Postgres pool.
+ *
+ * A single successful probe starts a bounded wave; an idle projector still
+ * performs only one indexed head read per safety wake. `SKIP LOCKED` leases
+ * distinct runs, retaining strict causal ordering within every run.
  */
-export const RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT = 2;
+// `run.created` has one event per admitted Run, so a burst must project
+// independently rather than wait for execution. Each Fly delivery first asks
+// the launch-pinned Vercel deployment for a lease-fenced executor token,
+// though. Keep that remote fan-out inside the Vercel control-plane budget:
+// the Fly checkout is short, but 128 simultaneous token requests previously
+// saturated the shared serverless scheduler pool before projection began.
+export const RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT = 32;
 
 type RecoveryTimer = ReturnType<typeof setTimeout>;
 
@@ -226,12 +239,33 @@ export function startRunLedgerOutboxRecovery(
     timer.unref?.();
   };
 
-  const drain = async (_reason: string): Promise<void> => {
+  const drain = async (reason: string): Promise<void> => {
     while (requested && !closed) {
       requested = false;
-      const settled = await Promise.allSettled(
-        Array.from({ length: parallelism }, () => deps.dispatchOnce()),
-      );
+      // Probe first so an idle projector issues one small indexed read per
+      // wake rather than `parallelism` identical empty scans. Once the probe
+      // claims a row, the remaining slots can safely lease unrelated runs;
+      // each dispatcher releases its database client before gateway I/O.
+      let first: PostgresSchedulerOutboxProjectionResult;
+      try {
+        first = await deps.dispatchOnce();
+      } catch (error) {
+        scheduleFailureRetry('dispatch', error);
+        return;
+      }
+      const settled = [
+        {
+          status: 'fulfilled' as const,
+          value: first,
+        },
+        ...(first.attempted === 0
+          ? []
+          : await Promise.allSettled(
+              Array.from({ length: parallelism - 1 }, () =>
+                deps.dispatchOnce(),
+              ),
+            )),
+      ];
       const rejected = settled.find(
         (result): result is PromiseRejectedResult =>
           result.status === 'rejected',
@@ -251,6 +285,18 @@ export function startRunLedgerOutboxRecovery(
         (total, result) => total + result.retrying,
         0,
       );
+      const attempted = results.reduce(
+        (total, result) => total + result.attempted,
+        0,
+      );
+      const projected = results.reduce(
+        (total, result) => total + result.projected,
+        0,
+      );
+      const blocked = results.reduce(
+        (total, result) => total + result.failed,
+        0,
+      );
 
       if (retrying > 0) {
         // The failed row owns its own durable retry clock. Do NOT stop this
@@ -263,11 +309,25 @@ export function startRunLedgerOutboxRecovery(
         failureCount = 0;
       }
 
+      if (attempted > 0) {
+        // This is the durable queue's high-signal throughput event. It makes
+        // a delayed public Run observable as a projector/outbox problem,
+        // without emitting one noisy log line per normal database checkout.
+        logger.info('[run-ledger-outbox.recovery_wave]', {
+          reason,
+          parallelism,
+          attempted,
+          projected,
+          retrying,
+          blocked,
+        });
+      }
+
       // Continue until no event was claimable. A transient failure schedules
       // its own durable retry, so the next pass naturally skips that run while
       // moving unrelated runs forward. A permanent failure is settled once and
       // likewise cannot block another run's causal prefix.
-      if (results.some((result) => result.attempted > 0)) {
+      if (attempted > 0) {
         requested = true;
         continue;
       }
@@ -324,10 +384,11 @@ export function startRunLedgerOutboxRecoveryController(input: {
         { tokenAuthority: 'gateway', limit: 8 },
         schedulerOptions,
       ),
-    parallelism: Math.min(
-      RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT,
-      Math.max(1, input.poolOptions.maxClients ?? 1),
-    ),
+    // `dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope` releases each
+    // short scheduler checkout before it calls the Vercel gateway. Keep the
+    // durable projection wave large enough for a webhook burst without
+    // expanding the process' Postgres connection budget.
+    parallelism: RUN_LEDGER_OUTBOX_RECOVERY_MAX_IN_FLIGHT,
     readNextDueAt: () =>
       withRuntimeSchedulerClient(input.poolOptions, (client) =>
         readPostgresSchedulerOutboxNextDueAt(client, schedulerOptions),

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-queue-health.ts

@@ -9,6 +9,11 @@ import {
   type QueueHealth,
   type QueueObservation,
 } from '../../observability/queue-health';
+import {
+  dlqInventoryFromQueueObservation,
+  getDlqTransitionPaths,
+  type DlqInventory,
+} from '../../observability/dlq';
 
 /**
  * Bounded, read-only health projection for the durable runtime queues.
@@ -59,13 +64,15 @@ export type RuntimeQueueFlow = {
 
 export type RuntimeQueueHealthProjection = {
   id: string;
-  kind: 'absurd' | 'run_ledger_outbox' | 'sandbox_cleanup';
+  kind: 'absurd' | 'run_ledger_outbox' | 'compute_billing' | 'sandbox_cleanup';
   source: 'absurd' | 'runtime_postgres';
   observedAt: string;
   status: RuntimeQueueHealthStatus;
   statusReason: string;
   /** Shared scheduler-neutral contract consumed by transports and dashboards. */
   observation: QueueObservation;
+  /** Shared DLQ contract consumed by operators and the admin CLI. */
+  dlq: DlqInventory;
   health: QueueHealth;
   depth: RuntimeQueueDepth;
   oldestAgeMs: RuntimeQueueOldestAge;
@@ -86,6 +93,8 @@ export type RuntimeQueueHealthProjection = {
 export type RuntimeQueueHealthSnapshot = {
   observedAt: string;
   staleAfterMs: number;
+  /** True when the control-plane lane inventory was capped before sampling. */
+  inventoryTruncated?: boolean;
   queues: RuntimeQueueHealthProjection[];
   summary: {
     green: number;
@@ -96,6 +105,8 @@ export type RuntimeQueueHealthSnapshot = {
 };
 
 export type RuntimeQueueHealthOptions = {
+  /** Explicit deployment label carried into the shared DLQ inventory. */
+  environment?: string;
   schema?: string;
   now?: Date;
   staleAfterMs?: number;
@@ -103,6 +114,8 @@ export type RuntimeQueueHealthOptions = {
   failAgeMs?: number;
   /** Restrict Absurd rows to these queue names. Registry values are validated. */
   absurdQueueNames?: string[];
+  /** The caller could not enumerate every live lane within the bounded cap. */
+  absurdQueueInventoryTruncated?: boolean;
   maxAbsurdQueues?: number;
 };
 
@@ -134,6 +147,8 @@ type CountRow = {
   in_flight_truncated?: boolean;
   blocked_truncated?: boolean;
   dead_lettered_truncated?: boolean;
+  /** False for durable worksets that expose blocked quarantine, not a DLQ. */
+  dead_lettered_available?: boolean;
   flow_available?: boolean;
   flow_unavailable_reason?: string | null;
 };
@@ -164,7 +179,9 @@ function numberValue(value: string | number | null | undefined): number {
 function ageValue(value: string | number | null | undefined): number | null {
   if (value === null || value === undefined) return null;
   const parsed = numberValue(value);
-  return parsed > 0 ? Math.floor(parsed) : null;
+  // Zero is a real age for freshly admitted work.  Keep it distinct from
+  // NULL, which means that the bucket is empty and should render as N/A.
+  return Math.floor(parsed);
 }
 
 function optionsWithDefaults(
@@ -246,6 +263,7 @@ function projectionFromRow(input: {
   source: RuntimeQueueHealthProjection['source'];
   row: CountRow;
   observedAt: Date;
+  environment?: string;
   options: RuntimeQueueHealthThresholds;
 }): RuntimeQueueHealthProjection {
   const depth: RuntimeQueueDepth = {
@@ -262,6 +280,7 @@ function projectionFromRow(input: {
     blocked: ageValue(input.row.oldest_blocked_ms),
     deadLettered: ageValue(input.row.oldest_dead_lettered_ms),
   };
+  const deadLetteredAvailable = input.row.dead_lettered_available !== false;
   const lifecycle = {
     started: numberValue(input.row.started),
     completed: numberValue(input.row.completed),
@@ -299,7 +318,9 @@ function projectionFromRow(input: {
         ? 'Run admitted Play runtime tasks to terminal state'
         : input.kind === 'run_ledger_outbox'
           ? 'Deliver causal Run Ledger events to control-plane projections'
-          : 'Complete durable sandbox deletion obligations',
+          : input.kind === 'compute_billing'
+            ? 'Settle durable compute billing obligations after a runtime run ends'
+            : 'Complete durable sandbox deletion obligations',
     unit: 'items',
     kind: input.kind === 'absurd' ? 'fifo' : 'leased_workset',
     ageSlo: {
@@ -328,20 +349,21 @@ function projectionFromRow(input: {
       delayed: measureDepth(depth.delayed, input.row.delayed_truncated),
       inFlight: measureDepth(depth.inFlight, input.row.in_flight_truncated),
       blocked: measureDepth(depth.blocked, input.row.blocked_truncated),
-      deadLettered: measureDepth(
-        depth.deadLettered,
-        input.row.dead_lettered_truncated,
-      ),
+      deadLettered: deadLetteredAvailable
+        ? measureDepth(depth.deadLettered, input.row.dead_lettered_truncated)
+        : notApplicable<number>('This workset has blocked quarantine, not a DLQ.'),
     },
     oldestAgeMs: {
       ready: measureAge(oldestAgeMs.ready, input.row.ready_truncated),
       delayed: measureAge(oldestAgeMs.delayed, input.row.delayed_truncated),
       inFlight: measureAge(oldestAgeMs.inFlight, input.row.in_flight_truncated),
       blocked: measureAge(oldestAgeMs.blocked, input.row.blocked_truncated),
-      deadLettered: measureAge(
-        oldestAgeMs.deadLettered,
-        input.row.dead_lettered_truncated,
-      ),
+      deadLettered: deadLetteredAvailable
+        ? measureAge(
+            oldestAgeMs.deadLettered,
+            input.row.dead_lettered_truncated,
+          )
+        : notApplicable<number>('This workset has blocked quarantine, not a DLQ.'),
     },
     flow: {
       admitted: flowAvailable
@@ -369,6 +391,20 @@ function projectionFromRow(input: {
     observation,
     input.options.now.getTime(),
   );
+  const dlq = dlqInventoryFromQueueObservation({
+    queueId: input.id,
+    environment:
+      input.environment?.trim() ||
+      process.env.DEEPLINE_ENV?.trim() ||
+      'unknown',
+    source: input.source,
+    observation,
+    paths: getDlqTransitionPaths({
+      queueId: input.id,
+      owner: 'runtime',
+      source: input.source,
+    }),
+  });
   return {
     id: input.id,
     kind: input.kind,
@@ -377,6 +413,7 @@ function projectionFromRow(input: {
     status: derived.status,
     statusReason: derived.statusReason,
     observation,
+    dlq,
     health,
     depth,
     oldestAgeMs,
@@ -465,6 +502,7 @@ function absurdProjectionQuery(queueTable: string): string {
       0::bigint AS completed,
       0::bigint AS failed,
       0::bigint AS retried,
+      true AS dead_lettered_available,
       false AS flow_available,
       'Absurd run history has no bounded exact rolling flow projection'::text AS flow_unavailable_reason
     FROM sampled
@@ -474,114 +512,216 @@ function absurdProjectionQuery(queueTable: string): string {
 function outboxProjectionQuery(schema: string): string {
   const table = `${schema}."outbox"`;
   return `
-    WITH pending_sample AS (
-      SELECT
-        CASE
-          WHEN lease_owner IS NULL AND coalesce(next_attempt_at, created_at) <= $1::timestamptz THEN 'ready'
-          WHEN lease_owner IS NULL THEN 'delayed'
-          WHEN lease_expires_at > $1::timestamptz THEN 'in_flight'
-          ELSE 'blocked'
-        END AS bucket,
-        CASE
-          WHEN lease_owner IS NULL AND coalesce(next_attempt_at, created_at) > $1::timestamptz
-            THEN coalesce(next_attempt_at, created_at)
-          WHEN lease_owner IS NULL THEN created_at
-          WHEN lease_expires_at > $1::timestamptz
-            THEN coalesce(delivery_batch_started_at, created_at)
-          ELSE coalesce(lease_expires_at, created_at)
-        END AS age_at
+    WITH ready AS (
+      SELECT coalesce(next_attempt_at, created_at) AS age_at
       FROM ${table}
-      WHERE delivered_at IS NULL AND failed_at IS NULL
+      WHERE delivered_at IS NULL
+        AND failed_at IS NULL
+        AND lease_owner IS NULL
+        AND coalesce(next_attempt_at, created_at) <= $1::timestamptz
       ORDER BY coalesce(next_attempt_at, created_at), created_at, event_id
       LIMIT $2::int
-    ), dead_sample AS (
-      SELECT 'dead_lettered'::text AS bucket, coalesce(failed_at, created_at) AS age_at
+    ), delayed AS (
+      SELECT coalesce(next_attempt_at, created_at) AS age_at
+      FROM ${table}
+      WHERE delivered_at IS NULL
+        AND failed_at IS NULL
+        AND lease_owner IS NULL
+        AND coalesce(next_attempt_at, created_at) > $1::timestamptz
+      ORDER BY coalesce(next_attempt_at, created_at), created_at, event_id
+      LIMIT $2::int
+    ), in_flight AS (
+      SELECT coalesce(delivery_batch_started_at, created_at) AS age_at
+      FROM ${table}
+      WHERE delivered_at IS NULL
+        AND failed_at IS NULL
+        AND lease_owner IS NOT NULL
+        AND lease_expires_at > $1::timestamptz
+      ORDER BY lease_expires_at, created_at, event_id
+      LIMIT $2::int
+    ), blocked AS (
+      SELECT coalesce(lease_expires_at, created_at) AS age_at
+      FROM ${table}
+      WHERE delivered_at IS NULL
+        AND failed_at IS NULL
+        AND lease_owner IS NOT NULL
+        AND (lease_expires_at IS NULL OR lease_expires_at <= $1::timestamptz)
+      ORDER BY coalesce(lease_expires_at, created_at), created_at, event_id
+      LIMIT $2::int
+    ), dead_lettered AS (
+      SELECT coalesce(failed_at, created_at) AS age_at
       FROM ${table}
       WHERE delivered_at IS NULL AND failed_at IS NOT NULL
       ORDER BY failed_at, event_id
       LIMIT $2::int
-    ), sampled AS (
-      SELECT * FROM pending_sample
-      UNION ALL
-      SELECT * FROM dead_sample
     )
     SELECT
-      least(count(*) FILTER (WHERE bucket = 'ready'), $3) AS ready,
-      least(count(*) FILTER (WHERE bucket = 'delayed'), $3) AS delayed,
-      least(count(*) FILTER (WHERE bucket = 'in_flight'), $3) AS in_flight,
-      least(count(*) FILTER (WHERE bucket = 'blocked'), $3) AS blocked,
-      least(count(*) FILTER (WHERE bucket = 'dead_lettered'), $3) AS dead_lettered,
-      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'ready') AS oldest_ready_ms,
-      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'delayed') AS oldest_delayed_ms,
-      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'in_flight') AS oldest_in_flight_ms,
-      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'blocked') AS oldest_blocked_ms,
-      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'dead_lettered') AS oldest_dead_lettered_ms,
-      count(*) FILTER (WHERE bucket <> 'dead_lettered') > $3 AS ready_truncated,
-      count(*) FILTER (WHERE bucket <> 'dead_lettered') > $3 AS delayed_truncated,
-      count(*) FILTER (WHERE bucket <> 'dead_lettered') > $3 AS in_flight_truncated,
-      count(*) FILTER (WHERE bucket <> 'dead_lettered') > $3 AS blocked_truncated,
-      count(*) FILTER (WHERE bucket = 'dead_lettered') > $3 AS dead_lettered_truncated,
+      least((SELECT count(*) FROM ready), $3) AS ready,
+      least((SELECT count(*) FROM delayed), $3) AS delayed,
+      least((SELECT count(*) FROM in_flight), $3) AS in_flight,
+      least((SELECT count(*) FROM blocked), $3) AS blocked,
+      least((SELECT count(*) FROM dead_lettered), $3) AS dead_lettered,
+      (SELECT max(${intervalExpr('age_at')}) FROM ready) AS oldest_ready_ms,
+      (SELECT max(${intervalExpr('age_at')}) FROM delayed) AS oldest_delayed_ms,
+      (SELECT max(${intervalExpr('age_at')}) FROM in_flight) AS oldest_in_flight_ms,
+      (SELECT max(${intervalExpr('age_at')}) FROM blocked) AS oldest_blocked_ms,
+      (SELECT max(${intervalExpr('age_at')}) FROM dead_lettered) AS oldest_dead_lettered_ms,
+      (SELECT count(*) FROM ready) > $3 AS ready_truncated,
+      (SELECT count(*) FROM delayed) > $3 AS delayed_truncated,
+      (SELECT count(*) FROM in_flight) > $3 AS in_flight_truncated,
+      (SELECT count(*) FROM blocked) > $3 AS blocked_truncated,
+      (SELECT count(*) FROM dead_lettered) > $3 AS dead_lettered_truncated,
       0::bigint AS admitted,
       0::bigint AS started,
       0::bigint AS completed,
       0::bigint AS failed,
       0::bigint AS retried,
+      true AS dead_lettered_available,
       false AS flow_available,
       'Outbox health has no bounded exact rolling flow projection'::text AS flow_unavailable_reason
-    FROM sampled
   `;
 }
 
 function cleanupProjectionQuery(schema: string): string {
   const table = `${schema}."sandbox_cleanup_jobs_v2"`;
   return `
-    WITH sampled AS (
+    WITH ready AS (
+      SELECT created_at AS age_at
+      FROM ${table}
+      WHERE status <> 'deleted'
+        AND blocked_at IS NULL
+        AND lease_owner IS NULL
+        AND status <> 'deleting'
+        AND available_at <= $1::timestamptz
+      ORDER BY available_at, created_at, provider, sandbox_id
+      LIMIT $2::int
+    ), delayed AS (
+      SELECT available_at AS age_at
+      FROM ${table}
+      WHERE status <> 'deleted'
+        AND blocked_at IS NULL
+        AND lease_owner IS NULL
+        AND status <> 'deleting'
+        AND available_at > $1::timestamptz
+      ORDER BY available_at, created_at, provider, sandbox_id
+      LIMIT $2::int
+    ), in_flight AS (
+      SELECT coalesce(updated_at, created_at) AS age_at
+      FROM ${table}
+      WHERE status <> 'deleted'
+        AND blocked_at IS NULL
+        AND lease_owner IS NOT NULL
+        AND lease_expires_at > $1::timestamptz
+      ORDER BY lease_expires_at, created_at, provider, sandbox_id
+      LIMIT $2::int
+    ), blocked AS (
       SELECT
-        CASE
-          WHEN blocked_at IS NOT NULL THEN 'blocked'
-          WHEN lease_owner IS NOT NULL AND lease_expires_at > $1::timestamptz THEN 'in_flight'
-          WHEN lease_owner IS NOT NULL OR status = 'deleting' THEN 'blocked'
-          WHEN available_at > $1::timestamptz THEN 'delayed'
-          ELSE 'ready'
-        END AS bucket,
         CASE
           WHEN blocked_at IS NOT NULL THEN blocked_at
-          WHEN lease_owner IS NOT NULL AND lease_expires_at > $1::timestamptz
-            THEN coalesce(updated_at, created_at)
-          WHEN lease_owner IS NOT NULL OR status = 'deleting'
-            THEN coalesce(lease_expires_at, created_at)
-          WHEN available_at > $1::timestamptz THEN available_at
-          ELSE created_at
+          ELSE coalesce(lease_expires_at, created_at)
         END AS age_at
       FROM ${table}
       WHERE status <> 'deleted'
-      ORDER BY available_at, created_at, provider, sandbox_id
+        AND (
+          blocked_at IS NOT NULL
+          OR lease_owner IS NOT NULL AND (lease_expires_at IS NULL OR lease_expires_at <= $1::timestamptz)
+          OR lease_owner IS NULL AND status = 'deleting'
+        )
+      ORDER BY coalesce(blocked_at, lease_expires_at, created_at), created_at, provider, sandbox_id
       LIMIT $2::int
     )
     SELECT
-      least(count(*) FILTER (WHERE bucket = 'ready'), $3) AS ready,
-      least(count(*) FILTER (WHERE bucket = 'delayed'), $3) AS delayed,
-      least(count(*) FILTER (WHERE bucket = 'in_flight'), $3) AS in_flight,
-      least(count(*) FILTER (WHERE bucket = 'blocked'), $3) AS blocked,
+      least((SELECT count(*) FROM ready), $3) AS ready,
+      least((SELECT count(*) FROM delayed), $3) AS delayed,
+      least((SELECT count(*) FROM in_flight), $3) AS in_flight,
+      least((SELECT count(*) FROM blocked), $3) AS blocked,
       0::bigint AS dead_lettered,
-      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'ready') AS oldest_ready_ms,
-      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'delayed') AS oldest_delayed_ms,
-      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'in_flight') AS oldest_in_flight_ms,
-      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'blocked') AS oldest_blocked_ms,
+      (SELECT max(${intervalExpr('age_at')}) FROM ready) AS oldest_ready_ms,
+      (SELECT max(${intervalExpr('age_at')}) FROM delayed) AS oldest_delayed_ms,
+      (SELECT max(${intervalExpr('age_at')}) FROM in_flight) AS oldest_in_flight_ms,
+      (SELECT max(${intervalExpr('age_at')}) FROM blocked) AS oldest_blocked_ms,
       NULL::bigint AS oldest_dead_lettered_ms,
-      count(*) > $3 AS ready_truncated,
-      count(*) > $3 AS delayed_truncated,
-      count(*) > $3 AS in_flight_truncated,
-      count(*) > $3 AS blocked_truncated,
+      (SELECT count(*) FROM ready) > $3 AS ready_truncated,
+      (SELECT count(*) FROM delayed) > $3 AS delayed_truncated,
+      (SELECT count(*) FROM in_flight) > $3 AS in_flight_truncated,
+      (SELECT count(*) FROM blocked) > $3 AS blocked_truncated,
       false AS dead_lettered_truncated,
       0::bigint AS admitted,
       0::bigint AS started,
       0::bigint AS completed,
       0::bigint AS failed,
       0::bigint AS retried,
+      false AS dead_lettered_available,
       false AS flow_available,
       'Sandbox cleanup health has no bounded exact rolling flow projection'::text AS flow_unavailable_reason
-    FROM sampled
+  `;
+}
+
+function computeBillingProjectionQuery(schema: string): string {
+  const table = `${schema}."compute_billing_jobs_v2"`;
+  return `
+    WITH ready AS (
+      SELECT created_at AS age_at
+      FROM ${table}
+      WHERE status <> 'settled'
+        AND blocked_at IS NULL
+        AND lease_owner IS NULL
+        AND available_at <= $1::timestamptz
+      ORDER BY available_at, created_at, run_id
+      LIMIT $2::int
+    ), delayed AS (
+      SELECT available_at AS age_at
+      FROM ${table}
+      WHERE status <> 'settled'
+        AND blocked_at IS NULL
+        AND lease_owner IS NULL
+        AND available_at > $1::timestamptz
+      ORDER BY available_at, created_at, run_id
+      LIMIT $2::int
+    ), in_flight AS (
+      SELECT coalesce(updated_at, created_at) AS age_at
+      FROM ${table}
+      WHERE status <> 'settled'
+        AND blocked_at IS NULL
+        AND lease_owner IS NOT NULL
+        AND lease_expires_at > $1::timestamptz
+      ORDER BY lease_expires_at, created_at, run_id
+      LIMIT $2::int
+    ), blocked AS (
+      SELECT coalesce(blocked_at, lease_expires_at, created_at) AS age_at
+      FROM ${table}
+      WHERE status <> 'settled'
+        AND (
+          blocked_at IS NOT NULL
+          OR lease_owner IS NOT NULL AND (lease_expires_at IS NULL OR lease_expires_at <= $1::timestamptz)
+        )
+      ORDER BY coalesce(blocked_at, lease_expires_at, created_at), created_at, run_id
+      LIMIT $2::int
+    )
+    SELECT
+      least((SELECT count(*) FROM ready), $3) AS ready,
+      least((SELECT count(*) FROM delayed), $3) AS delayed,
+      least((SELECT count(*) FROM in_flight), $3) AS in_flight,
+      least((SELECT count(*) FROM blocked), $3) AS blocked,
+      0::bigint AS dead_lettered,
+      (SELECT max(${intervalExpr('age_at')}) FROM ready) AS oldest_ready_ms,
+      (SELECT max(${intervalExpr('age_at')}) FROM delayed) AS oldest_delayed_ms,
+      (SELECT max(${intervalExpr('age_at')}) FROM in_flight) AS oldest_in_flight_ms,
+      (SELECT max(${intervalExpr('age_at')}) FROM blocked) AS oldest_blocked_ms,
+      NULL::bigint AS oldest_dead_lettered_ms,
+      (SELECT count(*) FROM ready) > $3 AS ready_truncated,
+      (SELECT count(*) FROM delayed) > $3 AS delayed_truncated,
+      (SELECT count(*) FROM in_flight) > $3 AS in_flight_truncated,
+      (SELECT count(*) FROM blocked) > $3 AS blocked_truncated,
+      false AS dead_lettered_truncated,
+      0::bigint AS admitted,
+      0::bigint AS started,
+      0::bigint AS completed,
+      0::bigint AS failed,
+      0::bigint AS retried,
+      false AS dead_lettered_available,
+      false AS flow_available,
+      'Compute billing health has no bounded exact rolling flow projection'::text AS flow_unavailable_reason
   `;
 }
 
@@ -649,9 +789,13 @@ export async function readRuntimeQueueHealthFromClient(
           )
         : await client.query<{ queue_name: string }>(
             `SELECT queue_name FROM absurd.queues ORDER BY queue_name LIMIT $1::int`,
-            [maxQueues],
+            [maxQueues + 1],
           );
-  const queueNames = registry.rows.map((row) => row.queue_name);
+  const registryTruncated =
+    requestedNames === undefined && registry.rows.length > maxQueues;
+  const queueNames = registry.rows
+    .slice(0, maxQueues)
+    .map((row) => row.queue_name);
   if (requestedNames) {
     const found = new Set(queueNames);
     const missing = requestedNames.filter((name) => !found.has(name));
@@ -674,16 +818,22 @@ export async function readRuntimeQueueHealthFromClient(
         source: 'absurd',
         row: rowFromResult(result),
         observedAt: now,
+        environment: input.environment,
         options,
       }),
     );
   }
-  const [outbox, cleanup] = await Promise.all([
+  const [outbox, computeBilling, cleanup] = await Promise.all([
     client.query<CountRow>(outboxProjectionQuery(schema), [
       now,
       sampleLimit + 1,
       sampleLimit,
     ]),
+    client.query<CountRow>(computeBillingProjectionQuery(schema), [
+      now,
+      sampleLimit + 1,
+      sampleLimit,
+    ]),
     client.query<CountRow>(cleanupProjectionQuery(schema), [
       now,
       sampleLimit + 1,
@@ -697,6 +847,16 @@ export async function readRuntimeQueueHealthFromClient(
       source: 'runtime_postgres',
       row: rowFromResult(outbox),
       observedAt: now,
+      environment: input.environment,
+      options,
+    }),
+    projectionFromRow({
+      id: 'runtime:compute-billing-jobs-v2',
+      kind: 'compute_billing',
+      source: 'runtime_postgres',
+      row: rowFromResult(computeBilling),
+      observedAt: now,
+      environment: input.environment,
       options,
     }),
     projectionFromRow({
@@ -705,6 +865,7 @@ export async function readRuntimeQueueHealthFromClient(
       source: 'runtime_postgres',
       row: rowFromResult(cleanup),
       observedAt: now,
+      environment: input.environment,
       options,
     }),
   );
@@ -717,6 +878,9 @@ export async function readRuntimeQueueHealthFromClient(
   return {
     observedAt: now.toISOString(),
     staleAfterMs: options.staleAfterMs,
+    ...(input.absurdQueueInventoryTruncated || registryTruncated
+      ? { inventoryTruncated: true }
+      : {}),
     queues: projections,
     summary,
   };