deepline 0.3.105 → 0.3.107

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-queue-health.ts

@@ -0,0 +1,718 @@
+import type { PostgresSchedulerQueryClient } from './postgres';
+import {
+  deriveQueueHealth,
+  exact,
+  lowerBound,
+  notApplicable,
+  unavailable,
+  type QueueDefinition,
+  type QueueHealth,
+  type QueueObservation,
+} from '../../observability/queue-health';
+
+/**
+ * Bounded, read-only health projection for the durable runtime queues.
+ *
+ * This intentionally lives beside the scheduler adapters rather than in the
+ * metrics package: Absurd and Runtime Postgres are the sources of truth for
+ * these rows.  A caller receives one coherent snapshot (the queries run on a
+ * single connection) and can publish it to any transport or dashboard.
+ */
+
+export const RUNTIME_QUEUE_HEALTH_MAX_ABSURD_QUEUES = 128;
+export const RUNTIME_QUEUE_HEALTH_DEFAULT_STALE_AFTER_MS = 90_000;
+export const RUNTIME_QUEUE_HEALTH_DEFAULT_WARN_AGE_MS = 60_000;
+export const RUNTIME_QUEUE_HEALTH_DEFAULT_FAIL_AGE_MS = 300_000;
+/**
+ * Queue health is a cheap control-plane read.  A bucket may contain more
+ * work than this cap; in that case its depth is explicitly reported as a
+ * lower bound instead of making the observer scan the whole queue.
+ */
+export const RUNTIME_QUEUE_HEALTH_BUCKET_SAMPLE_LIMIT = 1_000;
+
+export type RuntimeQueueHealthStatus = 'green' | 'amber' | 'red' | 'unknown';
+
+export type RuntimeQueueDepth = {
+  ready: number;
+  delayed: number;
+  inFlight: number;
+  blocked: number;
+  deadLettered: number;
+};
+
+export type RuntimeQueueOldestAge = {
+  ready: number | null;
+  delayed: number | null;
+  inFlight: number | null;
+  blocked: number | null;
+  deadLettered: number | null;
+};
+
+export type RuntimeQueueFlow = {
+  admitted: number;
+  started: number;
+  completed: number;
+  failed: number;
+  retried: number;
+  poisoned: number;
+};
+
+export type RuntimeQueueHealthProjection = {
+  id: string;
+  kind: 'absurd' | 'run_ledger_outbox' | 'sandbox_cleanup';
+  source: 'absurd' | 'runtime_postgres';
+  observedAt: string;
+  status: RuntimeQueueHealthStatus;
+  statusReason: string;
+  /** Shared scheduler-neutral contract consumed by transports and dashboards. */
+  observation: QueueObservation;
+  health: QueueHealth;
+  depth: RuntimeQueueDepth;
+  oldestAgeMs: RuntimeQueueOldestAge;
+  flow: RuntimeQueueFlow;
+  lifecycle: {
+    started: number;
+    completed: number;
+    unresolved: number;
+    triggerGap: number;
+  };
+  thresholds: {
+    warnAgeMs: number;
+    failAgeMs: number;
+    staleAfterMs: number;
+  };
+};
+
+export type RuntimeQueueHealthSnapshot = {
+  observedAt: string;
+  staleAfterMs: number;
+  queues: RuntimeQueueHealthProjection[];
+  summary: {
+    green: number;
+    amber: number;
+    red: number;
+    unknown: number;
+  };
+};
+
+export type RuntimeQueueHealthOptions = {
+  schema?: string;
+  now?: Date;
+  staleAfterMs?: number;
+  warnAgeMs?: number;
+  failAgeMs?: number;
+  /** Restrict Absurd rows to these queue names. Registry values are validated. */
+  absurdQueueNames?: string[];
+  maxAbsurdQueues?: number;
+};
+
+export type RuntimeQueueHealthThresholds = Required<
+  Pick<
+    RuntimeQueueHealthOptions,
+    'now' | 'staleAfterMs' | 'warnAgeMs' | 'failAgeMs'
+  >
+>;
+
+type CountRow = {
+  ready: string | number;
+  delayed: string | number;
+  in_flight: string | number;
+  blocked: string | number;
+  dead_lettered: string | number;
+  oldest_ready_ms: string | number | null;
+  oldest_delayed_ms: string | number | null;
+  oldest_in_flight_ms: string | number | null;
+  oldest_blocked_ms: string | number | null;
+  oldest_dead_lettered_ms: string | number | null;
+  admitted: string | number;
+  started: string | number;
+  completed: string | number;
+  failed: string | number;
+  retried: string | number;
+  ready_truncated?: boolean;
+  delayed_truncated?: boolean;
+  in_flight_truncated?: boolean;
+  blocked_truncated?: boolean;
+  dead_lettered_truncated?: boolean;
+  flow_available?: boolean;
+  flow_unavailable_reason?: string | null;
+};
+
+function quoteIdentifier(value: string, label: string): string {
+  if (!value || Buffer.byteLength(value, 'utf8') > 63) {
+    throw new Error(
+      `${label} must be a non-empty SQL identifier of at most 63 bytes.`,
+    );
+  }
+  return `"${value.replaceAll('"', '""')}"`;
+}
+
+function schemaName(schema?: string): string {
+  const value = schema?.trim() || 'runtime_scheduler';
+  if (!/^[A-Za-z_][A-Za-z0-9_]{0,62}$/.test(value)) {
+    throw new Error('schema must be a simple SQL identifier.');
+  }
+  return quoteIdentifier(value, 'schema');
+}
+
+function numberValue(value: string | number | null | undefined): number {
+  if (value === null || value === undefined) return 0;
+  const parsed = typeof value === 'number' ? value : Number(value);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : 0;
+}
+
+function ageValue(value: string | number | null | undefined): number | null {
+  if (value === null || value === undefined) return null;
+  const parsed = numberValue(value);
+  return parsed > 0 ? Math.floor(parsed) : null;
+}
+
+function optionsWithDefaults(
+  input: RuntimeQueueHealthOptions = {},
+): RuntimeQueueHealthThresholds {
+  const now = input.now ?? new Date();
+  const staleAfterMs =
+    input.staleAfterMs ?? RUNTIME_QUEUE_HEALTH_DEFAULT_STALE_AFTER_MS;
+  const warnAgeMs = input.warnAgeMs ?? RUNTIME_QUEUE_HEALTH_DEFAULT_WARN_AGE_MS;
+  const failAgeMs = input.failAgeMs ?? RUNTIME_QUEUE_HEALTH_DEFAULT_FAIL_AGE_MS;
+  if (
+    !Number.isFinite(staleAfterMs) ||
+    staleAfterMs <= 0 ||
+    !Number.isFinite(warnAgeMs) ||
+    warnAgeMs <= 0 ||
+    !Number.isFinite(failAgeMs) ||
+    failAgeMs < warnAgeMs
+  ) {
+    throw new Error('Queue health age thresholds are invalid.');
+  }
+  return { now, staleAfterMs, warnAgeMs, failAgeMs };
+}
+
+export function deriveRuntimeQueueHealthStatus(input: {
+  depth: RuntimeQueueDepth;
+  oldestAgeMs: RuntimeQueueOldestAge;
+  options: RuntimeQueueHealthThresholds;
+  observedAt: Date;
+  statusReason?: string;
+}): { status: RuntimeQueueHealthStatus; statusReason: string } {
+  if (input.statusReason)
+    return { status: 'unknown', statusReason: input.statusReason };
+  const age =
+    input.oldestAgeMs.ready ??
+    input.oldestAgeMs.inFlight ??
+    input.oldestAgeMs.delayed;
+  const ageForFailure = [
+    input.oldestAgeMs.ready,
+    input.oldestAgeMs.inFlight,
+    input.oldestAgeMs.blocked,
+    input.oldestAgeMs.deadLettered,
+  ].reduce<number | null>(
+    (max, current) =>
+      current === null ? max : max === null ? current : Math.max(max, current),
+    null,
+  );
+  const ageForWarning =
+    age ?? input.oldestAgeMs.blocked ?? input.oldestAgeMs.deadLettered;
+  if (
+    input.options.now.getTime() - input.observedAt.getTime() >
+    input.options.staleAfterMs
+  ) {
+    return { status: 'unknown', statusReason: 'observation_stale' };
+  }
+  if (input.depth.deadLettered > 0) {
+    return { status: 'red', statusReason: 'dead_lettered_work' };
+  }
+  if (input.depth.blocked > 0) {
+    return { status: 'red', statusReason: 'blocked_work' };
+  }
+  if (ageForFailure !== null && ageForFailure >= input.options.failAgeMs) {
+    return { status: 'red', statusReason: 'oldest_work_exceeded_failure_slo' };
+  }
+  if (ageForWarning !== null && ageForWarning >= input.options.warnAgeMs) {
+    return { status: 'amber', statusReason: 'oldest_work_approaching_slo' };
+  }
+  return {
+    status: 'green',
+    statusReason:
+      input.depth.ready + input.depth.delayed + input.depth.inFlight === 0
+        ? 'idle'
+        : 'within_slo',
+  };
+}
+
+function projectionFromRow(input: {
+  id: string;
+  kind: RuntimeQueueHealthProjection['kind'];
+  source: RuntimeQueueHealthProjection['source'];
+  row: CountRow;
+  observedAt: Date;
+  options: RuntimeQueueHealthThresholds;
+}): RuntimeQueueHealthProjection {
+  const depth: RuntimeQueueDepth = {
+    ready: numberValue(input.row.ready),
+    delayed: numberValue(input.row.delayed),
+    inFlight: numberValue(input.row.in_flight),
+    blocked: numberValue(input.row.blocked),
+    deadLettered: numberValue(input.row.dead_lettered),
+  };
+  const oldestAgeMs: RuntimeQueueOldestAge = {
+    ready: ageValue(input.row.oldest_ready_ms),
+    delayed: ageValue(input.row.oldest_delayed_ms),
+    inFlight: ageValue(input.row.oldest_in_flight_ms),
+    blocked: ageValue(input.row.oldest_blocked_ms),
+    deadLettered: ageValue(input.row.oldest_dead_lettered_ms),
+  };
+  const lifecycle = {
+    started: numberValue(input.row.started),
+    completed: numberValue(input.row.completed),
+    unresolved: depth.ready + depth.delayed + depth.inFlight + depth.blocked,
+    // A trigger gap is the durable work that has not started yet. It is a
+    // signal, not a failure: an expected-but-not-started schedule occurrence
+    // needs the scheduler's own occurrence ledger to become actionable.
+    triggerGap: depth.ready + depth.delayed,
+  };
+  const derivedStatus = deriveRuntimeQueueHealthStatus({
+    depth,
+    oldestAgeMs,
+    options: input.options,
+    observedAt: input.observedAt,
+  });
+  const hasTruncatedDepth = [
+    input.row.ready_truncated,
+    input.row.delayed_truncated,
+    input.row.in_flight_truncated,
+    input.row.blocked_truncated,
+    input.row.dead_lettered_truncated,
+  ].some(Boolean);
+  const derived =
+    hasTruncatedDepth && derivedStatus.status === 'green'
+      ? {
+          status: 'amber' as const,
+          statusReason: 'bounded_depth_sample',
+        }
+      : derivedStatus;
+  const definition: QueueDefinition = {
+    id: input.id,
+    owner: 'runtime',
+    purpose:
+      input.kind === 'absurd'
+        ? 'Run admitted Play runtime tasks to terminal state'
+        : input.kind === 'run_ledger_outbox'
+          ? 'Deliver causal Run Ledger events to control-plane projections'
+          : 'Complete durable sandbox deletion obligations',
+    unit: 'items',
+    kind: input.kind === 'absurd' ? 'fifo' : 'leased_workset',
+    ageSlo: {
+      warnMs: input.options.warnAgeMs,
+      failMs: input.options.failAgeMs,
+    },
+    staleAfterMs: input.options.staleAfterMs,
+  };
+  const measureAge = (value: number | null, truncated?: boolean) =>
+    value === null
+      ? notApplicable<number>('no work in this bucket')
+      : truncated
+        ? lowerBound(value)
+        : exact(value);
+  const measureDepth = (value: number, truncated?: boolean) =>
+    truncated ? lowerBound(value) : exact(value);
+  const flowAvailable = input.row.flow_available !== false;
+  const flowUnavailableReason =
+    input.row.flow_unavailable_reason ??
+    'runtime queue health does not include an exact rolling lifecycle projection';
+  const observation: QueueObservation = {
+    observedAt: input.observedAt.getTime(),
+    adapter: `runtime-queue-health/${input.source}`,
+    depth: {
+      ready: measureDepth(depth.ready, input.row.ready_truncated),
+      delayed: measureDepth(depth.delayed, input.row.delayed_truncated),
+      inFlight: measureDepth(depth.inFlight, input.row.in_flight_truncated),
+      blocked: measureDepth(depth.blocked, input.row.blocked_truncated),
+      deadLettered: measureDepth(
+        depth.deadLettered,
+        input.row.dead_lettered_truncated,
+      ),
+    },
+    oldestAgeMs: {
+      ready: measureAge(oldestAgeMs.ready, input.row.ready_truncated),
+      delayed: measureAge(oldestAgeMs.delayed, input.row.delayed_truncated),
+      inFlight: measureAge(oldestAgeMs.inFlight, input.row.in_flight_truncated),
+      blocked: measureAge(oldestAgeMs.blocked, input.row.blocked_truncated),
+      deadLettered: measureAge(
+        oldestAgeMs.deadLettered,
+        input.row.dead_lettered_truncated,
+      ),
+    },
+    flow: {
+      admitted: flowAvailable
+        ? exact(numberValue(input.row.admitted))
+        : unavailable(flowUnavailableReason),
+      started: flowAvailable
+        ? exact(lifecycle.started)
+        : unavailable(flowUnavailableReason),
+      completed: flowAvailable
+        ? exact(lifecycle.completed)
+        : unavailable(flowUnavailableReason),
+      failed: flowAvailable
+        ? exact(numberValue(input.row.failed))
+        : unavailable(flowUnavailableReason),
+      retried: flowAvailable
+        ? exact(numberValue(input.row.retried))
+        : unavailable(flowUnavailableReason),
+      poisoned: flowAvailable
+        ? exact(depth.deadLettered)
+        : unavailable(flowUnavailableReason),
+    },
+  };
+  const health = deriveQueueHealth(
+    definition,
+    observation,
+    input.options.now.getTime(),
+  );
+  return {
+    id: input.id,
+    kind: input.kind,
+    source: input.source,
+    observedAt: input.observedAt.toISOString(),
+    status: derived.status,
+    statusReason: derived.statusReason,
+    observation,
+    health,
+    depth,
+    oldestAgeMs,
+    flow: {
+      admitted: numberValue(input.row.admitted),
+      started: lifecycle.started,
+      completed: lifecycle.completed,
+      failed: numberValue(input.row.failed),
+      retried: numberValue(input.row.retried),
+      poisoned: depth.deadLettered,
+    },
+    lifecycle,
+    thresholds: {
+      warnAgeMs: input.options.warnAgeMs,
+      failAgeMs: input.options.failAgeMs,
+      staleAfterMs: input.options.staleAfterMs,
+    },
+  };
+}
+
+function intervalExpr(column: string, nowParameter = '$1'): string {
+  return `GREATEST(0, floor(extract(epoch FROM (${nowParameter}::timestamptz - ${column})) * 1000))`;
+}
+
+function absurdProjectionQuery(queueTable: string): string {
+  const table = `absurd.${quoteIdentifier(queueTable, 'queue table')}`;
+  return `
+    WITH sampled AS (
+      (SELECT 'ready'::text AS bucket, available_at AS age_at
+      FROM ${table}
+      WHERE state = 'pending' AND available_at <= $1::timestamptz
+      ORDER BY available_at, run_id
+      LIMIT $3)
+      UNION ALL
+      (SELECT 'delayed'::text, available_at
+      FROM ${table}
+      WHERE state = 'sleeping'
+      ORDER BY available_at, run_id
+      LIMIT $3)
+      UNION ALL
+      (SELECT 'delayed'::text, available_at
+      FROM ${table}
+      WHERE state = 'pending' AND available_at > $1::timestamptz
+      ORDER BY available_at, run_id
+      LIMIT $3)
+      UNION ALL
+      (SELECT 'in_flight'::text, coalesce(started_at, created_at)
+      FROM ${table}
+      WHERE state = 'running'
+        AND (claim_expires_at IS NULL OR claim_expires_at > $1::timestamptz)
+      ORDER BY available_at, run_id
+      LIMIT $3)
+      UNION ALL
+      (SELECT 'blocked'::text, coalesce(claim_expires_at, started_at, created_at)
+      FROM ${table}
+      WHERE state = 'running'
+        AND claim_expires_at IS NOT NULL
+        AND claim_expires_at <= $1::timestamptz
+      ORDER BY claim_expires_at, run_id
+      LIMIT $3)
+      UNION ALL
+      (SELECT 'dead_lettered'::text, coalesce(failed_at, created_at)
+      FROM ${table}
+      WHERE state = 'failed'
+      ORDER BY available_at, run_id
+      LIMIT $3)
+    )
+    SELECT
+      least(count(*) FILTER (WHERE bucket = 'ready'), $4) AS ready,
+      least(count(*) FILTER (WHERE bucket = 'delayed'), $4) AS delayed,
+      least(count(*) FILTER (WHERE bucket = 'in_flight'), $4) AS in_flight,
+      least(count(*) FILTER (WHERE bucket = 'blocked'), $4) AS blocked,
+      least(count(*) FILTER (WHERE bucket = 'dead_lettered'), $4) AS dead_lettered,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'ready') AS oldest_ready_ms,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'delayed') AS oldest_delayed_ms,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'in_flight') AS oldest_in_flight_ms,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'blocked') AS oldest_blocked_ms,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'dead_lettered') AS oldest_dead_lettered_ms,
+      count(*) FILTER (WHERE bucket = 'ready') > $4 AS ready_truncated,
+      count(*) FILTER (WHERE bucket = 'delayed') > $4 AS delayed_truncated,
+      count(*) FILTER (WHERE bucket = 'in_flight') > $4 AS in_flight_truncated,
+      count(*) FILTER (WHERE bucket = 'blocked') > $4 AS blocked_truncated,
+      count(*) FILTER (WHERE bucket = 'dead_lettered') > $4 AS dead_lettered_truncated,
+      0::bigint AS admitted,
+      0::bigint AS started,
+      0::bigint AS completed,
+      0::bigint AS failed,
+      0::bigint AS retried,
+      false AS flow_available,
+      'Absurd run history has no bounded exact rolling flow projection'::text AS flow_unavailable_reason
+    FROM sampled
+  `;
+}
+
+function outboxProjectionQuery(schema: string): string {
+  const table = `${schema}."outbox"`;
+  return `
+    WITH pending_sample AS (
+      SELECT
+        CASE
+          WHEN lease_owner IS NULL AND coalesce(next_attempt_at, created_at) <= $1::timestamptz THEN 'ready'
+          WHEN lease_owner IS NULL THEN 'delayed'
+          WHEN lease_expires_at > $1::timestamptz THEN 'in_flight'
+          ELSE 'blocked'
+        END AS bucket,
+        CASE
+          WHEN lease_owner IS NULL AND coalesce(next_attempt_at, created_at) > $1::timestamptz
+            THEN coalesce(next_attempt_at, created_at)
+          WHEN lease_owner IS NULL THEN created_at
+          WHEN lease_expires_at > $1::timestamptz
+            THEN coalesce(delivery_batch_started_at, created_at)
+          ELSE coalesce(lease_expires_at, created_at)
+        END AS age_at
+      FROM ${table}
+      WHERE delivered_at IS NULL AND failed_at IS NULL
+      ORDER BY coalesce(next_attempt_at, created_at), created_at, event_id
+      LIMIT $3
+    ), dead_sample AS (
+      SELECT 'dead_lettered'::text AS bucket, coalesce(failed_at, created_at) AS age_at
+      FROM ${table}
+      WHERE delivered_at IS NULL AND failed_at IS NOT NULL
+      ORDER BY failed_at, event_id
+      LIMIT $3
+    ), sampled AS (
+      SELECT * FROM pending_sample
+      UNION ALL
+      SELECT * FROM dead_sample
+    )
+    SELECT
+      least(count(*) FILTER (WHERE bucket = 'ready'), $4) AS ready,
+      least(count(*) FILTER (WHERE bucket = 'delayed'), $4) AS delayed,
+      least(count(*) FILTER (WHERE bucket = 'in_flight'), $4) AS in_flight,
+      least(count(*) FILTER (WHERE bucket = 'blocked'), $4) AS blocked,
+      least(count(*) FILTER (WHERE bucket = 'dead_lettered'), $4) AS dead_lettered,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'ready') AS oldest_ready_ms,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'delayed') AS oldest_delayed_ms,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'in_flight') AS oldest_in_flight_ms,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'blocked') AS oldest_blocked_ms,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'dead_lettered') AS oldest_dead_lettered_ms,
+      count(*) FILTER (WHERE bucket <> 'dead_lettered') > $4 AS ready_truncated,
+      count(*) FILTER (WHERE bucket <> 'dead_lettered') > $4 AS delayed_truncated,
+      count(*) FILTER (WHERE bucket <> 'dead_lettered') > $4 AS in_flight_truncated,
+      count(*) FILTER (WHERE bucket <> 'dead_lettered') > $4 AS blocked_truncated,
+      count(*) FILTER (WHERE bucket = 'dead_lettered') > $4 AS dead_lettered_truncated,
+      0::bigint AS admitted,
+      0::bigint AS started,
+      0::bigint AS completed,
+      0::bigint AS failed,
+      0::bigint AS retried,
+      false AS flow_available,
+      'Outbox health has no bounded exact rolling flow projection'::text AS flow_unavailable_reason
+    FROM sampled
+  `;
+}
+
+function cleanupProjectionQuery(schema: string): string {
+  const table = `${schema}."sandbox_cleanup_jobs_v2"`;
+  return `
+    WITH sampled AS (
+      SELECT
+        CASE
+          WHEN blocked_at IS NOT NULL THEN 'blocked'
+          WHEN lease_owner IS NOT NULL AND lease_expires_at > $1::timestamptz THEN 'in_flight'
+          WHEN lease_owner IS NOT NULL OR status = 'deleting' THEN 'blocked'
+          WHEN available_at > $1::timestamptz THEN 'delayed'
+          ELSE 'ready'
+        END AS bucket,
+        CASE
+          WHEN blocked_at IS NOT NULL THEN blocked_at
+          WHEN lease_owner IS NOT NULL AND lease_expires_at > $1::timestamptz
+            THEN coalesce(updated_at, created_at)
+          WHEN lease_owner IS NOT NULL OR status = 'deleting'
+            THEN coalesce(lease_expires_at, created_at)
+          WHEN available_at > $1::timestamptz THEN available_at
+          ELSE created_at
+        END AS age_at
+      FROM ${table}
+      WHERE status <> 'deleted'
+      ORDER BY available_at, created_at, provider, sandbox_id
+      LIMIT $3
+    )
+    SELECT
+      least(count(*) FILTER (WHERE bucket = 'ready'), $4) AS ready,
+      least(count(*) FILTER (WHERE bucket = 'delayed'), $4) AS delayed,
+      least(count(*) FILTER (WHERE bucket = 'in_flight'), $4) AS in_flight,
+      least(count(*) FILTER (WHERE bucket = 'blocked'), $4) AS blocked,
+      0::bigint AS dead_lettered,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'ready') AS oldest_ready_ms,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'delayed') AS oldest_delayed_ms,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'in_flight') AS oldest_in_flight_ms,
+      max(${intervalExpr('age_at')}) FILTER (WHERE bucket = 'blocked') AS oldest_blocked_ms,
+      NULL::bigint AS oldest_dead_lettered_ms,
+      count(*) > $4 AS ready_truncated,
+      count(*) > $4 AS delayed_truncated,
+      count(*) > $4 AS in_flight_truncated,
+      count(*) > $4 AS blocked_truncated,
+      false AS dead_lettered_truncated,
+      0::bigint AS admitted,
+      0::bigint AS started,
+      0::bigint AS completed,
+      0::bigint AS failed,
+      0::bigint AS retried,
+      false AS flow_available,
+      'Sandbox cleanup health has no bounded exact rolling flow projection'::text AS flow_unavailable_reason
+    FROM sampled
+  `;
+}
+
+function rowFromResult(result: { rows: CountRow[] }): CountRow {
+  return (
+    result.rows[0] ?? {
+      ready: 0,
+      delayed: 0,
+      in_flight: 0,
+      blocked: 0,
+      dead_lettered: 0,
+      oldest_ready_ms: null,
+      oldest_delayed_ms: null,
+      oldest_in_flight_ms: null,
+      oldest_blocked_ms: null,
+      oldest_dead_lettered_ms: null,
+      admitted: 0,
+      started: 0,
+      completed: 0,
+      failed: 0,
+      retried: 0,
+    }
+  );
+}
+
+/** Read all bounded runtime queue projections on one scheduler connection. */
+export async function readRuntimeQueueHealthFromClient(
+  client: PostgresSchedulerQueryClient,
+  input: RuntimeQueueHealthOptions = {},
+): Promise<RuntimeQueueHealthSnapshot> {
+  const options = optionsWithDefaults(input);
+  const now = options.now;
+  const schema = schemaName(input.schema);
+  const sampleLimit = RUNTIME_QUEUE_HEALTH_BUCKET_SAMPLE_LIMIT;
+  const maxQueues = Math.max(
+    1,
+    Math.min(
+      input.maxAbsurdQueues ?? RUNTIME_QUEUE_HEALTH_MAX_ABSURD_QUEUES,
+      RUNTIME_QUEUE_HEALTH_MAX_ABSURD_QUEUES,
+    ),
+  );
+  const requestedNames = input.absurdQueueNames?.length
+    ? [
+        ...new Set(
+          input.absurdQueueNames.map((name) => name.trim()).filter(Boolean),
+        ),
+      ]
+    : undefined;
+  if (requestedNames && requestedNames.length > maxQueues) {
+    throw new Error(
+      `Requested ${requestedNames.length} Absurd queues, exceeding the ${maxQueues} queue health limit.`,
+    );
+  }
+  const registry = requestedNames
+    ? await client.query<{ queue_name: string }>(
+        `SELECT queue_name FROM absurd.queues WHERE queue_name = ANY($1::text[]) ORDER BY queue_name LIMIT $2`,
+        [requestedNames, maxQueues],
+      )
+    : await client.query<{ queue_name: string }>(
+        `SELECT queue_name FROM absurd.queues ORDER BY queue_name LIMIT $1`,
+        [maxQueues],
+      );
+  const queueNames = registry.rows.map((row) => row.queue_name);
+  if (requestedNames) {
+    const found = new Set(queueNames);
+    const missing = requestedNames.filter((name) => !found.has(name));
+    if (missing.length > 0) {
+      throw new Error(
+        `Requested Absurd queues are not registered: ${missing.join(', ')}`,
+      );
+    }
+  }
+  const projections: RuntimeQueueHealthProjection[] = [];
+  for (const queueName of queueNames) {
+    const result = await client.query<CountRow>(
+      absurdProjectionQuery(`r_${queueName}`),
+      [now, now, sampleLimit + 1, sampleLimit],
+    );
+    projections.push(
+      projectionFromRow({
+        id: `absurd:${queueName}`,
+        kind: 'absurd',
+        source: 'absurd',
+        row: rowFromResult(result),
+        observedAt: now,
+        options,
+      }),
+    );
+  }
+  const [outbox, cleanup] = await Promise.all([
+    client.query<CountRow>(outboxProjectionQuery(schema), [
+      now,
+      now,
+      sampleLimit + 1,
+      sampleLimit,
+    ]),
+    client.query<CountRow>(cleanupProjectionQuery(schema), [
+      now,
+      now,
+      sampleLimit + 1,
+      sampleLimit,
+    ]),
+  ]);
+  projections.push(
+    projectionFromRow({
+      id: 'runtime:run-ledger-outbox',
+      kind: 'run_ledger_outbox',
+      source: 'runtime_postgres',
+      row: rowFromResult(outbox),
+      observedAt: now,
+      options,
+    }),
+    projectionFromRow({
+      id: 'runtime:sandbox-cleanup-jobs-v2',
+      kind: 'sandbox_cleanup',
+      source: 'runtime_postgres',
+      row: rowFromResult(cleanup),
+      observedAt: now,
+      options,
+    }),
+  );
+  const summary = {
+    green: projections.filter((p) => p.status === 'green').length,
+    amber: projections.filter((p) => p.status === 'amber').length,
+    red: projections.filter((p) => p.status === 'red').length,
+    unknown: projections.filter((p) => p.status === 'unknown').length,
+  };
+  return {
+    observedAt: now.toISOString(),
+    staleAfterMs: options.staleAfterMs,
+    queues: projections,
+    summary,
+  };
+}