deepline 0.3.114 → 0.3.116

package/dist/bundling-sources/shared_libs/observability/queue-health.ts

@@ -118,6 +118,15 @@ export const OPERATIONS_QUEUE_CATALOG = [
     60_000,
     180_000,
   ),
+  queue(
+    'compute-billing-jobs',
+    'Compute billing jobs',
+    'leased_workset',
+    'billing',
+    'Settle durable compute billing obligations after a runtime run ends',
+    300_000,
+    900_000,
+  ),
   queue(
     'play-webhook-receipts',
     'Play webhook receipts',
@@ -707,7 +716,10 @@ export type QueueHealthAttentionReason =
   | 'backlog_count_inexact'
   | 'backlog_age_unavailable';
 export type QueueHealthCount = { value: number | null; exact: boolean };
-export type QueueHealthCapacity = { active: number | null; limit: number | null };
+export type QueueHealthCapacity = {
+  active: number | null;
+  limit: number | null;
+};
 export type QueueHealthSnapshot = {
   schemaVersion: typeof QUEUE_HEALTH_SCHEMA_VERSION;
   queueId: string;
@@ -761,13 +773,16 @@ export function parseQueueHealthSnapshot(input: unknown): QueueHealthSnapshot {
   if (record.schemaVersion !== 1)
     throw new Error('Queue health schemaVersion must be 1.');
   const layer = legacyString(record.layer, 'layer') as QueueHealthLayer;
-  if (!legacyLayers.has(layer)) throw new Error('Queue health layer is not supported.');
+  if (!legacyLayers.has(layer))
+    throw new Error('Queue health layer is not supported.');
   const availability = legacyString(
     record.availability,
     'availability',
   ) as QueueHealthAvailability;
   if (availability !== 'available' && availability !== 'unavailable')
-    throw new Error('Queue health availability must be available or unavailable.');
+    throw new Error(
+      'Queue health availability must be available or unavailable.',
+    );
   const freshness = legacyRecord(record.freshness, 'freshness');
   const capacity = legacyRecord(record.capacity, 'capacity');
   const thresholds = legacyRecord(record.thresholds, 'thresholds');
@@ -778,15 +793,26 @@ export function parseQueueHealthSnapshot(input: unknown): QueueHealthSnapshot {
     layer,
     transport: legacyIdentifier(record.transport, 'transport'),
     observedAt: legacyNumber(record.observedAt, 'observedAt'),
-    freshness: { staleAfterMs: legacyPositive(freshness.staleAfterMs, 'freshness.staleAfterMs') },
+    freshness: {
+      staleAfterMs: legacyPositive(
+        freshness.staleAfterMs,
+        'freshness.staleAfterMs',
+      ),
+    },
     availability,
     queued: legacyCount(record.queued, 'queued'),
     running: legacyCount(record.running, 'running'),
     retryWaiting: legacyCount(record.retryWaiting, 'retryWaiting'),
     blocked: legacyCount(record.blocked, 'blocked'),
     deadLetter: legacyCount(record.deadLetter, 'deadLetter'),
-    oldestQueuedAgeMs: legacyNullableNumber(record.oldestQueuedAgeMs, 'oldestQueuedAgeMs'),
-    oldestBlockedAgeMs: legacyNullableNumber(record.oldestBlockedAgeMs, 'oldestBlockedAgeMs'),
+    oldestQueuedAgeMs: legacyNullableNumber(
+      record.oldestQueuedAgeMs,
+      'oldestQueuedAgeMs',
+    ),
+    oldestBlockedAgeMs: legacyNullableNumber(
+      record.oldestBlockedAgeMs,
+      'oldestBlockedAgeMs',
+    ),
     capacity: {
       active: legacyNullableNumber(capacity.active, 'capacity.active'),
       limit: legacyNullableNumber(capacity.limit, 'capacity.limit'),
@@ -802,10 +828,17 @@ export function parseQueueHealthSnapshot(input: unknown): QueueHealthSnapshot {
     for (const name of legacyCountNames) {
       const count = snapshot[name];
       if (count.value !== null || count.exact)
-        throw new Error(`Queue health ${name} must be unknown when availability is unavailable.`);
+        throw new Error(
+          `Queue health ${name} must be unknown when availability is unavailable.`,
+        );
     }
-    if (snapshot.oldestQueuedAgeMs !== null || snapshot.oldestBlockedAgeMs !== null)
-      throw new Error('Queue health oldestQueuedAgeMs must be null when availability is unavailable.');
+    if (
+      snapshot.oldestQueuedAgeMs !== null ||
+      snapshot.oldestBlockedAgeMs !== null
+    )
+      throw new Error(
+        'Queue health oldestQueuedAgeMs must be null when availability is unavailable.',
+      );
   }
   if (
     snapshot.queued.exact &&
@@ -814,7 +847,9 @@ export function parseQueueHealthSnapshot(input: unknown): QueueHealthSnapshot {
     snapshot.retryWaiting.value === 0 &&
     snapshot.oldestQueuedAgeMs !== null
   ) {
-    throw new Error('Queue health oldestQueuedAgeMs requires queued or retry-waiting work.');
+    throw new Error(
+      'Queue health oldestQueuedAgeMs requires queued or retry-waiting work.',
+    );
   }
   if (
     snapshot.blocked.exact &&
@@ -823,7 +858,9 @@ export function parseQueueHealthSnapshot(input: unknown): QueueHealthSnapshot {
     snapshot.deadLetter.value === 0 &&
     snapshot.oldestBlockedAgeMs !== null
   ) {
-    throw new Error('Queue health oldestBlockedAgeMs requires blocked or dead-letter work.');
+    throw new Error(
+      'Queue health oldestBlockedAgeMs requires blocked or dead-letter work.',
+    );
   }
   return snapshot;
 }
@@ -843,31 +880,57 @@ export function assessQueueHealth(
   const freshness: QueueHealthFreshness =
     observedAgeMs > snapshot.freshness.staleAfterMs ? 'stale' : 'fresh';
   if (snapshot.availability === 'unavailable')
-    return { status: 'unknown', freshness, observedAgeMs, attentionReasons: ['source_unavailable'] };
+    return {
+      status: 'unknown',
+      freshness,
+      observedAgeMs,
+      attentionReasons: ['source_unavailable'],
+    };
   if (freshness === 'stale')
-    return { status: 'unknown', freshness, observedAgeMs, attentionReasons: ['snapshot_stale'] };
+    return {
+      status: 'unknown',
+      freshness,
+      observedAgeMs,
+      attentionReasons: ['snapshot_stale'],
+    };
   const attentionReasons: QueueHealthAttentionReason[] = [];
   if (knownItems(snapshot.blocked)) attentionReasons.push('blocked_work');
-  if (knownItems(snapshot.deadLetter)) attentionReasons.push('dead_letter_work');
+  if (knownItems(snapshot.deadLetter))
+    attentionReasons.push('dead_letter_work');
   if (attentionReasons.length)
     return { status: 'blocked', freshness, observedAgeMs, attentionReasons };
   if (
     snapshot.oldestQueuedAgeMs !== null &&
     snapshot.oldestQueuedAgeMs >= snapshot.thresholds.degradedOldestQueuedAgeMs
   )
-    return { status: 'degraded', freshness, observedAgeMs, attentionReasons: ['backlog_aged'] };
+    return {
+      status: 'degraded',
+      freshness,
+      observedAgeMs,
+      attentionReasons: ['backlog_aged'],
+    };
   if (
     !snapshot.queued.exact ||
     !snapshot.retryWaiting.exact ||
     !snapshot.blocked.exact ||
     !snapshot.deadLetter.exact
   )
-    return { status: 'unknown', freshness, observedAgeMs, attentionReasons: ['backlog_count_inexact'] };
+    return {
+      status: 'unknown',
+      freshness,
+      observedAgeMs,
+      attentionReasons: ['backlog_count_inexact'],
+    };
   if (
     (knownItems(snapshot.queued) || knownItems(snapshot.retryWaiting)) &&
     snapshot.oldestQueuedAgeMs === null
   )
-    return { status: 'unknown', freshness, observedAgeMs, attentionReasons: ['backlog_age_unavailable'] };
+    return {
+      status: 'unknown',
+      freshness,
+      observedAgeMs,
+      attentionReasons: ['backlog_age_unavailable'],
+    };
   return { status: 'healthy', freshness, observedAgeMs, attentionReasons: [] };
 }
 
@@ -902,7 +965,8 @@ export function queueHealthTelemetryFields(
     queueOldestBlockedAgeMs: snapshot.oldestBlockedAgeMs,
     queueCapacityActive: snapshot.capacity.active,
     queueCapacityLimit: snapshot.capacity.limit,
-    queueDegradedOldestQueuedAgeMs: snapshot.thresholds.degradedOldestQueuedAgeMs,
+    queueDegradedOldestQueuedAgeMs:
+      snapshot.thresholds.degradedOldestQueuedAgeMs,
     queueAttentionReasons: assessment.attentionReasons,
   };
 }
@@ -933,12 +997,16 @@ function legacyString(value: unknown, label: string): string {
 function legacyIdentifier(value: unknown, label: string): string {
   const result = legacyString(value, label);
   if (result.length > 96 || !/^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$/.test(result))
-    throw new Error(`Queue health ${label} must be a stable low-cardinality identifier.`);
+    throw new Error(
+      `Queue health ${label} must be a stable low-cardinality identifier.`,
+    );
   return result;
 }
 function legacyNumber(value: unknown, label: string): number {
   if (typeof value !== 'number' || !Number.isFinite(value) || value < 0)
-    throw new Error(`Queue health ${label} must be a finite non-negative number.`);
+    throw new Error(
+      `Queue health ${label} must be a finite non-negative number.`,
+    );
   return value;
 }
 function legacyPositive(value: unknown, label: string): number {
@@ -955,7 +1023,9 @@ function legacyCount(value: unknown, label: string): QueueHealthCount {
     throw new Error(`Queue health ${label}.exact must be a boolean.`);
   const count = legacyNullableNumber(record.value, `${label}.value`);
   if (count === null && record.exact)
-    throw new Error(`Queue health ${label}.exact cannot be true when value is unknown.`);
+    throw new Error(
+      `Queue health ${label}.exact cannot be true when value is unknown.`,
+    );
   return { value: count, exact: record.exact };
 }
 function knownItems(count: QueueHealthCount): boolean {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -217,11 +217,12 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
  * interrupted deployment cannot claim a newer version without its required
  * objects.
  */
-// v37 adds the runtime-owned cron schedule cursor and its due-time index. The
-// table is additive: v36 admission and outbox projection remain valid while a
-// deploy expands the shared schema, then the marker records the completed
-// runtime-cron capability for deployment tooling.
-export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 37;
+// v38 adds bounded health indexes for the runtime-owned outbox, sandbox
+// cleanup, and compute-billing queues. The indexes are additive: v37 cron
+// scheduling, admission, and outbox projection remain valid while a deploy
+// expands the shared schema, then the marker records the completed health
+// capability for deployment tooling.
+export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 38;
 const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
   'scheduler_schema_migrations';
 export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
@@ -1567,6 +1568,87 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
             AND index_row.indrelid = to_regclass($3)
             AND index_row.indisready AND index_row.indisvalid
         )
+        AND EXISTS (
+          SELECT 1 FROM pg_index AS index_row
+          JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
+          JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
+          WHERE index_namespace.nspname = $1
+            AND index_class.relname = 'runtime_scheduler_outbox_ready_health_idx'
+            AND index_row.indrelid = to_regclass($3)
+            AND index_row.indisready AND index_row.indisvalid
+        )
+        AND EXISTS (
+          SELECT 1 FROM pg_index AS index_row
+          JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
+          JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
+          WHERE index_namespace.nspname = $1
+            AND index_class.relname = 'runtime_scheduler_outbox_dead_health_idx'
+            AND index_row.indrelid = to_regclass($3)
+            AND index_row.indisready AND index_row.indisvalid
+        )
+        AND EXISTS (
+          SELECT 1 FROM pg_index AS index_row
+          JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
+          JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
+          WHERE index_namespace.nspname = $1
+            AND index_class.relname = 'runtime_scheduler_outbox_lease_health_idx'
+            AND index_row.indrelid = to_regclass($3)
+            AND index_row.indisready AND index_row.indisvalid
+        )
+        AND EXISTS (
+          SELECT 1 FROM pg_index AS index_row
+          JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
+          JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
+          WHERE index_namespace.nspname = $1
+            AND index_class.relname = 'runtime_scheduler_sandbox_cleanup_v2_ready_health_idx'
+            AND index_row.indrelid = to_regclass($7)
+            AND index_row.indisready AND index_row.indisvalid
+        )
+        AND EXISTS (
+          SELECT 1 FROM pg_index AS index_row
+          JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
+          JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
+          WHERE index_namespace.nspname = $1
+            AND index_class.relname = 'runtime_scheduler_sandbox_cleanup_v2_lease_health_idx'
+            AND index_row.indrelid = to_regclass($7)
+            AND index_row.indisready AND index_row.indisvalid
+        )
+        AND EXISTS (
+          SELECT 1 FROM pg_index AS index_row
+          JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
+          JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
+          WHERE index_namespace.nspname = $1
+            AND index_class.relname = 'runtime_scheduler_compute_billing_ready_health_idx'
+            AND index_row.indrelid = to_regclass($8)
+            AND index_row.indisready AND index_row.indisvalid
+        )
+        AND EXISTS (
+          SELECT 1 FROM pg_index AS index_row
+          JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
+          JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
+          WHERE index_namespace.nspname = $1
+            AND index_class.relname = 'runtime_scheduler_sandbox_cleanup_v2_blocked_health_idx'
+            AND index_row.indrelid = to_regclass($7)
+            AND index_row.indisready AND index_row.indisvalid
+        )
+        AND EXISTS (
+          SELECT 1 FROM pg_index AS index_row
+          JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
+          JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
+          WHERE index_namespace.nspname = $1
+            AND index_class.relname = 'runtime_scheduler_compute_billing_lease_health_idx'
+            AND index_row.indrelid = to_regclass($8)
+            AND index_row.indisready AND index_row.indisvalid
+        )
+        AND EXISTS (
+          SELECT 1 FROM pg_index AS index_row
+          JOIN pg_class AS index_class ON index_class.oid = index_row.indexrelid
+          JOIN pg_namespace AS index_namespace ON index_namespace.oid = index_class.relnamespace
+          WHERE index_namespace.nspname = $1
+            AND index_class.relname = 'runtime_scheduler_compute_billing_blocked_health_idx'
+            AND index_row.indrelid = to_regclass($8)
+            AND index_row.indisready AND index_row.indisvalid
+        )
         AND (
           $5::boolean = false
           OR EXISTS (
@@ -1608,6 +1690,8 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
       `${quotedSchema}.${quoteIdent('runs')}`,
       requirements.deliveredOutboxRetentionIndex !== false,
       requirements.blockedOutboxIndex !== false,
+      `${quotedSchema}.${quoteIdent('sandbox_cleanup_jobs_v2')}`,
+      `${quotedSchema}.${quoteIdent('compute_billing_jobs_v2')}`,
     ],
   );
   return runtimeAdminIndexes.rows[0]?.ready === true;
@@ -3134,6 +3218,30 @@ export async function ensurePostgresSchedulerSchema(
         (available_at, created_at, provider, sandbox_id)
       WHERE status <> 'deleted'
   `);
+    if (!options?.deferRuntimeAdminIndexes) {
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS runtime_scheduler_sandbox_cleanup_v2_ready_health_idx
+          ON ${tableName('sandbox_cleanup_jobs_v2', options)}
+            (available_at, created_at, provider, sandbox_id)
+          WHERE status <> 'deleted'
+            AND blocked_at IS NULL
+            AND lease_owner IS NULL
+            AND status <> 'deleting'
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS runtime_scheduler_sandbox_cleanup_v2_lease_health_idx
+          ON ${tableName('sandbox_cleanup_jobs_v2', options)}
+            (lease_expires_at, created_at, provider, sandbox_id)
+          WHERE status <> 'deleted' AND lease_owner IS NOT NULL
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS runtime_scheduler_sandbox_cleanup_v2_blocked_health_idx
+          ON ${tableName('sandbox_cleanup_jobs_v2', options)}
+            (blocked_at, lease_expires_at, created_at, provider, sandbox_id)
+          WHERE status <> 'deleted'
+            AND (blocked_at IS NOT NULL OR status = 'deleting')
+      `);
+    }
     await client.query(`
     CREATE INDEX IF NOT EXISTS ${POSTGRES_SCHEDULER_COMPUTE_BILLING_PENDING_INDEX}
       ON ${tableName('compute_billing_jobs_v2', options)}
@@ -3187,6 +3295,27 @@ export async function ensurePostgresSchedulerSchema(
           ON ${tableName('outbox', options)} (created_at, event_id)
           WHERE delivered_at IS NULL AND failed_at IS NULL
       `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_lease_health_idx
+          ON ${tableName('outbox', options)}
+            (lease_expires_at, created_at, event_id)
+          WHERE delivered_at IS NULL
+            AND failed_at IS NULL
+            AND lease_owner IS NOT NULL
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_ready_health_idx
+          ON ${tableName('outbox', options)}
+            (coalesce(next_attempt_at, created_at), created_at, event_id)
+          WHERE delivered_at IS NULL
+            AND failed_at IS NULL
+            AND lease_owner IS NULL
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_dead_health_idx
+          ON ${tableName('outbox', options)} (failed_at, event_id)
+          WHERE delivered_at IS NULL AND failed_at IS NOT NULL
+      `);
       await client.query(`
         CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_delivered_retention_idx
           ON ${tableName('outbox', options)} (delivered_at, event_id)
@@ -3198,6 +3327,28 @@ export async function ensurePostgresSchedulerSchema(
           WHERE delivered_at IS NULL AND failed_at IS NOT NULL
       `);
     }
+    if (!options?.deferRuntimeAdminIndexes) {
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS runtime_scheduler_compute_billing_ready_health_idx
+          ON ${tableName('compute_billing_jobs_v2', options)}
+            (available_at, created_at, run_id)
+          WHERE status <> 'settled'
+            AND blocked_at IS NULL
+            AND lease_owner IS NULL
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS runtime_scheduler_compute_billing_lease_health_idx
+          ON ${tableName('compute_billing_jobs_v2', options)}
+            (lease_expires_at, created_at, run_id)
+          WHERE status <> 'settled' AND lease_owner IS NOT NULL
+      `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS runtime_scheduler_compute_billing_blocked_health_idx
+          ON ${tableName('compute_billing_jobs_v2', options)}
+            (blocked_at, lease_expires_at, created_at, run_id)
+          WHERE status <> 'settled' AND blocked_at IS NOT NULL
+      `);
+    }
     await client.query(`
     CREATE INDEX IF NOT EXISTS runtime_scheduler_outbox_run_created_idx
       ON ${tableName('outbox', options)} (run_id, created_at, event_id)
@@ -3274,6 +3425,28 @@ const POSTGRES_SCHEDULER_RUNTIME_ADMIN_INDEXES = [
     definition: `(created_at, event_id)
       WHERE delivered_at IS NULL AND failed_at IS NULL`,
   },
+  {
+    name: 'runtime_scheduler_outbox_lease_health_idx',
+    table: 'outbox',
+    definition: `(lease_expires_at, created_at, event_id)
+      WHERE delivered_at IS NULL
+        AND failed_at IS NULL
+        AND lease_owner IS NOT NULL`,
+  },
+  {
+    name: 'runtime_scheduler_outbox_ready_health_idx',
+    table: 'outbox',
+    definition: `(coalesce(next_attempt_at, created_at), created_at, event_id)
+      WHERE delivered_at IS NULL
+        AND failed_at IS NULL
+        AND lease_owner IS NULL`,
+  },
+  {
+    name: 'runtime_scheduler_outbox_dead_health_idx',
+    table: 'outbox',
+    definition: `(failed_at, event_id)
+      WHERE delivered_at IS NULL AND failed_at IS NOT NULL`,
+  },
   {
     name: 'runtime_scheduler_outbox_admission_pending_idx',
     table: 'outbox',
@@ -3297,6 +3470,48 @@ const POSTGRES_SCHEDULER_RUNTIME_ADMIN_INDEXES = [
     definition: `(created_at, event_id)
       WHERE delivered_at IS NULL AND failed_at IS NOT NULL`,
   },
+  {
+    name: 'runtime_scheduler_sandbox_cleanup_v2_ready_health_idx',
+    table: 'sandbox_cleanup_jobs_v2',
+    definition: `(available_at, created_at, provider, sandbox_id)
+      WHERE status <> 'deleted'
+        AND blocked_at IS NULL
+        AND lease_owner IS NULL
+        AND status <> 'deleting'`,
+  },
+  {
+    name: 'runtime_scheduler_sandbox_cleanup_v2_lease_health_idx',
+    table: 'sandbox_cleanup_jobs_v2',
+    definition: `(lease_expires_at, created_at, provider, sandbox_id)
+      WHERE status <> 'deleted' AND lease_owner IS NOT NULL`,
+  },
+  {
+    name: 'runtime_scheduler_sandbox_cleanup_v2_blocked_health_idx',
+    table: 'sandbox_cleanup_jobs_v2',
+    definition: `(blocked_at, lease_expires_at, created_at, provider, sandbox_id)
+      WHERE status <> 'deleted'
+        AND (blocked_at IS NOT NULL OR status = 'deleting')`,
+  },
+  {
+    name: 'runtime_scheduler_compute_billing_ready_health_idx',
+    table: 'compute_billing_jobs_v2',
+    definition: `(available_at, created_at, run_id)
+      WHERE status <> 'settled'
+        AND blocked_at IS NULL
+        AND lease_owner IS NULL`,
+  },
+  {
+    name: 'runtime_scheduler_compute_billing_lease_health_idx',
+    table: 'compute_billing_jobs_v2',
+    definition: `(lease_expires_at, created_at, run_id)
+      WHERE status <> 'settled' AND lease_owner IS NOT NULL`,
+  },
+  {
+    name: 'runtime_scheduler_compute_billing_blocked_health_idx',
+    table: 'compute_billing_jobs_v2',
+    definition: `(blocked_at, lease_expires_at, created_at, run_id)
+      WHERE status <> 'settled' AND blocked_at IS NOT NULL`,
+  },
   {
     name: 'runtime_scheduler_runs_terminal_finished_idx',
     table: 'runs',