deepline 0.3.141 → 0.3.142

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-queue-health.ts

@@ -8,6 +8,7 @@ import {
   type QueueDefinition,
   type QueueHealth,
   type QueueObservation,
+  type QueueRetirementCandidate,
 } from '../../observability/queue-health';
 import {
   dlqInventoryFromQueueObservation,
@@ -73,6 +74,8 @@ export type RuntimeQueueHealthProjection = {
   observation: QueueObservation;
   /** Shared DLQ contract consumed by operators and the admin CLI. */
   dlq: DlqInventory;
+  /** Internal-only bounded sample used to join operator dispositions. */
+  operatorRetirementCandidates?: QueueRetirementCandidate[];
   health: QueueHealth;
   depth: RuntimeQueueDepth;
   oldestAgeMs: RuntimeQueueOldestAge;
@@ -117,6 +120,8 @@ export type RuntimeQueueHealthOptions = {
   /** The caller could not enumerate every live lane within the bounded cap. */
   absurdQueueInventoryTruncated?: boolean;
   maxAbsurdQueues?: number;
+  /** Include bounded attention-row identities for an app-side disposition join. */
+  includeRetirementCandidates?: boolean;
 };
 
 export type RuntimeQueueHealthThresholds = Required<
@@ -151,6 +156,8 @@ type CountRow = {
   dead_lettered_available?: boolean;
   flow_available?: boolean;
   flow_unavailable_reason?: string | null;
+  blocked_retirement_candidates?: unknown;
+  dead_lettered_retirement_candidates?: unknown;
 };
 
 function quoteIdentifier(value: string, label: string): string {
@@ -184,6 +191,59 @@ function ageValue(value: string | number | null | undefined): number | null {
   return Math.floor(parsed);
 }
 
+function parseJsonArray(value: unknown): Record<string, unknown>[] {
+  let parsed = value;
+  if (typeof value === 'string') {
+    try {
+      parsed = JSON.parse(value) as unknown;
+    } catch {
+      return [];
+    }
+  }
+  return Array.isArray(parsed)
+    ? parsed.filter(
+        (row): row is Record<string, unknown> =>
+          typeof row === 'object' && row !== null && !Array.isArray(row),
+      )
+    : [];
+}
+
+function operatorRetirementCandidates(
+  row: CountRow,
+  kind: RuntimeQueueHealthProjection['kind'],
+): QueueRetirementCandidate[] {
+  const candidates: QueueRetirementCandidate[] = [];
+  const append = (
+    value: unknown,
+    bucket: QueueRetirementCandidate['bucket'],
+  ) => {
+    for (const candidate of parseJsonArray(value)) {
+      let itemId: string | undefined;
+      if (kind === 'sandbox_cleanup') {
+        const provider = candidate.provider;
+        const sandboxId = candidate.sandboxId;
+        const routingDomainKey = candidate.routingDomainKey;
+        if (
+          typeof provider === 'string' &&
+          typeof sandboxId === 'string' &&
+          typeof routingDomainKey === 'string'
+        ) {
+          itemId = encodeURIComponent(
+            JSON.stringify([provider, sandboxId, routingDomainKey]),
+          );
+        }
+      } else if (typeof candidate.itemId === 'string') {
+        itemId = candidate.itemId;
+      }
+      const ageMs = numberValue(candidate.ageMs as string | number | undefined);
+      if (itemId !== undefined) candidates.push({ itemId, bucket, ageMs });
+    }
+  };
+  append(row.blocked_retirement_candidates, 'blocked');
+  append(row.dead_lettered_retirement_candidates, 'deadLettered');
+  return candidates;
+}
+
 function optionsWithDefaults(
   input: RuntimeQueueHealthOptions = {},
 ): RuntimeQueueHealthThresholds {
@@ -265,6 +325,7 @@ function projectionFromRow(input: {
   observedAt: Date;
   environment?: string;
   options: RuntimeQueueHealthThresholds;
+  includeRetirementCandidates?: boolean;
 }): RuntimeQueueHealthProjection {
   const depth: RuntimeQueueDepth = {
     ready: numberValue(input.row.ready),
@@ -351,7 +412,9 @@ function projectionFromRow(input: {
       blocked: measureDepth(depth.blocked, input.row.blocked_truncated),
       deadLettered: deadLetteredAvailable
         ? measureDepth(depth.deadLettered, input.row.dead_lettered_truncated)
-        : notApplicable<number>('This workset has blocked quarantine, not a DLQ.'),
+        : notApplicable<number>(
+            'This workset has blocked quarantine, not a DLQ.',
+          ),
     },
     oldestAgeMs: {
       ready: measureAge(oldestAgeMs.ready, input.row.ready_truncated),
@@ -363,7 +426,9 @@ function projectionFromRow(input: {
             oldestAgeMs.deadLettered,
             input.row.dead_lettered_truncated,
           )
-        : notApplicable<number>('This workset has blocked quarantine, not a DLQ.'),
+        : notApplicable<number>(
+            'This workset has blocked quarantine, not a DLQ.',
+          ),
     },
     flow: {
       admitted: flowAvailable
@@ -410,6 +475,14 @@ function projectionFromRow(input: {
     kind: input.kind,
     source: input.source,
     observedAt: input.observedAt.toISOString(),
+    ...(input.includeRetirementCandidates
+      ? {
+          operatorRetirementCandidates: operatorRetirementCandidates(
+            input.row,
+            input.kind,
+          ),
+        }
+      : {}),
     status: derived.status,
     statusReason: derived.statusReason,
     observation,
@@ -438,36 +511,51 @@ function intervalExpr(column: string, nowParameter = '$1'): string {
   return `GREATEST(0, floor(extract(epoch FROM (${nowParameter}::timestamptz - ${column})) * 1000))`;
 }
 
-function absurdProjectionQuery(queueTable: string): string {
+function absurdProjectionQuery(
+  queueTable: string,
+  includeRetirementCandidates: boolean,
+  candidateSampleLimit: number,
+): string {
   const table = `absurd.${quoteIdentifier(queueTable, 'queue table')}`;
+  const retirementColumns = includeRetirementCandidates
+    ? `,
+      COALESCE((
+        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}))
+        FROM (SELECT item_id, age_at FROM sampled WHERE bucket = 'blocked' ORDER BY age_at ASC, item_id ASC LIMIT $4::int) candidates
+      ), '[]'::json) AS blocked_retirement_candidates,
+      COALESCE((
+        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}))
+        FROM (SELECT item_id, age_at FROM sampled WHERE bucket = 'dead_lettered' ORDER BY age_at ASC, item_id ASC LIMIT $4::int) candidates
+      ), '[]'::json) AS dead_lettered_retirement_candidates`
+    : '';
   return `
     WITH sampled AS (
-      (SELECT 'ready'::text AS bucket, available_at AS age_at
+      (SELECT 'ready'::text AS bucket, run_id::text AS item_id, available_at AS age_at
       FROM ${table}
       WHERE state = 'pending' AND available_at <= $1::timestamptz
       ORDER BY available_at, run_id
       LIMIT $2::int)
       UNION ALL
-      (SELECT 'delayed'::text, available_at
+      (SELECT 'delayed'::text, run_id::text, available_at
       FROM ${table}
       WHERE state = 'sleeping'
       ORDER BY available_at, run_id
       LIMIT $2::int)
       UNION ALL
-      (SELECT 'delayed'::text, available_at
+      (SELECT 'delayed'::text, run_id::text, available_at
       FROM ${table}
       WHERE state = 'pending' AND available_at > $1::timestamptz
       ORDER BY available_at, run_id
       LIMIT $2::int)
       UNION ALL
-      (SELECT 'in_flight'::text, coalesce(started_at, created_at)
+      (SELECT 'in_flight'::text, run_id::text, coalesce(started_at, created_at)
       FROM ${table}
       WHERE state = 'running'
         AND (claim_expires_at IS NULL OR claim_expires_at > $1::timestamptz)
       ORDER BY available_at, run_id
       LIMIT $2::int)
       UNION ALL
-      (SELECT 'blocked'::text, coalesce(claim_expires_at, started_at, created_at)
+      (SELECT 'blocked'::text, run_id::text, coalesce(claim_expires_at, started_at, created_at)
       FROM ${table}
       WHERE state = 'running'
         AND claim_expires_at IS NOT NULL
@@ -475,7 +563,7 @@ function absurdProjectionQuery(queueTable: string): string {
       ORDER BY claim_expires_at, run_id
       LIMIT $2::int)
       UNION ALL
-      (SELECT 'dead_lettered'::text, coalesce(failed_at, created_at)
+      (SELECT 'dead_lettered'::text, run_id::text, coalesce(failed_at, created_at)
       FROM ${table}
       WHERE state = 'failed'
       ORDER BY available_at, run_id
@@ -496,7 +584,7 @@ function absurdProjectionQuery(queueTable: string): string {
       count(*) FILTER (WHERE bucket = 'delayed') > $3 AS delayed_truncated,
       count(*) FILTER (WHERE bucket = 'in_flight') > $3 AS in_flight_truncated,
       count(*) FILTER (WHERE bucket = 'blocked') > $3 AS blocked_truncated,
-      count(*) FILTER (WHERE bucket = 'dead_lettered') > $3 AS dead_lettered_truncated,
+      count(*) FILTER (WHERE bucket = 'dead_lettered') > $3 AS dead_lettered_truncated${retirementColumns},
       0::bigint AS admitted,
       0::bigint AS started,
       0::bigint AS completed,
@@ -509,8 +597,22 @@ function absurdProjectionQuery(queueTable: string): string {
   `;
 }
 
-function outboxProjectionQuery(schema: string): string {
+function outboxProjectionQuery(
+  schema: string,
+  includeRetirementCandidates: boolean,
+): string {
   const table = `${schema}."outbox"`;
+  const retirementColumns = includeRetirementCandidates
+    ? `,
+      COALESCE((
+        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}))
+        FROM (SELECT item_id, age_at FROM blocked ORDER BY age_at ASC, item_id ASC LIMIT $3::int) candidates
+      ), '[]'::json) AS blocked_retirement_candidates,
+      COALESCE((
+        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}))
+        FROM (SELECT item_id, age_at FROM dead_lettered ORDER BY age_at ASC, item_id ASC LIMIT $3::int) candidates
+      ), '[]'::json) AS dead_lettered_retirement_candidates`
+    : '';
   return `
     WITH ready AS (
       SELECT coalesce(next_attempt_at, created_at) AS age_at
@@ -540,7 +642,7 @@ function outboxProjectionQuery(schema: string): string {
       ORDER BY lease_expires_at, created_at, event_id
       LIMIT $2::int
     ), blocked AS (
-      SELECT coalesce(lease_expires_at, created_at) AS age_at
+      SELECT event_id::text AS item_id, coalesce(lease_expires_at, created_at) AS age_at
       FROM ${table}
       WHERE delivered_at IS NULL
         AND failed_at IS NULL
@@ -549,7 +651,7 @@ function outboxProjectionQuery(schema: string): string {
       ORDER BY coalesce(lease_expires_at, created_at), created_at, event_id
       LIMIT $2::int
     ), dead_lettered AS (
-      SELECT coalesce(failed_at, created_at) AS age_at
+      SELECT event_id::text AS item_id, coalesce(failed_at, created_at) AS age_at
       FROM ${table}
       WHERE delivered_at IS NULL AND failed_at IS NOT NULL
       ORDER BY failed_at, event_id
@@ -570,7 +672,7 @@ function outboxProjectionQuery(schema: string): string {
       (SELECT count(*) FROM delayed) > $3 AS delayed_truncated,
       (SELECT count(*) FROM in_flight) > $3 AS in_flight_truncated,
       (SELECT count(*) FROM blocked) > $3 AS blocked_truncated,
-      (SELECT count(*) FROM dead_lettered) > $3 AS dead_lettered_truncated,
+      (SELECT count(*) FROM dead_lettered) > $3 AS dead_lettered_truncated${retirementColumns},
       0::bigint AS admitted,
       0::bigint AS started,
       0::bigint AS completed,
@@ -582,8 +684,19 @@ function outboxProjectionQuery(schema: string): string {
   `;
 }
 
-function cleanupProjectionQuery(schema: string): string {
+function cleanupProjectionQuery(
+  schema: string,
+  includeRetirementCandidates: boolean,
+): string {
   const table = `${schema}."sandbox_cleanup_jobs_v2"`;
+  const retirementColumns = includeRetirementCandidates
+    ? `,
+      COALESCE((
+        SELECT json_agg(json_build_object('provider', provider, 'sandboxId', sandbox_id, 'routingDomainKey', routing_domain_key, 'ageMs', ${intervalExpr('age_at')}))
+        FROM (SELECT provider, sandbox_id, routing_domain_key, created_at, age_at FROM blocked ORDER BY age_at ASC, created_at ASC, provider ASC, sandbox_id ASC LIMIT $3::int) candidates
+      ), '[]'::json) AS blocked_retirement_candidates,
+      '[]'::json AS dead_lettered_retirement_candidates`
+    : '';
   return `
     WITH ready AS (
       SELECT created_at AS age_at
@@ -616,6 +729,10 @@ function cleanupProjectionQuery(schema: string): string {
       LIMIT $2::int
     ), blocked AS (
       SELECT
+        provider,
+        sandbox_id,
+        routing_domain_key,
+        created_at,
         CASE
           WHEN blocked_at IS NOT NULL THEN blocked_at
           ELSE coalesce(lease_expires_at, created_at)
@@ -645,7 +762,7 @@ function cleanupProjectionQuery(schema: string): string {
       (SELECT count(*) FROM delayed) > $3 AS delayed_truncated,
       (SELECT count(*) FROM in_flight) > $3 AS in_flight_truncated,
       (SELECT count(*) FROM blocked) > $3 AS blocked_truncated,
-      false AS dead_lettered_truncated,
+      false AS dead_lettered_truncated${retirementColumns},
       0::bigint AS admitted,
       0::bigint AS started,
       0::bigint AS completed,
@@ -657,8 +774,19 @@ function cleanupProjectionQuery(schema: string): string {
   `;
 }
 
-function computeBillingProjectionQuery(schema: string): string {
+function computeBillingProjectionQuery(
+  schema: string,
+  includeRetirementCandidates: boolean,
+): string {
   const table = `${schema}."compute_billing_jobs_v2"`;
+  const retirementColumns = includeRetirementCandidates
+    ? `,
+      COALESCE((
+        SELECT json_agg(json_build_object('itemId', item_id, 'ageMs', ${intervalExpr('age_at')}))
+        FROM (SELECT item_id, age_at FROM blocked ORDER BY age_at ASC, item_id ASC LIMIT $3::int) candidates
+      ), '[]'::json) AS blocked_retirement_candidates,
+      '[]'::json AS dead_lettered_retirement_candidates`
+    : '';
   return `
     WITH ready AS (
       SELECT created_at AS age_at
@@ -688,7 +816,7 @@ function computeBillingProjectionQuery(schema: string): string {
       ORDER BY lease_expires_at, created_at, run_id
       LIMIT $2::int
     ), blocked AS (
-      SELECT coalesce(blocked_at, lease_expires_at, created_at) AS age_at
+      SELECT run_id::text AS item_id, coalesce(blocked_at, lease_expires_at, created_at) AS age_at
       FROM ${table}
       WHERE status <> 'settled'
         AND (
@@ -713,7 +841,7 @@ function computeBillingProjectionQuery(schema: string): string {
       (SELECT count(*) FROM delayed) > $3 AS delayed_truncated,
       (SELECT count(*) FROM in_flight) > $3 AS in_flight_truncated,
       (SELECT count(*) FROM blocked) > $3 AS blocked_truncated,
-      false AS dead_lettered_truncated,
+      false AS dead_lettered_truncated${retirementColumns},
       0::bigint AS admitted,
       0::bigint AS started,
       0::bigint AS completed,
@@ -796,6 +924,10 @@ export async function readRuntimeQueueHealthFromClient(
   const queueNames = registry.rows
     .slice(0, maxQueues)
     .map((row) => row.queue_name);
+  const candidateSampleLimit = Math.max(
+    1,
+    Math.floor(sampleLimit / Math.max(queueNames.length, 1)),
+  );
   if (requestedNames) {
     const found = new Set(queueNames);
     const missing = requestedNames.filter((name) => !found.has(name));
@@ -807,9 +939,17 @@ export async function readRuntimeQueueHealthFromClient(
   }
   const projections: RuntimeQueueHealthProjection[] = [];
   for (const queueName of queueNames) {
+    const includeRetirementCandidates =
+      input.includeRetirementCandidates === true;
     const result = await client.query<CountRow>(
-      absurdProjectionQuery(`r_${queueName}`),
-      [now, sampleLimit + 1, sampleLimit],
+      absurdProjectionQuery(
+        `r_${queueName}`,
+        includeRetirementCandidates,
+        candidateSampleLimit,
+      ),
+      includeRetirementCandidates
+        ? [now, sampleLimit + 1, sampleLimit, candidateSampleLimit]
+        : [now, sampleLimit + 1, sampleLimit],
     );
     projections.push(
       projectionFromRow({
@@ -820,25 +960,29 @@ export async function readRuntimeQueueHealthFromClient(
         observedAt: now,
         environment: input.environment,
         options,
+        includeRetirementCandidates: input.includeRetirementCandidates,
       }),
     );
   }
   const [outbox, computeBilling, cleanup] = await Promise.all([
-    client.query<CountRow>(outboxProjectionQuery(schema), [
-      now,
-      sampleLimit + 1,
-      sampleLimit,
-    ]),
-    client.query<CountRow>(computeBillingProjectionQuery(schema), [
-      now,
-      sampleLimit + 1,
-      sampleLimit,
-    ]),
-    client.query<CountRow>(cleanupProjectionQuery(schema), [
-      now,
-      sampleLimit + 1,
-      sampleLimit,
-    ]),
+    client.query<CountRow>(
+      outboxProjectionQuery(schema, input.includeRetirementCandidates === true),
+      [now, sampleLimit + 1, sampleLimit],
+    ),
+    client.query<CountRow>(
+      computeBillingProjectionQuery(
+        schema,
+        input.includeRetirementCandidates === true,
+      ),
+      [now, sampleLimit + 1, sampleLimit],
+    ),
+    client.query<CountRow>(
+      cleanupProjectionQuery(
+        schema,
+        input.includeRetirementCandidates === true,
+      ),
+      [now, sampleLimit + 1, sampleLimit],
+    ),
   ]);
   projections.push(
     projectionFromRow({
@@ -849,6 +993,7 @@ export async function readRuntimeQueueHealthFromClient(
       observedAt: now,
       environment: input.environment,
       options,
+      includeRetirementCandidates: input.includeRetirementCandidates,
     }),
     projectionFromRow({
       id: 'runtime:compute-billing-jobs-v2',
@@ -858,6 +1003,7 @@ export async function readRuntimeQueueHealthFromClient(
       observedAt: now,
       environment: input.environment,
       options,
+      includeRetirementCandidates: input.includeRetirementCandidates,
     }),
     projectionFromRow({
       id: 'runtime:sandbox-cleanup-jobs-v2',
@@ -867,6 +1013,7 @@ export async function readRuntimeQueueHealthFromClient(
       observedAt: now,
       environment: input.environment,
       options,
+      includeRetirementCandidates: input.includeRetirementCandidates,
     }),
   );
   const summary = {

package/dist/bundling-sources/shared_libs/product-notifications/events.ts

@@ -11,6 +11,7 @@ export const PRODUCT_NOTIFICATION_EVENT_SCHEMA_VERSION = 1 as const;
 export const PRODUCT_NOTIFICATION_MAX_EVENT_BYTES = 16_000;
 export const PRODUCT_NOTIFICATION_MAX_EVENT_FACTS = 20;
 export const PRODUCT_NOTIFICATION_MAX_EVENT_AGE_MS = 30 * 24 * 60 * 60 * 1_000;
+export const PRODUCT_NOTIFICATION_MAX_IDENTIFIER_LENGTH = 512;
 export const PRODUCT_NOTIFICATION_MAX_SOURCE_CLOCK_SKEW_MS = 5 * 60 * 1_000;
 
 export const ProductNotificationEvent = {
@@ -255,7 +256,9 @@ function requiredString(value: unknown, field: string) {
   if (typeof value !== 'string' || !value.trim()) {
     throw new Error(`${field} must be a non-empty string.`);
   }
-  if (value.length > 512) throw new Error(`${field} is too long.`);
+  if (value.length > PRODUCT_NOTIFICATION_MAX_IDENTIFIER_LENGTH) {
+    throw new Error(`${field} is too long.`);
+  }
 }
 
 function requireInternalTargetPath(value: unknown) {