deepline 0.3.105 → 0.3.107

package/dist/bundling-sources/sdk/src/release.ts

@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
   // getters keep their established compatibility behavior.
   // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
   // 0.3.90 is the first deliberately versioned SDK release for API v3.
-  version: '0.3.105',
+  version: '0.3.107',
   updateSummary:
     'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
   packageCapabilities: {

package/dist/bundling-sources/shared_libs/observability/queue-health.ts

@@ -0,0 +1,678 @@
+/** Scheduler-neutral queue/workset health contract. */
+
+import { measuredValue, type Measured } from './scheduled-work-v2';
+export type { Measured } from './scheduled-work-v2';
+export {
+  exact,
+  lowerBound,
+  measuredValue,
+  notApplicable,
+  unavailable,
+} from './scheduled-work-v2';
+
+export const QUEUE_HEALTH_CONTRACT_VERSION = 2 as const;
+export type QueueKind = 'fifo' | 'leased_workset' | 'cursor' | 'provider_queue';
+
+export type QueueDefinition = {
+  readonly id: string;
+  readonly owner: string;
+  readonly purpose: string;
+  readonly unit: string;
+  readonly kind: QueueKind;
+  readonly ageSlo: { readonly warnMs: number; readonly failMs: number };
+  /** How old a snapshot may be before it is no longer evidence of health. */
+  readonly staleAfterMs?: number;
+};
+
+export type QueueCatalogEntry = QueueDefinition & {
+  readonly name: string;
+  /** Human-readable policy rendered beside warning signals. */
+  readonly warningDefinition: string;
+  /** Human-readable policy rendered beside failure signals. */
+  readonly failureDefinition: string;
+  /** A dashboard must render every one of these fields, even when N/A. */
+  readonly requiredMetrics: readonly [
+    'depth.ready',
+    'depth.delayed',
+    'depth.inFlight',
+    'depth.blocked',
+    'depth.deadLettered',
+    'oldestAgeMs.ready',
+    'oldestAgeMs.delayed',
+    'oldestAgeMs.inFlight',
+    'oldestAgeMs.blocked',
+    'oldestAgeMs.deadLettered',
+    'flow.admitted',
+    'flow.started',
+    'flow.completed',
+    'flow.failed',
+    'flow.retried',
+    'flow.poisoned',
+  ];
+};
+
+const requiredQueueMetrics: QueueCatalogEntry['requiredMetrics'] = [
+  'depth.ready',
+  'depth.delayed',
+  'depth.inFlight',
+  'depth.blocked',
+  'depth.deadLettered',
+  'oldestAgeMs.ready',
+  'oldestAgeMs.delayed',
+  'oldestAgeMs.inFlight',
+  'oldestAgeMs.blocked',
+  'oldestAgeMs.deadLettered',
+  'flow.admitted',
+  'flow.started',
+  'flow.completed',
+  'flow.failed',
+  'flow.retried',
+  'flow.poisoned',
+];
+
+function queue(
+  id: string,
+  name: string,
+  kind: QueueKind,
+  owner: string,
+  purpose: string,
+  warnMs: number,
+  failMs: number,
+): QueueCatalogEntry {
+  return {
+    id,
+    name,
+    kind,
+    owner,
+    purpose,
+    unit: 'items',
+    ageSlo: { warnMs, failMs },
+    warningDefinition: `Oldest work is at least ${warnMs} ms old, or a bounded observer can only prove a lower bound.`,
+    failureDefinition: `Blocked or dead-lettered work exists, oldest work is at least ${failMs} ms old, or the observation is invalid or unavailable.`,
+    requiredMetrics: requiredQueueMetrics,
+  };
+}
+
+/**
+ * The durable work inventory used by the operations dashboard. This is a
+ * catalog, not a claim that every adapter already exists: an unobserved row is
+ * intentionally unhealthy until its Adapter publishes an observation.
+ */
+export const OPERATIONS_QUEUE_CATALOG = [
+  queue(
+    'play-runtime-tasks',
+    'Play runtime tasks',
+    'fifo',
+    'runtime',
+    'Run admitted Play tasks to terminal state',
+    60_000,
+    180_000,
+  ),
+  queue(
+    'run-ledger-projector-outbox',
+    'Run Ledger projector outbox',
+    'leased_workset',
+    'runtime',
+    'Deliver causal run events to control-plane projections',
+    60_000,
+    180_000,
+  ),
+  queue(
+    'play-webhook-receipts',
+    'Play webhook receipts',
+    'leased_workset',
+    'runtime',
+    'Admit authenticated webhook receipts exactly once',
+    300_000,
+    900_000,
+  ),
+  queue(
+    'workflow-cron-bindings',
+    'Workflow cron bindings',
+    'leased_workset',
+    'workflows',
+    'Turn due bindings into terminal workflow runs',
+    120_000,
+    3_600_000,
+  ),
+  queue(
+    'play-cron-bindings',
+    'Play cron bindings',
+    'leased_workset',
+    'workflows',
+    'Admit due Play triggers and resulting runs',
+    120_000,
+    300_000,
+  ),
+  queue(
+    'ingestion-jobs',
+    'Ingestion jobs',
+    'provider_queue',
+    'ingestion',
+    'Launch and complete tenant ingestion jobs',
+    300_000,
+    600_000,
+  ),
+  queue(
+    'monitor-fleet-convergence',
+    'Monitor fleet convergence',
+    'leased_workset',
+    'monitors',
+    'Converge desired monitor definitions to provider truth',
+    120_000,
+    300_000,
+  ),
+  queue(
+    'monitor-deployment-jobs',
+    'Monitor deployment jobs',
+    'leased_workset',
+    'monitors',
+    'Settle and verify monitor deployment jobs',
+    180_000,
+    600_000,
+  ),
+  queue(
+    'monitor-batch-operations',
+    'Monitor batch operations',
+    'cursor',
+    'monitors',
+    'Apply and terminalize provider batch operation pages',
+    600_000,
+    1_800_000,
+  ),
+  queue(
+    'provider-monitor-reconciliation',
+    'Provider-monitor reconciliation',
+    'cursor',
+    'monitors',
+    'Scan and align provider monitor inventory',
+    1_800_000,
+    3_600_000,
+  ),
+  queue(
+    'signal-radar-inbound-dlq',
+    'Signal-radar inbound / DLQ',
+    'provider_queue',
+    'monitors',
+    'Persist inbound provider events and isolate poison events',
+    300_000,
+    900_000,
+  ),
+  queue(
+    'charge-reconciliation-backlog',
+    'Charge reconciliation backlog',
+    'cursor',
+    'billing',
+    'Settle or release reservations from provider truth',
+    1_800_000,
+    7_200_000,
+  ),
+  queue(
+    'workspace-fast-settlement',
+    'Per-workspace fast settlement',
+    'fifo',
+    'billing',
+    'Complete isolated workspace settlements promptly',
+    30_000,
+    60_000,
+  ),
+  queue(
+    'metronome-usage-delivery',
+    'Metronome usage delivery',
+    'leased_workset',
+    'billing',
+    'Deliver commercial usage to Metronome projections',
+    120_000,
+    600_000,
+  ),
+  queue(
+    'billing-operations',
+    'Billing operations',
+    'leased_workset',
+    'billing',
+    'Reach authoritative applied or aborted outcomes',
+    300_000,
+    900_000,
+  ),
+  queue(
+    'stripe-financial-inbox',
+    'Stripe financial inbox',
+    'leased_workset',
+    'billing',
+    'Process and acknowledge Stripe financial events',
+    300_000,
+    900_000,
+  ),
+  queue(
+    'async-usage-recovery',
+    'Async usage recovery',
+    'leased_workset',
+    'billing',
+    'Reconcile provider jobs into settlement',
+    600_000,
+    3_600_000,
+  ),
+  queue(
+    'notification-deliveries',
+    'Notification deliveries',
+    'leased_workset',
+    'notifications',
+    'Send eligible product notifications',
+    120_000,
+    300_000,
+  ),
+  queue(
+    'usage-rollup-fold',
+    'Usage rollup fold',
+    'cursor',
+    'billing',
+    'Fold usage events into projections',
+    120_000,
+    300_000,
+  ),
+  queue(
+    'play-deployment-jobs',
+    'Play deployment jobs',
+    'leased_workset',
+    'runtime',
+    'Build and activate Play deployments',
+    300_000,
+    900_000,
+  ),
+  queue(
+    'github-runner-wait',
+    'GitHub runner wait',
+    'provider_queue',
+    'ci',
+    'Start and complete queued GitHub jobs',
+    300_000,
+    900_000,
+  ),
+  queue(
+    'sandbox-cleanup-jobs',
+    'Sandbox cleanup jobs',
+    'leased_workset',
+    'runtime',
+    'Complete durable sandbox deletion obligations',
+    300_000,
+    900_000,
+  ),
+] as const satisfies readonly QueueCatalogEntry[];
+
+export type OperationsQueueId = (typeof OPERATIONS_QUEUE_CATALOG)[number]['id'];
+
+export function validateQueueCatalog(
+  catalog: readonly QueueCatalogEntry[],
+): void {
+  const ids = new Set<string>();
+  for (const entry of catalog) {
+    validateQueueDefinition(entry);
+    if (ids.has(entry.id)) throw new Error(`duplicate queue id: ${entry.id}`);
+    ids.add(entry.id);
+    if (entry.requiredMetrics.length !== requiredQueueMetrics.length) {
+      throw new Error(`${entry.id}: required metric declaration is incomplete`);
+    }
+    if (!entry.warningDefinition.trim() || !entry.failureDefinition.trim()) {
+      throw new Error(
+        `${entry.id}: warning and failure definitions are required`,
+      );
+    }
+  }
+}
+
+export type QueueDepth = {
+  readonly ready: Measured<number>;
+  readonly delayed: Measured<number>;
+  readonly inFlight: Measured<number>;
+  readonly blocked: Measured<number>;
+  readonly deadLettered: Measured<number>;
+};
+
+export type QueueOldestAge = {
+  readonly ready: Measured<number>;
+  readonly delayed: Measured<number>;
+  readonly inFlight: Measured<number>;
+  readonly blocked: Measured<number>;
+  readonly deadLettered: Measured<number>;
+};
+
+export type QueueFlow = {
+  readonly admitted: Measured<number>;
+  readonly started: Measured<number>;
+  readonly completed: Measured<number>;
+  readonly failed: Measured<number>;
+  readonly retried: Measured<number>;
+  readonly poisoned: Measured<number>;
+};
+
+export type QueueObservation = {
+  readonly observedAt: number;
+  readonly depth: QueueDepth;
+  readonly oldestAgeMs: QueueOldestAge;
+  readonly flow: QueueFlow;
+  readonly adapter: string;
+};
+
+function combineMeasurements(
+  observations: readonly QueueObservation[],
+  select: (observation: QueueObservation) => Measured<number>,
+  mode: 'sum' | 'max',
+): Measured<number> {
+  const measurements = observations.map(select);
+  if (measurements.some((measurement) => measurement.kind === 'unavailable')) {
+    return {
+      kind: 'unavailable',
+      reason: 'At least one constituent queue measurement is unavailable.',
+    };
+  }
+  const values = measurements.flatMap((measurement) =>
+    measurement.kind === 'exact' || measurement.kind === 'lower_bound'
+      ? [measurement.value]
+      : [],
+  );
+  if (values.length === 0) {
+    return {
+      kind: 'not_applicable',
+      reason: 'No constituent queue exposes this measurement.',
+    };
+  }
+  const value =
+    mode === 'sum'
+      ? values.reduce((total, current) => total + current, 0)
+      : Math.max(...values);
+  return measurements.some((measurement) => measurement.kind === 'lower_bound')
+    ? { kind: 'lower_bound', value }
+    : { kind: 'exact', value };
+}
+
+/** Combine sharded/provider-native queues into one catalog queue contract. */
+export function combineQueueObservations(input: {
+  observations: readonly QueueObservation[];
+  observedAt: number;
+  adapter: string;
+}): QueueObservation {
+  if (input.observations.length === 0) {
+    throw new Error('At least one queue observation is required.');
+  }
+  const sum = (select: (observation: QueueObservation) => Measured<number>) =>
+    combineMeasurements(input.observations, select, 'sum');
+  const max = (select: (observation: QueueObservation) => Measured<number>) =>
+    combineMeasurements(input.observations, select, 'max');
+  return {
+    observedAt: input.observedAt,
+    adapter: input.adapter,
+    depth: {
+      ready: sum((row) => row.depth.ready),
+      delayed: sum((row) => row.depth.delayed),
+      inFlight: sum((row) => row.depth.inFlight),
+      blocked: sum((row) => row.depth.blocked),
+      deadLettered: sum((row) => row.depth.deadLettered),
+    },
+    oldestAgeMs: {
+      ready: max((row) => row.oldestAgeMs.ready),
+      delayed: max((row) => row.oldestAgeMs.delayed),
+      inFlight: max((row) => row.oldestAgeMs.inFlight),
+      blocked: max((row) => row.oldestAgeMs.blocked),
+      deadLettered: max((row) => row.oldestAgeMs.deadLettered),
+    },
+    flow: {
+      admitted: sum((row) => row.flow.admitted),
+      started: sum((row) => row.flow.started),
+      completed: sum((row) => row.flow.completed),
+      failed: sum((row) => row.flow.failed),
+      retried: sum((row) => row.flow.retried),
+      poisoned: sum((row) => row.flow.poisoned),
+    },
+  };
+}
+
+export type QueueHealthStatus = 'healthy' | 'warning' | 'failed' | 'unobserved';
+export type QueueHealthReason =
+  | 'empty'
+  | 'draining'
+  | 'age_warning'
+  | 'age_breached'
+  | 'blocked_work'
+  | 'dead_lettered_work'
+  | 'partial_measurements'
+  | 'stale_observation'
+  | 'missing_measurements'
+  | 'invalid_observation';
+
+export type QueueHealth = {
+  readonly status: QueueHealthStatus;
+  readonly reason: QueueHealthReason;
+  readonly worstAgeMs?: number;
+  readonly worstAgeRatio?: number;
+};
+
+export const QUEUE_HEALTH_TELEMETRY_EVENT = 'operations.queue.health' as const;
+export const QUEUE_HEALTH_TELEMETRY_SCHEMA_VERSION = 1 as const;
+
+export type QueueHealthSignal = {
+  readonly schemaVersion: typeof QUEUE_HEALTH_TELEMETRY_SCHEMA_VERSION;
+  readonly event: typeof QUEUE_HEALTH_TELEMETRY_EVENT;
+  readonly level: 'info' | 'warn' | 'error';
+  readonly signal: 'problem' | 'recovery';
+  readonly queueId: string;
+  readonly queueName: string;
+  readonly owner: string;
+  readonly purpose: string;
+  readonly kind: QueueKind;
+  readonly unit: string;
+  readonly environment: string;
+  readonly adapter: string;
+  readonly observedAt: number;
+  readonly health: QueueHealth;
+  readonly previousHealth?: QueueHealth;
+  readonly warningDefinition: string;
+  readonly failureDefinition: string;
+  readonly ageWarnMs: number;
+  readonly ageFailMs: number;
+  readonly depth: QueueDepth;
+  readonly oldestAgeMs: QueueOldestAge;
+  readonly flow: QueueFlow;
+};
+
+/**
+ * Project a durable observation into the one cross-runtime queue signal.
+ * Healthy steady state is deliberately silent. Actionable backlog problems
+ * repeat on every observation so a log-based monitor cannot resolve while the
+ * queue remains unhealthy; persistent adapter absence is emitted once because
+ * the dashboard already keeps that configuration state visible. The first
+ * healthy observation after a problem emits recovery.
+ */
+export function deriveQueueHealthSignal(input: {
+  definition: QueueCatalogEntry;
+  environment: string;
+  observation: QueueObservation;
+  previousObservation?: QueueObservation;
+  now?: number;
+}): QueueHealthSignal | null {
+  const now = input.now ?? input.observation.observedAt;
+  const health = deriveQueueHealth(input.definition, input.observation, now);
+  const previousHealth = input.previousObservation
+    ? deriveQueueHealth(input.definition, input.previousObservation, now)
+    : undefined;
+  const recovering =
+    health.status === 'healthy' &&
+    previousHealth !== undefined &&
+    previousHealth.status !== 'healthy';
+  if (health.status === 'healthy' && !recovering) return null;
+  // A permanently unavailable adapter is a catalog/configuration finding, not
+  // a one-minute incident. Emit it once; the dashboard keeps the state visible
+  // and a later observation with real evidence emits recovery.
+  if (
+    health.status === 'unobserved' &&
+    previousHealth?.status === 'unobserved' &&
+    previousHealth.reason === health.reason
+  ) {
+    return null;
+  }
+  return {
+    schemaVersion: QUEUE_HEALTH_TELEMETRY_SCHEMA_VERSION,
+    event: QUEUE_HEALTH_TELEMETRY_EVENT,
+    level:
+      health.status === 'healthy'
+        ? 'info'
+        : health.status === 'warning'
+          ? 'warn'
+          : 'error',
+    signal: recovering ? 'recovery' : 'problem',
+    queueId: input.definition.id,
+    queueName: input.definition.name,
+    owner: input.definition.owner,
+    purpose: input.definition.purpose,
+    kind: input.definition.kind,
+    unit: input.definition.unit,
+    environment: input.environment,
+    adapter: input.observation.adapter,
+    observedAt: input.observation.observedAt,
+    health,
+    ...(previousHealth === undefined ? {} : { previousHealth }),
+    warningDefinition: input.definition.warningDefinition,
+    failureDefinition: input.definition.failureDefinition,
+    ageWarnMs: input.definition.ageSlo.warnMs,
+    ageFailMs: input.definition.ageSlo.failMs,
+    depth: input.observation.depth,
+    oldestAgeMs: input.observation.oldestAgeMs,
+    flow: input.observation.flow,
+  };
+}
+
+export function validateQueueDefinition(definition: QueueDefinition): void {
+  for (const [name, value] of Object.entries(definition)) {
+    if (['ageSlo', 'staleAfterMs', 'requiredMetrics', 'name'].includes(name))
+      continue;
+    if (typeof value !== 'string' || value.trim() === '') {
+      throw new Error(
+        `queue ${definition.id || '<unknown>'}: ${name} is required`,
+      );
+    }
+  }
+  if (
+    !Number.isFinite(definition.ageSlo.warnMs) ||
+    !Number.isFinite(definition.ageSlo.failMs) ||
+    definition.ageSlo.warnMs <= 0 ||
+    definition.ageSlo.failMs <= definition.ageSlo.warnMs
+  )
+    throw new Error(
+      `${definition.id}: ageSlo must be positive and fail > warn`,
+    );
+  if (definition.staleAfterMs !== undefined && definition.staleAfterMs <= 0) {
+    throw new Error(`${definition.id}: staleAfterMs must be positive`);
+  }
+}
+
+export function validateQueueObservation(
+  definition: QueueDefinition,
+  observation: QueueObservation,
+): void {
+  if (!Number.isFinite(observation.observedAt) || observation.observedAt <= 0) {
+    throw new Error(`${definition.id}: observedAt is required`);
+  }
+  if (!observation.adapter.trim())
+    throw new Error(`${definition.id}: adapter is required`);
+  const measurements = [
+    ...Object.values(observation.depth),
+    ...Object.values(observation.oldestAgeMs),
+    ...Object.values(observation.flow),
+  ];
+  for (const measurement of measurements) {
+    if (
+      (measurement.kind === 'exact' || measurement.kind === 'lower_bound') &&
+      (!Number.isFinite(measurement.value) || measurement.value < 0)
+    )
+      throw new Error(
+        `${definition.id}: measured values must be finite and non-negative`,
+      );
+    if (
+      (measurement.kind === 'unavailable' ||
+        measurement.kind === 'not_applicable') &&
+      !measurement.reason.trim()
+    )
+      throw new Error(`${definition.id}: unavailable values require a reason`);
+  }
+}
+
+export function deriveQueueHealth(
+  definition: QueueDefinition,
+  observation: QueueObservation | undefined,
+  now = Date.now(),
+): QueueHealth {
+  if (!observation)
+    return { status: 'unobserved', reason: 'stale_observation' };
+  try {
+    validateQueueObservation(definition, observation);
+  } catch {
+    return { status: 'failed', reason: 'invalid_observation' };
+  }
+  const staleAfter = definition.staleAfterMs ?? definition.ageSlo.failMs;
+  if (now - observation.observedAt > staleAfter) {
+    return { status: 'unobserved', reason: 'stale_observation' };
+  }
+  const depths = Object.values(observation.depth)
+    .map(measuredValue)
+    .filter((value): value is number => value !== undefined);
+  const ages = Object.values(observation.oldestAgeMs)
+    .map(measuredValue)
+    .filter((value): value is number => value !== undefined);
+  if (depths.length === 0 && ages.length === 0) {
+    return { status: 'unobserved', reason: 'missing_measurements' };
+  }
+  const hasDepth = depths.some((value) => value > 0);
+  const deadLettered = measuredValue(observation.depth.deadLettered) ?? 0;
+  const blocked = measuredValue(observation.depth.blocked) ?? 0;
+  const worstAgeMs = ages.length > 0 ? Math.max(...ages) : undefined;
+  const ratio =
+    worstAgeMs === undefined
+      ? undefined
+      : worstAgeMs / definition.ageSlo.warnMs;
+  if (deadLettered > 0)
+    return {
+      status: 'failed',
+      reason: 'dead_lettered_work',
+      worstAgeMs,
+      worstAgeRatio: ratio,
+    };
+  if (blocked > 0)
+    return {
+      status: 'failed',
+      reason: 'blocked_work',
+      worstAgeMs,
+      worstAgeRatio: ratio,
+    };
+  if (worstAgeMs !== undefined && worstAgeMs >= definition.ageSlo.failMs) {
+    return {
+      status: 'failed',
+      reason: 'age_breached',
+      worstAgeMs,
+      worstAgeRatio: ratio,
+    };
+  }
+  if (worstAgeMs !== undefined && worstAgeMs >= definition.ageSlo.warnMs) {
+    return {
+      status: 'warning',
+      reason: 'age_warning',
+      worstAgeMs,
+      worstAgeRatio: ratio,
+    };
+  }
+  if (
+    [
+      ...Object.values(observation.depth),
+      ...Object.values(observation.oldestAgeMs),
+    ].some((measurement) => measurement.kind === 'lower_bound')
+  ) {
+    return {
+      status: 'warning',
+      reason: 'partial_measurements',
+      ...(worstAgeMs === undefined ? {} : { worstAgeMs, worstAgeRatio: ratio }),
+    };
+  }
+  return {
+    status: 'healthy',
+    reason: hasDepth ? 'draining' : 'empty',
+    ...(worstAgeMs === undefined ? {} : { worstAgeMs, worstAgeRatio: ratio }),
+  };
+}