deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-dead-letter.ts

@@ -0,0 +1,245 @@
+import { randomUUID } from 'node:crypto';
+import { isRepairableTenantDataPlaneError } from '@deepline/ingestion-v2/tenant-data-plane-errors';
+import type { SignalRadarInboundMessage } from '@deepline/monitors/signal-radars/inbound-message';
+
+// Minimal pg client seam shared with the ingestion stack: a `query(sql, args)`
+// that returns `{ rows }`. Deliberately structural so PGlite (tests) and the
+// Neon pooled client (prod) both satisfy it without adapters.
+export type InboundPgClient = {
+  query: (
+    sql: string,
+    args?: unknown[],
+  ) => Promise<{ rows?: unknown[]; rowCount?: number }>;
+};
+
+export type InboundFailureClass = 'transient' | 'permanent';
+
+function errorMessage(error: unknown): string {
+  if (error instanceof Error && error.message.trim()) return error.message;
+  if (typeof error === 'string' && error.trim()) return error;
+  return String(error ?? 'unknown error');
+}
+
+// A TRANSIENT failure is an infra outage the event can survive: keep retrying
+// via backoff so a long Neon/Convex flap drains on recovery instead of being
+// dead-lettered (which would drop a real customer signal). A PERMANENT failure
+// is a poison message — an unrecognized/invalid payload shape that will never
+// succeed no matter how many times we redeliver — so it must give up + record.
+//
+// The transient side reuses the SAME classification the rest of the ingestion
+// stack already treats as repairable (permission-denied / role drift / conn
+// resets), so the worker never disagrees with the route on "transient".
+export function classifyInboundFailure(error: unknown): InboundFailureClass {
+  if (isRepairableTenantDataPlaneError(error)) return 'transient';
+
+  const code =
+    error && typeof error === 'object'
+      ? String((error as { code?: unknown }).code ?? '')
+      : '';
+  const transientCodes = new Set([
+    'tenant_data_plane_repair_required',
+    'provider_monitor_convex_sync_pending',
+    // The worker emits this after capture-first receipt persistence cannot be
+    // completed. The event has not entered durable storage, so it must remain
+    // retryable even after the ordinary poison-message delivery budget.
+    'provider_callback_raw_capture_unavailable',
+    // A concurrent worker still owns the durable receipt. Acknowledging this
+    // redelivery as poison could dead-letter a live callback and clear the
+    // owner's receipt before its lease completes.
+    'provider_callback_raw_receipt_in_progress',
+    'ECONNRESET',
+    'ECONNREFUSED',
+    'ETIMEDOUT',
+    'EPIPE',
+    'ENOTFOUND',
+    'EAI_AGAIN',
+    'snitcher_ip2company_retryable',
+    '57P01', // admin shutdown
+    '57P02', // crash shutdown
+    '57P03', // cannot connect now
+    '08000', // connection exception
+    '08003', // connection does not exist
+    '08006', // connection failure
+    '53300', // too many connections
+  ]);
+  if (code && transientCodes.has(code)) return 'transient';
+
+  const message = errorMessage(error);
+  const transientPatterns = [
+    /connection (?:terminated|refused|reset|closed|failure)/i,
+    /could not connect to server/i,
+    /connection to server .* failed/i,
+    /the database system is (?:starting up|shutting down|in recovery)/i,
+    /timeout(?: exceeded)?(?: when trying)? to connect/i,
+    /connect(?:ion)? timeout/i,
+    /terminating connection due to/i,
+    /too many connections/i,
+    /ECONNRESET|ECONNREFUSED|ETIMEDOUT/i,
+    /convex.*(?:unavailable|timeout|network)/i,
+    /fetch failed/i,
+    /socket hang up/i,
+  ];
+  if (transientPatterns.some((pattern) => pattern.test(message))) {
+    return 'transient';
+  }
+
+  // Default to permanent: an unclassified failure is treated as poison so it is
+  // recorded for human review rather than retried forever. Transient outages are
+  // the ones we can positively identify (above); everything else is give-up-able.
+  return 'permanent';
+}
+
+type DeadLetterMessageInput = Pick<
+  SignalRadarInboundMessage,
+  'provider' | 'orgId' | 'token' | 'headers' | 'receivedAt'
+> & {
+  admission?: SignalRadarInboundMessage['admission'];
+  radarKey?: string | null;
+  eventIdempotencyKey?: string | null;
+  rawBody?: string;
+};
+
+async function markCapturedRawWriteFailed(
+  client: InboundPgClient,
+  eventIdempotencyKey: string,
+  lastError: string,
+) {
+  // The capture-first raw row (dl_meta.monitor_inbound_raw) may already exist for
+  // this event. Flip it to write_failed so on-call sees "captured, dead-lettered,
+  // replayable" in one place. Best-effort: a missing table/row must not block the
+  // dead-letter record itself.
+  try {
+    await client.query(
+      `UPDATE dl_meta.monitor_inbound_raw
+          SET status = 'write_failed', error = $2, updated_at = now()
+        WHERE event_idempotency_key = $1`,
+      [eventIdempotencyKey, lastError.slice(0, 4000)],
+    );
+  } catch (error) {
+    console.warn('[signal-radar.dead_letter.raw_mark_failed]', {
+      event_idempotency_key: eventIdempotencyKey,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+}
+
+// Idempotent dead-letter writer. Keyed on the provider event idempotency key via
+// the partial-unique index, so the SAME poison event redelivered N times yields
+// exactly ONE dead letter (attempts/last_error bumped), while two distinct
+// null-keyed poison events stay distinct (the partial index only enforces
+// uniqueness WHERE event_idempotency_key IS NOT NULL). Also marks the captured
+// raw row write_failed when one exists for the key.
+export async function recordSignalRadarDeadLetter(input: {
+  client: InboundPgClient;
+  message: DeadLetterMessageInput;
+  error: unknown;
+  attempts: number;
+}): Promise<void> {
+  const { client, message } = input;
+  const lastError = errorMessage(input.error);
+  const eventIdempotencyKey = message.eventIdempotencyKey ?? null;
+  const payload = {
+    provider: message.provider,
+    orgId: message.orgId,
+    token: message.token,
+    radarKey: message.radarKey ?? null,
+    eventIdempotencyKey,
+    rawBody: message.rawBody ?? null,
+    headers: message.headers ?? {},
+    receivedAt: message.receivedAt,
+    admission: message.admission ?? null,
+  };
+
+  if (eventIdempotencyKey) {
+    await client.query(
+      `INSERT INTO dl_meta.signal_radar_inbound_dead_letter
+         (dead_letter_id, provider, radar_key, org_id, event_idempotency_key,
+          payload, headers, attempts, last_error, status)
+       VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7::jsonb, $8, $9, 'open')
+       ON CONFLICT (event_idempotency_key)
+         WHERE event_idempotency_key IS NOT NULL
+         DO UPDATE SET
+           attempts = EXCLUDED.attempts,
+           last_error = EXCLUDED.last_error,
+           payload = EXCLUDED.payload,
+           headers = EXCLUDED.headers,
+           updated_at = now()`,
+      [
+        randomUUID(),
+        message.provider,
+        message.radarKey ?? null,
+        message.orgId ?? null,
+        eventIdempotencyKey,
+        JSON.stringify(payload),
+        JSON.stringify(message.headers ?? {}),
+        input.attempts,
+        lastError.slice(0, 8000),
+      ],
+    );
+    await markCapturedRawWriteFailed(client, eventIdempotencyKey, lastError);
+    return;
+  }
+
+  // No idempotency key -> the partial index does not apply, so each null-keyed
+  // poison event is its own dead letter (never merged).
+  await client.query(
+    `INSERT INTO dl_meta.signal_radar_inbound_dead_letter
+       (dead_letter_id, provider, radar_key, org_id, event_idempotency_key,
+        payload, headers, attempts, last_error, status)
+     VALUES ($1, $2, $3, $4, NULL, $5::jsonb, $6::jsonb, $7, $8, 'open')`,
+    [
+      randomUUID(),
+      message.provider,
+      message.radarKey ?? null,
+      message.orgId ?? null,
+      JSON.stringify(payload),
+      JSON.stringify(message.headers ?? {}),
+      input.attempts,
+      lastError.slice(0, 8000),
+    ],
+  );
+}
+
+// The load-bearing retry-callback decision (plan §4a). Runs on every handler
+// throw:
+//   - transient -> back off + redeliver, NEVER acknowledge, regardless of how
+//     far past maxDeliveries we are. A long Neon outage must drain on recovery.
+//   - permanent, still within maxDeliveries -> back off (give it its allotted
+//     retries; a "permanent" verdict can be a transient mislabel early on).
+//   - permanent, past maxDeliveries -> record the dead letter + acknowledge
+//     (stop retrying; it is now durably captured for human review + replay).
+export type InboundRetryDecision =
+  | { afterSeconds: number }
+  | { acknowledge: true };
+
+export function inboundBackoffSeconds(deliveryCount: number): number {
+  // Mirror the plan's bounded exponential backoff: 2^n * 5, capped at 300s.
+  const exponent = Math.max(1, deliveryCount);
+  return Math.min(300, 2 ** exponent * 5);
+}
+
+export async function resolveInboundRetryDecision(input: {
+  client: InboundPgClient;
+  message: DeadLetterMessageInput;
+  error: unknown;
+  deliveryCount: number;
+  maxDeliveries: number;
+}): Promise<InboundRetryDecision> {
+  const failureClass = classifyInboundFailure(input.error);
+
+  if (failureClass === 'transient') {
+    return { afterSeconds: inboundBackoffSeconds(input.deliveryCount) };
+  }
+
+  if (input.deliveryCount <= input.maxDeliveries) {
+    return { afterSeconds: inboundBackoffSeconds(input.deliveryCount) };
+  }
+
+  await recordSignalRadarDeadLetter({
+    client: input.client,
+    message: input.message,
+    error: input.error,
+    attempts: input.deliveryCount,
+  });
+  return { acknowledge: true };
+}

package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-message.ts

@@ -0,0 +1,38 @@
+// The durable message the thin Monitor inbound receiver enqueues onto the
+// Vercel Queues front-queue. It carries ONLY what the worker needs that is not
+// derivable without the provider's HTTP request — nothing Neon-derived. The
+// worker re-runs the full ingestion machinery from this envelope alone (resolve
+// the radar binding from Neon, capture-first raw, typed write, inline dispatch),
+// so the receiver never has to touch Neon before returning 200.
+
+export type SignalRadarInboundMessage = {
+  provider: string;
+  orgId: string;
+  token: string;
+  rawBody: string;
+  headers: Record<string, string>;
+  receivedAt: string;
+  admission: {
+    id: string;
+    bindingVersion: number;
+    /** A batch callback is resolved by its provider resource ID in Tier 2. */
+    routingMode?: 'single_monitor' | 'batch_by_upstream_resource';
+    verificationMode:
+      | 'hmac_signature'
+      | 'shared_secret_body'
+      | 'secret_header'
+      | 'callback_token_only';
+  };
+  // The Neon-free provider-event publish idempotency key, computed by the
+  // receiver from what is on the wire. Carried so the worker and dead-letter
+  // writer can key on the same grain the publish deduped on.
+  eventIdempotencyKey?: string | null;
+  // A preview monitor callback is received by Vercel but its run executes on
+  // an isolated scheduler lane. Preserve that request-scoped context across
+  // the durable queue boundary.
+  postgresSchedulerSchema?: string | null;
+  runtimeApiBaseUrl?: string | null;
+  // Optional radar hint the worker resolves durably from Neon; present only on
+  // dead-letter/replay envelopes where it was already known.
+  radarKey?: string | null;
+};

package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-ack-probe.ts

@@ -0,0 +1,94 @@
+import { getVercelOidcToken } from '@vercel/oidc';
+
+import { SIGNAL_RADAR_INBOUND_TOPIC } from '@deepline/monitors/signal-radars/inbound-queue';
+
+// Must match the function key in apps/deepline-api/vercel.json. Vercel Queue's push-trigger
+// adapter derives the consumer group from this path. Deriving rather than
+// copying the encoded value makes the transformation reviewable and testable.
+export const SIGNAL_RADAR_INBOUND_TRIGGER_PATH =
+  'src/app/api/queues/signal-radar-inbound/route.ts';
+
+const DEEPLINE_API_PROJECT_ROOT = 'apps/deepline-api/';
+
+export function vercelQueueConsumerGroupForFunctionPath(functionPath: string) {
+  return functionPath
+    .replaceAll('_', '__')
+    .replaceAll('/', '_S')
+    .replaceAll('.', '_D');
+}
+
+export const SIGNAL_RADAR_INBOUND_CONSUMER_GROUP =
+  vercelQueueConsumerGroupForFunctionPath(
+    SIGNAL_RADAR_INBOUND_TRIGGER_PATH.replace(DEEPLINE_API_PROJECT_ROOT, ''),
+  );
+
+export type SignalRadarInboundQueueMessageState =
+  | 'acknowledged'
+  | 'available'
+  | 'in_flight'
+  | 'not_found';
+
+type ProbeDependencies = {
+  fetch?: typeof fetch;
+  getOidcToken?: () => Promise<string>;
+};
+
+type ProbeInput = {
+  messageId: string;
+  region: string | undefined;
+  deploymentId: string | undefined;
+};
+
+function requiredRuntimeValue(name: string, value: string | undefined) {
+  if (value?.trim()) return value.trim();
+  throw new Error(
+    `${name} is required for the Vercel Queue acknowledgement probe.`,
+  );
+}
+
+/**
+ * Peeks at the exact message in Vercel's push-consumer group without leasing,
+ * acknowledging, or otherwise changing it. A 410 is the direct proof that
+ * the consumer successfully acknowledged this message.
+ */
+export async function signalRadarInboundQueueMessageState(
+  input: ProbeInput,
+  dependencies: ProbeDependencies = {},
+): Promise<SignalRadarInboundQueueMessageState> {
+  const region = requiredRuntimeValue('VERCEL_REGION', input.region);
+  const deploymentId = requiredRuntimeValue(
+    'VERCEL_DEPLOYMENT_ID',
+    input.deploymentId,
+  );
+  const requestFetch = dependencies.fetch ?? fetch;
+  const getToken = dependencies.getOidcToken ?? getVercelOidcToken;
+  const response = await requestFetch(
+    `https://${region}.vercel-queue.com/api/v3/topic/${encodeURIComponent(
+      SIGNAL_RADAR_INBOUND_TOPIC,
+    )}/consumer/${encodeURIComponent(
+      SIGNAL_RADAR_INBOUND_CONSUMER_GROUP,
+    )}/id/${encodeURIComponent(input.messageId)}`,
+    {
+      method: 'POST',
+      headers: {
+        authorization: `Bearer ${await getToken()}`,
+        accept: 'multipart/mixed',
+        // Vercel documents zero as a non-leasing peek. This probe must never
+        // become a competing consumer or acknowledge an unprocessed message.
+        'vqs-visibility-timeout-seconds': '0',
+        'vqs-deployment-id': deploymentId,
+      },
+    },
+  );
+
+  if (response.status === 410) return 'acknowledged';
+  if (response.status === 409) return 'in_flight';
+  if (response.status === 404) return 'not_found';
+  if (response.status === 200) return 'available';
+
+  throw new Error(
+    `Vercel Queue acknowledgement probe failed with HTTP ${response.status}: ${(
+      await response.text()
+    ).slice(0, 500)}`,
+  );
+}

package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue-delivery.ts

@@ -0,0 +1,60 @@
+import {
+  classifyInboundFailure,
+  inboundBackoffSeconds,
+} from '@deepline/monitors/signal-radars/inbound-dead-letter';
+import type { SignalRadarInboundMessage } from '@deepline/monitors/signal-radars/inbound-message';
+
+export class SignalRadarInboundDeliveryError extends Error {
+  readonly cause_error: unknown;
+
+  constructor(cause: unknown) {
+    super(cause instanceof Error ? cause.message : String(cause));
+    this.name = 'SignalRadarInboundDeliveryError';
+    this.cause_error = cause;
+  }
+}
+
+export async function handleSignalRadarInboundDelivery(input: {
+  message: SignalRadarInboundMessage;
+  metadata: { deliveryCount: number; messageId?: string };
+  maxDeliveries: number;
+  runWorker: (
+    message: SignalRadarInboundMessage,
+    metadata: { deliveryCount: number; messageId?: string },
+  ) => Promise<unknown>;
+  recordDeadLetter: (input: {
+    message: SignalRadarInboundMessage;
+    error: unknown;
+    deliveryCount: number;
+  }) => Promise<void>;
+}) {
+  try {
+    await input.runWorker(input.message, input.metadata);
+  } catch (error) {
+    if (
+      classifyInboundFailure(error) === 'permanent' &&
+      input.metadata.deliveryCount > input.maxDeliveries
+    ) {
+      try {
+        await input.recordDeadLetter({
+          message: input.message,
+          error,
+          deliveryCount: input.metadata.deliveryCount,
+        });
+        return;
+      } catch (recordError) {
+        throw new SignalRadarInboundDeliveryError(recordError);
+      }
+    }
+    throw new SignalRadarInboundDeliveryError(error);
+  }
+}
+
+export function signalRadarInboundRetry(input: {
+  error: unknown;
+  deliveryCount: number;
+}) {
+  // This function intentionally does no I/O. Vercel's retry callback is
+  // synchronous; terminal persistence belongs in the awaited handler above.
+  return { afterSeconds: inboundBackoffSeconds(input.deliveryCount) };
+}

package/dist/bundling-sources/shared_libs/monitors/signal-radars/inbound-queue.ts

@@ -0,0 +1,60 @@
+import { send as vercelQueueSend } from '@vercel/queue';
+import type { SignalRadarInboundMessage } from '@deepline/monitors/signal-radars/inbound-message';
+import { isTestOnlyRuntime } from '../../play-runtime/runtime-env';
+
+// Durable Vercel Queues front-queue for Monitor inbound webhook events.
+// The receiver produces onto this topic (auth -> enqueue -> 200); the air-gapped
+// worker at apps/deepline-api/src/app/api/queues/signal-radar-inbound/route.ts consumes it and runs
+// the full ingestion machinery. See docs/provider-monitor-playbook.md.
+export const SIGNAL_RADAR_INBOUND_TOPIC = 'signal-radar-inbound';
+
+// Retention set to the Vercel Queues maximum (7 days). This is the ceiling on
+// the worst Neon/Convex outage the buffer can survive: while downstream is down,
+// the receiver still returns 200 and the message sits here, redelivering until it
+// drains or expires. Sized to exceed any tolerable Neon outage window.
+export const SIGNAL_RADAR_INBOUND_RETENTION_SECONDS = 604_800;
+
+export type SignalRadarInboundSendOptions = {
+  idempotencyKey?: string;
+  /**
+   * Test-runtime delivery driver. The public receiver may set this only after
+   * proving the request is synthetic and test-authorized. It exercises the
+   * same queue worker locally, where Vercel Queues cannot mint its OIDC token.
+   */
+  inlineTestWorker?: boolean;
+  runInlineTestWorker?: (message: SignalRadarInboundMessage) => Promise<unknown>;
+};
+
+// Publish onto the inbound topic. `idempotencyKey` is the Neon-free provider
+// event key (belt-and-suspenders publish-side dedupe within the message TTL;
+// the real guarantee is the worker's consumer-side dedupe on the same grain).
+export async function send(
+  message: SignalRadarInboundMessage,
+  options?: SignalRadarInboundSendOptions,
+) {
+  if (options?.inlineTestWorker) {
+    if (!isTestOnlyRuntime()) {
+      throw new Error(
+        'The synthetic inbound queue driver is unavailable outside a test runtime.',
+      );
+    }
+    // The owning app supplies its real worker; packages cannot import app routes.
+    if (!options.runInlineTestWorker) {
+      throw new Error('The synthetic inbound queue driver requires an app worker.');
+    }
+    const result = await options.runInlineTestWorker(message);
+    return {
+      messageId: `test-inline:${options.idempotencyKey ?? crypto.randomUUID()}`,
+      // The test-only driver executes Tier 2 synchronously. Preserve its
+      // outcome for the synthetic receiver so a Gauntlet can assert a billing
+      // suspension without changing the real provider-facing queue contract.
+      result,
+    };
+  }
+  return vercelQueueSend(SIGNAL_RADAR_INBOUND_TOPIC, message, {
+    retentionSeconds: SIGNAL_RADAR_INBOUND_RETENTION_SECONDS,
+    ...(options?.idempotencyKey
+      ? { idempotencyKey: options.idempotencyKey }
+      : {}),
+  });
+}

package/dist/bundling-sources/shared_libs/monitors/snitcher-install-validation.ts

@@ -0,0 +1,93 @@
+import { lookup } from 'node:dns/promises';
+import { isIP } from 'node:net';
+import { DEEPLINE_ANALYTICS_BROWSER_OBJECT } from '@deepline/monitors/analytics-public';
+
+export type SnitcherInstallationCheck = {
+  domain: string;
+  status: 'installed' | 'not_found' | 'unreachable';
+  message: string;
+};
+
+export async function validateSnitcherInstallation(input: {
+  domains: string[];
+  trackingScriptId: string;
+}) {
+  return await Promise.all(
+    input.domains.map(async (domain): Promise<SnitcherInstallationCheck> => {
+      // Domains were validated and stored during setup. Never accept arbitrary URLs here.
+      if (isIP(domain) || domain === 'localhost')
+        return {
+          domain,
+          status: 'unreachable',
+          message: 'This domain cannot be checked.',
+        };
+      try {
+        const addresses = await lookup(domain, { all: true });
+        if (
+          !addresses.length ||
+          addresses.some(({ address }) => {
+            if (isIP(address) === 6) {
+              const lower = address.toLowerCase();
+              return (
+                lower === '::1' ||
+                lower.startsWith('fc') ||
+                lower.startsWith('fd') ||
+                lower.startsWith('fe80:')
+              );
+            }
+            const [a, b] = address.split('.').map(Number);
+            return (
+              a === 0 ||
+              a === 10 ||
+              a === 127 ||
+              (a === 169 && b === 254) ||
+              (a === 172 && b >= 16 && b <= 31) ||
+              (a === 192 && b === 168) ||
+              (a === 100 && b >= 64 && b <= 127)
+            );
+          })
+        )
+          return {
+            domain,
+            status: 'unreachable',
+            message: 'This domain cannot be checked.',
+          };
+        const response = await fetch(`https://${domain}`, {
+          redirect: 'error',
+          signal: AbortSignal.timeout(10_000),
+          headers: {
+            Accept: 'text/html',
+            'User-Agent': 'DeeplineTrackerValidator/1.0',
+          },
+        });
+        if (!response.ok)
+          return {
+            domain,
+            status: 'unreachable',
+            message: `The site returned HTTP ${response.status}.`,
+          };
+        const body = (await response.text()).slice(0, 500_000);
+        const installed =
+          body.includes(input.trackingScriptId) &&
+          body.includes(DEEPLINE_ANALYTICS_BROWSER_OBJECT);
+        return installed
+          ? {
+              domain,
+              status: 'installed',
+              message: 'Tracker found in the page source.',
+            }
+          : {
+              domain,
+              status: 'not_found',
+              message: 'Tracker was not found in the page source yet.',
+            };
+      } catch {
+        return {
+          domain,
+          status: 'unreachable',
+          message: 'Could not reach this site over HTTPS.',
+        };
+      }
+    }),
+  );
+}