deepline 0.1.91 → 0.1.94

package/dist/repo/sdk/src/runs/observe-transport.ts

@@ -0,0 +1,481 @@
+import { DeeplineError } from '../errors.js';
+import {
+  buildPlayRunStatusSnapshot,
+  diffPlayRunStreamEvents,
+  isTerminalPlayRunLiveStatus,
+  resolvePlayRunLogGap,
+  EMPTY_PLAY_RUN_STREAM_DIFF_STATE,
+  type LedgerBackedRunLike,
+  type PlayRunStreamDiffState,
+  type PlayRunStreamEvent,
+} from '../../../shared_libs/play-runtime/run-snapshot-stream.js';
+
+/**
+ * Run Observe Transport (ADR-0008).
+ *
+ * Watches one Play Run by subscribing directly to the Convex Run Snapshot:
+ *
+ *   1. `POST /api/v2/runs/:runId/observe-grant` mints a short-lived Run
+ *      Observe Grant JWT (the Deepline app stays the only auth authority).
+ *   2. A lazily-imported `convex/browser` ConvexClient authenticates with the
+ *      grant and subscribes to `runObservers.getPlayRunSnapshotForObserver`.
+ *   3. Each snapshot update is run through the shared snapshot differ, so the
+ *      emitted `play.*` events are identical to the legacy SSE stream.
+ *
+ * Steady-state observation therefore holds no Vercel function open and never
+ * consults the coordinator. The legacy SSE reconnect loop remains only as the
+ * support-window fallback: callers catch
+ * {@link RunObserveTransportUnavailableError} (grant endpoint missing on an
+ * older/unconfigured server, or Convex unreachable) and fall back with a
+ * single visible notice.
+ */
+
+/** Grant response from POST /api/v2/runs/:runId/observe-grant. */
+export type RunObserveGrantResponse = {
+  convexUrl: string;
+  token: string;
+  expiresAt: number;
+};
+
+/**
+ * Bootstrap failure: this server/deployment cannot serve the subscription
+ * transport. Callers fall back to the support-window SSE stream (ADR-0008).
+ */
+export class RunObserveTransportUnavailableError extends Error {
+  constructor(
+    message: string,
+    public readonly reason: string,
+  ) {
+    super(message);
+    this.name = 'RunObserveTransportUnavailableError';
+  }
+}
+
+type ObserveHttp = {
+  post<T = unknown>(path: string, body: unknown): Promise<T>;
+};
+
+export type ObserveRunEventsOptions = {
+  http: ObserveHttp;
+  runId: string;
+  signal?: AbortSignal;
+  /**
+   * Display-only notices (human mode): websocket reconnecting >10s, or a
+   * non-terminal run with no snapshot update for >120s.
+   */
+  onNotice?: (message: string) => void;
+};
+
+/** How long to wait for the first snapshot before declaring Convex unreachable. */
+const OBSERVE_BOOTSTRAP_TIMEOUT_MS = 10_000;
+/** Surface a "reconnecting" notice when the websocket is down this long. */
+const OBSERVE_RECONNECT_NOTICE_MS = 10_000;
+/** Display-only staleness warning for non-terminal runs. */
+const OBSERVE_STALE_WARNING_MS = 120_000;
+const OBSERVE_WATCHDOG_TICK_MS = 5_000;
+/** Re-mint the grant slightly before expiry. */
+const GRANT_REFRESH_MARGIN_MS = 5 * 60_000;
+const BACKFILL_PAGE_LIMIT = 1000;
+const BACKFILL_MAX_PAGES = 30;
+
+const OBSERVER_SNAPSHOT_QUERY = 'runObservers:getPlayRunSnapshotForObserver';
+const OBSERVER_LOG_PAGE_QUERY = 'runObservers:getRunLogPageForObserver';
+
+function errorText(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+async function mintRunObserveGrant(
+  http: ObserveHttp,
+  runId: string,
+): Promise<RunObserveGrantResponse> {
+  let response: unknown;
+  try {
+    response = await http.post(
+      `/api/v2/runs/${encodeURIComponent(runId)}/observe-grant`,
+      {},
+    );
+  } catch (error) {
+    if (error instanceof DeeplineError) {
+      // 401/403 are real auth failures on a server that HAS the endpoint —
+      // surface them loudly. 404/405 mean an older server without the grant
+      // route; 5xx/503 means the deployment cannot mint grants. Both of those
+      // are bootstrap failures handled by the dated SSE fallback (ADR-0008).
+      if (error.statusCode === 401 || error.statusCode === 403) {
+        throw error;
+      }
+      throw new RunObserveTransportUnavailableError(
+        `observe-grant endpoint unavailable (${error.statusCode ?? 'network'}): ${error.message}`,
+        'grant_endpoint_unavailable',
+      );
+    }
+    throw new RunObserveTransportUnavailableError(
+      `observe-grant request failed: ${errorText(error)}`,
+      'grant_request_failed',
+    );
+  }
+  const grant = response as Partial<RunObserveGrantResponse> | null;
+  if (
+    !grant ||
+    typeof grant.convexUrl !== 'string' ||
+    !grant.convexUrl.trim() ||
+    typeof grant.token !== 'string' ||
+    !grant.token.trim() ||
+    typeof grant.expiresAt !== 'number'
+  ) {
+    throw new RunObserveTransportUnavailableError(
+      'observe-grant endpoint returned an invalid grant payload.',
+      'grant_payload_invalid',
+    );
+  }
+  return grant as RunObserveGrantResponse;
+}
+
+type ObserverRunDoc = LedgerBackedRunLike & { status: string };
+
+type ObserverRunLogPage = {
+  runId: string;
+  lastStoredSeq: number;
+  entries: Array<{ seq: number; line: string }>;
+} | null;
+
+type QueueItem =
+  | { kind: 'run'; run: ObserverRunDoc | null }
+  | { kind: 'error'; error: unknown };
+
+/**
+ * Log-gap backfill from the durable Run Log Stream (ADR-0009). Log line
+ * bodies live in `playRunLogChunks` with exact absolute sequence numbers
+ * (they are no longer persisted to `playRunEvents`), so the grant-scoped
+ * `getRunLogPageForObserver` query returns precisely the missing seq range.
+ * Returns null when the page read fails or the range is not fully stored —
+ * the differ's loud `[stream] N log lines not retained` marker remains the
+ * truth in that case.
+ */
+async function backfillLogGap(input: {
+  queryLogPage: (
+    afterSeq: number,
+    limit: number,
+  ) => Promise<ObserverRunLogPage>;
+  lastLogSeq: number;
+  tailFirstSeq: number;
+}): Promise<string[] | null> {
+  const lines: string[] = [];
+  let cursor = input.lastLogSeq;
+  for (let page = 0; page < BACKFILL_MAX_PAGES; page += 1) {
+    if (cursor >= input.tailFirstSeq - 1) {
+      break;
+    }
+    let logPage: ObserverRunLogPage;
+    try {
+      logPage = await input.queryLogPage(
+        cursor,
+        Math.min(BACKFILL_PAGE_LIMIT, input.tailFirstSeq - 1 - cursor),
+      );
+    } catch {
+      return null;
+    }
+    const entries = (logPage?.entries ?? []).filter(
+      (entry) => entry.seq > cursor && entry.seq < input.tailFirstSeq,
+    );
+    if (entries.length === 0) {
+      break;
+    }
+    for (const entry of entries) {
+      if (entry.seq !== cursor + 1) {
+        // Stored seqs must be dense for the emitted payload's `firstSeq` to
+        // describe a contiguous run; a hole (e.g. a legacy run whose chunk
+        // storage starts mid-run) falls back to the differ's loud marker.
+        return null;
+      }
+      lines.push(entry.line);
+      cursor = entry.seq;
+    }
+  }
+  if (cursor < input.tailFirstSeq - 1) {
+    return null;
+  }
+  return lines;
+}
+
+/**
+ * Subscribe to one run's snapshot and yield the same `play.*` live events the
+ * SSE stream emits, ending after the terminal snapshot has been emitted.
+ *
+ * Failure contract:
+ * - bootstrap failure → {@link RunObserveTransportUnavailableError} (caller
+ *   falls back to SSE with one notice)
+ * - snapshot `null` → loud `RUN_NOT_FOUND`
+ * - grant rejected after one re-mint → loud failure
+ * - websocket drops mid-run → Convex client auto-reconnects; a display-only
+ *   "reconnecting" notice surfaces after 10s
+ */
+export async function* observeRunEvents(
+  options: ObserveRunEventsOptions,
+): AsyncGenerator<PlayRunStreamEvent> {
+  const { http, runId } = options;
+  let grant = await mintRunObserveGrant(http, runId);
+
+  let convexBrowser: typeof import('convex/browser');
+  let convexServer: typeof import('convex/server');
+  try {
+    // Lazy: non-watch commands never load the Convex client.
+    convexBrowser = await import('convex/browser');
+    convexServer = await import('convex/server');
+  } catch (error) {
+    throw new RunObserveTransportUnavailableError(
+      `convex client module unavailable: ${errorText(error)}`,
+      'convex_module_unavailable',
+    );
+  }
+
+  let webSocketConstructor: typeof WebSocket | undefined;
+  if (typeof WebSocket === 'undefined') {
+    // Node 18/20 have no global WebSocket; use the ws shim.
+    try {
+      // 'ws' lives in sdk/package.json (not the repo root, which typechecks
+      // this file without sdk's node_modules) — resolve dynamically so the
+      // root tsc does not need @types/ws.
+      const wsModuleName = 'ws';
+      const ws = (await import(wsModuleName)) as { default: unknown };
+      webSocketConstructor = ws.default as typeof WebSocket;
+    } catch (error) {
+      throw new RunObserveTransportUnavailableError(
+        `no WebSocket implementation available: ${errorText(error)}`,
+        'websocket_unavailable',
+      );
+    }
+  }
+
+  const snapshotQuery = convexServer.makeFunctionReference<'query'>(
+    OBSERVER_SNAPSHOT_QUERY,
+  );
+  const logPageQuery = convexServer.makeFunctionReference<'query'>(
+    OBSERVER_LOG_PAGE_QUERY,
+  );
+
+  const client = new convexBrowser.ConvexClient(grant.convexUrl, {
+    ...(webSocketConstructor ? { webSocketConstructor } : {}),
+    unsavedChangesWarning: false,
+  });
+
+  const queue: QueueItem[] = [];
+  let wake: (() => void) | null = null;
+  const push = (item: QueueItem) => {
+    queue.push(item);
+    wake?.();
+    wake = null;
+  };
+
+  let lastForcedRefreshAt = 0;
+  client.setAuth(async ({ forceRefreshToken }) => {
+    const now = Date.now();
+    if (!forceRefreshToken && grant.expiresAt - now > GRANT_REFRESH_MARGIN_MS) {
+      return grant.token;
+    }
+    if (forceRefreshToken && now - lastForcedRefreshAt < 5_000) {
+      // Convex rejected a token we just re-minted: one re-mint retry has
+      // already happened, so fail loudly instead of looping silently.
+      push({
+        kind: 'error',
+        error: new DeeplineError(
+          `Run observe grant for ${runId} was rejected after a re-mint. ` +
+            'The server and Convex deployment disagree on the grant issuer/JWKS.',
+          401,
+          'RUN_OBSERVE_GRANT_REJECTED',
+        ),
+      });
+      return null;
+    }
+    if (forceRefreshToken) {
+      lastForcedRefreshAt = now;
+    }
+    try {
+      grant = await mintRunObserveGrant(http, runId);
+      return grant.token;
+    } catch (error) {
+      push({ kind: 'error', error });
+      return null;
+    }
+  });
+
+  const unsubscribe = client.onUpdate(
+    snapshotQuery,
+    { workflowId: runId },
+    (run) => push({ kind: 'run', run: (run ?? null) as ObserverRunDoc | null }),
+    (error) => push({ kind: 'error', error }),
+  );
+
+  let lastUpdateAt = Date.now();
+  let lastStatusTerminal = false;
+  let disconnectedSince: number | null = null;
+  let warnedReconnecting = false;
+  let warnedStale = false;
+  const watchdog = setInterval(() => {
+    const now = Date.now();
+    try {
+      const connectionState = client.connectionState();
+      if (connectionState.isWebSocketConnected) {
+        disconnectedSince = null;
+        warnedReconnecting = false;
+      } else {
+        disconnectedSince ??= now;
+        if (
+          !warnedReconnecting &&
+          now - disconnectedSince >= OBSERVE_RECONNECT_NOTICE_MS
+        ) {
+          warnedReconnecting = true;
+          options.onNotice?.(
+            `[observe] connection lost; reconnecting to live run ${runId}…`,
+          );
+        }
+      }
+    } catch {
+      // connectionState is diagnostic only.
+    }
+    if (
+      !lastStatusTerminal &&
+      !warnedStale &&
+      now - lastUpdateAt >= OBSERVE_STALE_WARNING_MS
+    ) {
+      warnedStale = true;
+      options.onNotice?.(
+        `[observe] no live updates for ${Math.round((now - lastUpdateAt) / 1000)}s; ` +
+          `run ${runId} may be stalled (status checks continue)`,
+      );
+    }
+  }, OBSERVE_WATCHDOG_TICK_MS);
+  // Don't keep the process alive for the watchdog.
+  watchdog.unref?.();
+
+  const abortListener = () =>
+    push({
+      kind: 'error',
+      error: new DeeplineError(
+        'Run observation aborted.',
+        undefined,
+        'ABORTED',
+      ),
+    });
+  options.signal?.addEventListener('abort', abortListener, { once: true });
+
+  let diffState: PlayRunStreamDiffState = EMPTY_PLAY_RUN_STREAM_DIFF_STATE;
+  const streamId = ['observe', runId].join(':');
+  let sawFirstSnapshot = false;
+
+  try {
+    for (;;) {
+      if (queue.length === 0) {
+        const waitForItem = new Promise<void>((resolve) => {
+          wake = resolve;
+        });
+        if (!sawFirstSnapshot) {
+          // Convex unreachable is a bootstrap failure: no first result within
+          // the bootstrap window means the caller should fall back to SSE.
+          const timedOut = await Promise.race([
+            waitForItem.then(() => false),
+            new Promise<boolean>((resolve) =>
+              setTimeout(() => resolve(true), OBSERVE_BOOTSTRAP_TIMEOUT_MS),
+            ),
+          ]);
+          if (timedOut && queue.length === 0) {
+            throw new RunObserveTransportUnavailableError(
+              `no snapshot from Convex at ${grant.convexUrl} within ${OBSERVE_BOOTSTRAP_TIMEOUT_MS}ms`,
+              'convex_unreachable',
+            );
+          }
+        } else {
+          await waitForItem;
+        }
+        continue;
+      }
+
+      const item = queue.shift()!;
+      if (item.kind === 'error') {
+        if (options.signal?.aborted) {
+          return;
+        }
+        throw item.error;
+      }
+
+      sawFirstSnapshot = true;
+      lastUpdateAt = Date.now();
+      warnedStale = false;
+
+      if (item.run === null) {
+        throw new DeeplineError(
+          `Run ${runId} was not found (or is not visible to this grant).`,
+          404,
+          'RUN_NOT_FOUND',
+        );
+      }
+
+      const snapshot = buildPlayRunStatusSnapshot({ run: item.run });
+      lastStatusTerminal = isTerminalPlayRunLiveStatus(snapshot.status);
+
+      // Backfill log lines that rotated out of the retained tail window from
+      // the durable Run Log Stream before diffing, so the differ emits the
+      // retained tail without its loud gap marker. First-connect to an
+      // in-flight run (lastLogSeq === 0) is intentionally NOT backfilled:
+      // watchers get the tail plus the marker pointing at `runs logs`.
+      const gap = resolvePlayRunLogGap(snapshot, diffState.lastLogSeq);
+      if (gap && diffState.lastLogSeq > 0) {
+        const backfilled = await backfillLogGap({
+          queryLogPage: (afterSeq, limit) =>
+            client.query(logPageQuery, {
+              workflowId: runId,
+              afterSeq,
+              limit,
+            }) as Promise<ObserverRunLogPage>,
+          lastLogSeq: diffState.lastLogSeq,
+          tailFirstSeq: gap.tailFirstSeq,
+        });
+        if (backfilled && backfilled.length > 0) {
+          yield {
+            cursor: String(snapshot.updatedAt ?? Date.now()),
+            streamId,
+            scope: 'play',
+            type: 'play.run.log',
+            at: new Date().toISOString(),
+            payload: {
+              runId: snapshot.runId,
+              lines: backfilled,
+              source: 'worker',
+              firstSeq: diffState.lastLogSeq + 1,
+              totalLogCount: snapshot.totalLogCount,
+            },
+          };
+          diffState = { ...diffState, lastLogSeq: gap.tailFirstSeq - 1 };
+        }
+      }
+
+      const { events, next } = diffPlayRunStreamEvents({
+        streamId,
+        snapshot,
+        previous: diffState,
+      });
+      diffState = next;
+      // Snapshot event first, mirroring the SSE stream's ordering.
+      const ordered = [
+        ...events.filter((event) => event.type === 'play.run.snapshot'),
+        ...events.filter((event) => event.type !== 'play.run.snapshot'),
+      ];
+      for (const event of ordered) {
+        yield event;
+      }
+
+      if (lastStatusTerminal) {
+        return;
+      }
+    }
+  } finally {
+    clearInterval(watchdog);
+    options.signal?.removeEventListener('abort', abortListener);
+    try {
+      unsubscribe();
+    } catch {
+      // Already closed.
+    }
+    await client.close().catch(() => undefined);
+  }
+}

package/dist/repo/sdk/src/stream-reconnect.ts

@@ -0,0 +1,44 @@
+import { DeeplineError } from './errors.js';
+
+/**
+ * Shared reconnect policy for the canonical run SSE stream.
+ *
+ * Server stream windows are finite: the platform ends them cleanly at the
+ * function ceiling even while the run keeps executing. Consumers that wait for
+ * a terminal status (`plays run --watch`, `runs tail`, `client.runs.tail`)
+ * therefore reconnect with full-jitter exponential backoff instead of treating
+ * a window end as a failure.
+ */
+
+/** Base delay for the full-jitter exponential backoff between reconnects. */
+export const STREAM_RECONNECT_BASE_DELAY_MS = 500;
+/** Upper bound for a single reconnect delay. */
+export const STREAM_RECONNECT_MAX_DELAY_MS = 15_000;
+/**
+ * A connection that stayed open at least this long (or delivered any event)
+ * counts as healthy and resets the backoff sequence.
+ */
+export const STREAM_HEALTHY_CONNECTION_MS = 30_000;
+
+/** Full-jitter exponential backoff: uniform in [1, min(cap, base * 2^attempt)]. */
+export function streamReconnectDelayMs(attempt: number): number {
+  const cappedExponentialMs = Math.min(
+    STREAM_RECONNECT_MAX_DELAY_MS,
+    STREAM_RECONNECT_BASE_DELAY_MS * 2 ** Math.max(0, attempt),
+  );
+  return Math.max(1, Math.floor(Math.random() * (cappedExponentialMs + 1)));
+}
+
+export function isTransientPlayStreamError(error: unknown): boolean {
+  if (error instanceof DeeplineError && typeof error.statusCode === 'number') {
+    // Server-shaped errors with a definite status code are NOT transient by
+    // pattern — only network-level failures are. 5xx counts as transient
+    // since the server may recover, but 4xx (especially 404 = run gone) is
+    // terminal and should not be hidden behind a silent retry loop.
+    return error.statusCode >= 500 && error.statusCode < 600;
+  }
+  const text = error instanceof Error ? error.message : String(error);
+  return /auth validation backend timed out|fetch failed|eaddrnotavail|econnreset|etimedout|eai_again|socket hang up/i.test(
+    text,
+  );
+}

package/dist/repo/sdk/src/types.ts

@@ -540,12 +540,19 @@ export type PlayLiveEvent = LiveEventEnvelope<unknown> & {
  * Result returned by {@link DeeplineClient.stopPlay}.
  */
 export interface StopPlayRunResult {
-  /** Public play-run identifier that was stopped. */
+  /** Public play-run identifier the stop request targeted. */
   runId: string;
-  /** Stop request acknowledgement. */
-  stopped: true;
+  /** Whether the server confirmed the run was stopped. */
+  stopped: boolean;
   /** Number of open HITL interactions marked cancelled. */
   hitlCancelledCount: number;
+  /**
+   * True when the scheduler state for the run was stale and the stop could
+   * not be confirmed. Absent on older servers (treated as confirmed).
+   */
+  staleSchedulerState?: boolean;
+  /** Server-side error detail when the stop was not confirmed. */
+  error?: string;
 }
 
 /**

package/dist/repo/shared_libs/play-runtime/email-status.ts

@@ -89,41 +89,15 @@ export type EmailStatusBuildInput = {
   values: Record<string, unknown>;
 };
 
-const DEFAULT_STATUS_MAP: Record<string, EmailStatusMapEntry> = {
-  verified: { status: 'valid', verdict: 'send', verified: true },
-  valid: { status: 'valid', verdict: 'send', verified: true },
-  deliverable: { status: 'valid', verdict: 'send', verified: true },
-  true: { status: 'valid', verdict: 'send', verified: true },
-  invalid: { status: 'invalid', verdict: 'drop', verified: false },
-  undeliverable: { status: 'invalid', verdict: 'drop', verified: false },
-  false: { status: 'invalid', verdict: 'drop', verified: false },
-  'catch-all': {
-    status: 'catch_all',
-    verdict: 'verify_next',
-    verified: false,
-  },
-  catch_all: {
-    status: 'catch_all',
-    verdict: 'verify_next',
-    verified: false,
-  },
-  valid_catch_all: {
-    status: 'valid_catch_all',
-    verdict: 'send_with_caution',
-    verified: true,
-  },
-  accept_all: {
-    status: 'catch_all',
-    verdict: 'verify_next',
-    verified: false,
-  },
-  unknown: { status: 'unknown', verdict: 'hold', verified: false },
-  unavailable: { status: 'unknown', verdict: 'hold', verified: false },
-  do_not_mail: { status: 'do_not_mail', verdict: 'drop', verified: false },
-  spamtrap: { status: 'spamtrap', verdict: 'drop', verified: false },
-  abuse: { status: 'abuse', verdict: 'drop', verified: false },
-  disposable: { status: 'disposable', verdict: 'drop', verified: false },
-};
+// There is intentionally NO shared default status map. Each provider must
+// declare its own `statusMap` / `rules` for the raw status strings it emits
+// (see the provider registries under src/lib/integrations/**). A coarse global
+// map silently coerced provider-specific strings ("deliverable", "accept_all",
+// etc.) into canonical verdicts and let validators depend on guesses instead of
+// documented behavior. With it gone, an unmapped raw status falls through to
+// the typed signal inference below (and ultimately to `unknown`/`hold`) rather
+// than being normalized by a global lookup — loud and provider-declared over
+// silent and implicit.
 
 function normalizeKey(value: unknown): string | null {
   if (value == null) return null;
@@ -186,7 +160,7 @@ function entryForStatus(
   map: Record<string, EmailStatusMapEntry> | undefined,
 ): EmailStatusMapEntry | null {
   if (!key) return null;
-  return map?.[key] ?? DEFAULT_STATUS_MAP[key] ?? null;
+  return map?.[key] ?? null;
 }
 
 function read(values: Record<string, unknown>, name: string): unknown {

package/dist/repo/shared_libs/play-runtime/extractor-targets.ts

@@ -1,4 +1,4 @@
-import type { EmailStatus, EmailStatusValue } from './email-status';
+import type { EmailStatus } from './email-status';
 
 export const JOB_CHANGE_STATUS_VALUES = [
   'moved',
@@ -65,7 +65,7 @@ export const DEEPLINE_EXTRACTOR_TARGETS = Object.keys(
   DEEPLINE_EXTRACTOR_TARGET_DEFINITIONS,
 ) as DeeplineExtractorTarget[];
 
-export type DeeplineEmailStatusGetterValue = EmailStatus | EmailStatusValue;
+export type DeeplineEmailStatusGetterValue = EmailStatus;
 
 export type DeeplineGetterValueMap = {
   id: string;
@@ -91,7 +91,7 @@ export type DeeplineGetterValueMap = {
   status: string;
   job_change: JobChangeGetterValue;
   job_change_status: JobChangeStatus;
-  email_status: DeeplineEmailStatusGetterValue;
+  email_status: EmailStatus;
   phone_status: PhoneStatus;
 };