@agent-native/core 0.84.67 → 0.85.0

package/corpus/core/src/client/sse-event-processor.ts

@@ -148,6 +148,36 @@ export class AgentAutoContinueSignal extends Error {
 
 export const SSE_NO_PROGRESS_TIMEOUT_MS = 75_000;
 export const SSE_ACTION_PREPARATION_STALL_TIMEOUT_MS = 90_000;
+/**
+ * Widened client watchdog windows for durable background runs. The SERVER is
+ * the recovery brain for these runs: its run-manager no-progress backstop
+ * (150s, `RUN_NO_PROGRESS_HARD_TIMEOUT_MS`) emits `auto_continue` over the
+ * same stream the client is already reading, and its unclaimed-run sweep reaps
+ * dead workers into loud terminal errors. The client watchdogs therefore sit
+ * ABOVE the server's 150s backstop so a healthy background run never trips
+ * them — the server's own recovery event arrives first over the wire. When one
+ * does fire, the thrown signal only means "reattach the read" (the adapter's
+ * background follow loop re-polls /runs/active); it never escalates to a
+ * client-declared error or a synthetic continuation POST. Progress ACCOUNTING
+ * (what counts as a meaningful event) is unchanged — only the client-initiated
+ * recovery timing is relaxed.
+ */
+export const SSE_DURABLE_NO_PROGRESS_TIMEOUT_MS = 180_000;
+export const SSE_DURABLE_ACTION_PREPARATION_STALL_TIMEOUT_MS = 180_000;
+
+export function sseNoProgressTimeoutMs(options?: SSEStreamOptions): number {
+  return options?.durableBackgroundRun === true
+    ? SSE_DURABLE_NO_PROGRESS_TIMEOUT_MS
+    : SSE_NO_PROGRESS_TIMEOUT_MS;
+}
+
+function sseActionPreparationStallTimeoutMs(
+  options?: SSEStreamOptions,
+): number {
+  return options?.durableBackgroundRun === true
+    ? SSE_DURABLE_ACTION_PREPARATION_STALL_TIMEOUT_MS
+    : SSE_ACTION_PREPARATION_STALL_TIMEOUT_MS;
+}
 
 export interface SSEStreamOptions {
   /**
@@ -155,7 +185,9 @@ export interface SSEStreamOptions {
    * heartbeat. While one is active, generic keepalive-only periods keep the
    * client attached. Tool-input preparation is stricter: real byte progress
    * keeps long payloads alive, but zero-byte/silent preparation still recovers
-   * so one stuck action cannot pin the chat forever.
+   * so one stuck action cannot pin the chat forever — just on the wider
+   * durable windows above (behind the server's own 150s backstop) instead of
+   * the tight foreground 75s/90s windows.
    */
   durableBackgroundRun?: boolean;
   /**
@@ -435,20 +467,25 @@ function updatePreparingActionState(
   return undefined;
 }
 
-function hasStalledPreparingAction(state: PreparingActionState, now: number) {
+function hasStalledPreparingAction(
+  state: PreparingActionState,
+  now: number,
+  stallTimeoutMs: number,
+) {
   // Fire only when a tool input has gone SILENT — no further streaming deltas
   // for the whole window — never merely because a large input has been
   // streaming for a long time. `lastProgressAt` advances on every delta
   // heartbeat, so an actively-streaming large output keeps resetting this and
   // survives; a genuinely stuck prep (keepalive-only, no deltas) trips it.
+  // Durable background reads pass the wider durable window so the server's
+  // own 150s no-progress backstop gets first chance to recover the stall.
   for (const entry of [
     ...(state.toolEntries?.values() ?? []),
     ...(state.entries?.values() ?? []),
   ]) {
     if (
       entry.startedAt !== undefined &&
-      now - (entry.lastProgressAt ?? entry.startedAt) >=
-        SSE_ACTION_PREPARATION_STALL_TIMEOUT_MS
+      now - (entry.lastProgressAt ?? entry.startedAt) >= stallTimeoutMs
     ) {
       return true;
     }
@@ -459,9 +496,10 @@ function hasStalledPreparingAction(state: PreparingActionState, now: number) {
 async function readChunkWithProgressTimeout(
   reader: ReadableStreamDefaultReader<Uint8Array>,
   lastMeaningfulEventAt: number,
+  noProgressTimeoutMs: number,
 ): Promise<ReadableStreamReadResult<Uint8Array>> {
   const elapsed = Date.now() - lastMeaningfulEventAt;
-  const timeoutMs = Math.max(0, SSE_NO_PROGRESS_TIMEOUT_MS - elapsed);
+  const timeoutMs = Math.max(0, noProgressTimeoutMs - elapsed);
   let timeoutId: ReturnType<typeof setTimeout> | null = null;
   const readPromise = reader.read();
   // If the timeout wins and cancellation causes the pending read to reject,
@@ -1305,6 +1343,8 @@ export async function* readSSEStream(
   const decoder = new TextDecoder();
   let buf = "";
   let lastMeaningfulEventAt = Date.now();
+  const noProgressTimeoutMs = sseNoProgressTimeoutMs(options);
+  const preparationStallTimeoutMs = sseActionPreparationStallTimeoutMs(options);
   const activityTrail: ActivityTrailEntry[] = [];
   const preparingActionState: PreparingActionState =
     options?.preparingActionState ?? {};
@@ -1349,6 +1389,7 @@ export async function* readSSEStream(
         readResult = await readChunkWithProgressTimeout(
           reader,
           lastMeaningfulEventAt,
+          noProgressTimeoutMs,
         );
       } catch (err) {
         if (err instanceof AgentAutoContinueSignal) {
@@ -1428,7 +1469,13 @@ export async function* readSSEStream(
         );
 
         if (result) yield withStreamMetadata(result);
-        if (hasStalledPreparingAction(preparingActionState, Date.now())) {
+        if (
+          hasStalledPreparingAction(
+            preparingActionState,
+            Date.now(),
+            preparationStallTimeoutMs,
+          )
+        ) {
           throw new AgentAutoContinueSignal({
             reason: "no_progress",
             activityTrail: [...activityTrail],
@@ -1452,7 +1499,7 @@ export async function* readSSEStream(
 
       if (
         !sawProgressEvent &&
-        Date.now() - lastMeaningfulEventAt >= SSE_NO_PROGRESS_TIMEOUT_MS
+        Date.now() - lastMeaningfulEventAt >= noProgressTimeoutMs
       ) {
         throw new AgentAutoContinueSignal({
           reason: "no_progress",
@@ -1498,6 +1545,8 @@ export async function readSSEStreamRaw(
   const decoder = new TextDecoder();
   let buf = "";
   let lastMeaningfulEventAt = Date.now();
+  const noProgressTimeoutMs = sseNoProgressTimeoutMs(options);
+  const preparationStallTimeoutMs = sseActionPreparationStallTimeoutMs(options);
   const activityTrail: ActivityTrailEntry[] = [];
   const preparingActionState: PreparingActionState =
     options?.preparingActionState ?? {};
@@ -1517,6 +1566,7 @@ export async function readSSEStreamRaw(
         readResult = await readChunkWithProgressTimeout(
           reader,
           lastMeaningfulEventAt,
+          noProgressTimeoutMs,
         );
       } catch (err) {
         if (err instanceof AgentAutoContinueSignal) {
@@ -1612,7 +1662,13 @@ export async function readSSEStreamRaw(
               : { reason: "stream_ended", activityTrail: [...activityTrail] },
           );
         }
-        if (hasStalledPreparingAction(preparingActionState, Date.now())) {
+        if (
+          hasStalledPreparingAction(
+            preparingActionState,
+            Date.now(),
+            preparationStallTimeoutMs,
+          )
+        ) {
           onUpdate(contentSnapshot(content));
           throw new AgentAutoContinueSignal({
             reason: "no_progress",
@@ -1630,7 +1686,7 @@ export async function readSSEStreamRaw(
 
       if (
         !sawProgressEvent &&
-        Date.now() - lastMeaningfulEventAt >= SSE_NO_PROGRESS_TIMEOUT_MS
+        Date.now() - lastMeaningfulEventAt >= noProgressTimeoutMs
       ) {
         throw new AgentAutoContinueSignal({
           reason: "no_progress",

package/corpus/core/src/client/use-run-stuck-detection.ts

@@ -34,11 +34,20 @@ export interface UseRunStuckDetectionOptions {
   /** The thread to monitor. Pass null/undefined to disable polling. */
   threadId: string | null | undefined;
   /**
-   * Threshold above which an in-flight run is considered stuck. The default
-   * sits comfortably above the adapter's 75s no-progress reconnect — by then
-   * automatic recovery has already had its chance.
+   * Threshold above which an in-flight FOREGROUND run is considered stuck.
+   * The default sits comfortably above the adapter's 75s no-progress
+   * reconnect — by then automatic recovery has already had its chance.
    */
   stuckThresholdMs?: number;
+  /**
+   * Threshold for BACKGROUND-dispatched runs (dispatchMode starts with
+   * "background"). The server owns recovery for these — its run-manager
+   * no-progress backstop (150s) and unclaimed-run sweep act first — so the
+   * user-facing "stuck" affordance is a late fallback, not a race against
+   * them. Selected inside the hook because the dispatch mode is only known
+   * from the same poll response that computes the elapsed time.
+   */
+  backgroundStuckThresholdMs?: number;
   /** Poll interval. Default 5_000ms. */
   pollIntervalMs?: number;
   /** API base path. Default `/_agent-native/agent-chat`. */
@@ -46,6 +55,7 @@ export interface UseRunStuckDetectionOptions {
 }
 
 const DEFAULT_STUCK_THRESHOLD_MS = 90_000;
+export const DEFAULT_BACKGROUND_STUCK_THRESHOLD_MS = 180_000;
 const DEFAULT_POLL_INTERVAL_MS = 5_000;
 const IDLE_BACKOFF_INTERVAL_MS = 15_000;
 
@@ -74,6 +84,7 @@ const EMPTY_STATE: RunStuckState = {
 export function useRunStuckDetection({
   threadId,
   stuckThresholdMs = DEFAULT_STUCK_THRESHOLD_MS,
+  backgroundStuckThresholdMs = DEFAULT_BACKGROUND_STUCK_THRESHOLD_MS,
   pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,
   apiUrl,
 }: UseRunStuckDetectionOptions): RunStuckState {
@@ -112,11 +123,19 @@ export function useRunStuckDetection({
           const heartbeatAt = data.heartbeatAt ?? null;
           const heartbeatSinceMs =
             heartbeatAt != null ? nowMs - heartbeatAt : null;
+          const dispatchMode =
+            typeof data.dispatchMode === "string" ? data.dispatchMode : null;
+          // Background-dispatched runs get the wider threshold: the server's
+          // own recovery (150s no-progress backstop + chained continuations)
+          // must get its chance before the user sees a "stuck" affordance.
+          const effectiveThresholdMs = dispatchMode?.startsWith("background")
+            ? backgroundStuckThresholdMs
+            : stuckThresholdMs;
           const isStuck = Boolean(
             data.active &&
             data.status === "running" &&
             stuckSinceMs != null &&
-            stuckSinceMs > stuckThresholdMs,
+            stuckSinceMs > effectiveThresholdMs,
           );
           setState({
             isStuck,
@@ -126,8 +145,7 @@ export function useRunStuckDetection({
             stuckSinceMs,
             heartbeatAt,
             heartbeatSinceMs,
-            dispatchMode:
-              typeof data.dispatchMode === "string" ? data.dispatchMode : null,
+            dispatchMode,
           });
           // Back off polling when nothing is in flight — there's no point
           // hammering the endpoint while the chat is idle. We still poll
@@ -153,7 +171,13 @@ export function useRunStuckDetection({
       cancelled = true;
       if (timer) clearTimeout(timer);
     };
-  }, [threadId, stuckThresholdMs, pollIntervalMs, apiUrl]);
+  }, [
+    threadId,
+    stuckThresholdMs,
+    backgroundStuckThresholdMs,
+    pollIntervalMs,
+    apiUrl,
+  ]);
 
   return state;
 }

package/corpus/core/src/db/runtime-diagnostics.ts

@@ -0,0 +1,331 @@
+import { createHash } from "node:crypto";
+
+import {
+  getDatabaseUrl,
+  getDialect,
+  getDbExec,
+  type DbExec,
+  type Dialect,
+} from "./client.js";
+
+export interface DatabaseRuntimeFingerprint {
+  configured: boolean;
+  source: string;
+  dialect: Dialect;
+  urlHash?: string;
+  protocol?: string;
+  host?: string;
+  database?: string;
+  appName?: string;
+  authTokenConfigured: boolean;
+  netlifyDatabaseUrlConfigured: boolean;
+  neon?: {
+    endpointId?: string;
+    pooled: boolean;
+    projectHost?: string;
+  };
+}
+
+export interface RuntimeDebugFingerprint {
+  app: string;
+  environment: string;
+  deployContext?: string;
+  deployId?: string;
+  commitRef?: string;
+  branch?: string;
+  siteName?: string;
+  database: DatabaseRuntimeFingerprint;
+}
+
+export interface RequiredSchemaTable {
+  table: string;
+  columns: string[];
+}
+
+export interface DatabaseSchemaHealthResult {
+  ok: boolean;
+  checked: boolean;
+  dialect: Dialect;
+  missingTables: string[];
+  missingColumns: Array<{ table: string; column: string }>;
+  error?: string;
+}
+
+const DEFAULT_REQUIRED_SCHEMA: RequiredSchemaTable[] = [
+  {
+    table: "agent_runs",
+    columns: [
+      "id",
+      "thread_id",
+      "status",
+      "started_at",
+      "completed_at",
+      "heartbeat_at",
+      "last_progress_at",
+      "turn_id",
+      "error_code",
+      "error_detail",
+      "terminal_reason",
+      "dispatch_mode",
+      "diag_stage",
+      "worker_stage",
+    ],
+  },
+  {
+    table: "agent_run_events",
+    columns: ["run_id", "seq", "event_at", "event_data"],
+  },
+  {
+    table: "agent_tool_ledger",
+    columns: ["thread_id", "tool_key", "result_summary", "completed_at"],
+  },
+  {
+    table: "chat_threads",
+    columns: [
+      "id",
+      "owner_email",
+      "title",
+      "preview",
+      "thread_data",
+      "message_count",
+      "created_at",
+      "updated_at",
+      "scope_type",
+      "scope_id",
+      "scope_label",
+      "pinned_at",
+      "archived_at",
+    ],
+  },
+  {
+    table: "settings",
+    columns: ["key", "value", "updated_at"],
+  },
+  {
+    table: "application_state",
+    columns: ["session_id", "key", "value", "updated_at"],
+  },
+];
+
+function envValue(key: string): string | undefined {
+  const value = process.env[key]?.trim();
+  return value || undefined;
+}
+
+function appEnvPrefix(): string | undefined {
+  return envValue("APP_NAME")?.toUpperCase().replace(/-/g, "_");
+}
+
+function databaseUrlSource(): string {
+  const appName = appEnvPrefix();
+  if (appName && envValue(`${appName}_DATABASE_URL`)) {
+    return `${appName}_DATABASE_URL`;
+  }
+  if (envValue("DATABASE_URL")) return "DATABASE_URL";
+  if (envValue("NETLIFY_DATABASE_URL")) return "NETLIFY_DATABASE_URL";
+  return "default";
+}
+
+function databaseAuthTokenConfigured(): boolean {
+  const appName = appEnvPrefix();
+  return Boolean(
+    (appName && envValue(`${appName}_DATABASE_AUTH_TOKEN`)) ||
+    envValue("DATABASE_AUTH_TOKEN") ||
+    envValue("NETLIFY_DATABASE_AUTH_TOKEN"),
+  );
+}
+
+function shortHash(value: string): string | undefined {
+  if (!value) return undefined;
+  return createHash("sha256").update(value).digest("hex").slice(0, 12);
+}
+
+function parseDatabaseUrl(url: string): Partial<DatabaseRuntimeFingerprint> {
+  if (!url) return {};
+  if (url.startsWith("pglite:")) {
+    return { protocol: "pglite", database: url.slice("pglite:".length) };
+  }
+  if (url.startsWith("file:")) {
+    return { protocol: "file", database: url.slice("file:".length) };
+  }
+  if (!/^[a-z][a-z0-9+.-]*:\/\//i.test(url)) {
+    return { protocol: "sqlite", database: url };
+  }
+
+  try {
+    const parsed = new URL(url);
+    const host = parsed.hostname || undefined;
+    const neonMatch = host?.match(/^(ep-[a-z0-9-]+?)(-pooler)?\.(.+)$/i);
+    const isNeon = Boolean(host?.endsWith(".neon.tech") && neonMatch);
+    return {
+      protocol: parsed.protocol.replace(/:$/, ""),
+      host,
+      database: parsed.pathname.replace(/^\//, "") || undefined,
+      ...(isNeon
+        ? {
+            neon: {
+              endpointId: neonMatch?.[1],
+              pooled: Boolean(neonMatch?.[2]),
+              projectHost: neonMatch?.[3],
+            },
+          }
+        : {}),
+    };
+  } catch {
+    return {};
+  }
+}
+
+export function getDatabaseRuntimeFingerprint(): DatabaseRuntimeFingerprint {
+  const url = getDatabaseUrl();
+  const parsed = parseDatabaseUrl(url);
+  return {
+    configured: Boolean(url),
+    source: databaseUrlSource(),
+    dialect: getDialect(),
+    urlHash: shortHash(url),
+    appName: envValue("APP_NAME"),
+    authTokenConfigured: databaseAuthTokenConfigured(),
+    netlifyDatabaseUrlConfigured: Boolean(envValue("NETLIFY_DATABASE_URL")),
+    ...parsed,
+  };
+}
+
+export function getRuntimeDebugFingerprint(): RuntimeDebugFingerprint {
+  return {
+    app: envValue("APP_NAME") ?? "unknown",
+    environment: envValue("NODE_ENV") ?? "unknown",
+    deployContext: envValue("CONTEXT") ?? envValue("VERCEL_ENV"),
+    deployId: envValue("DEPLOY_ID") ?? envValue("VERCEL_DEPLOYMENT_ID"),
+    commitRef:
+      envValue("COMMIT_REF") ??
+      envValue("NETLIFY_COMMIT_REF") ??
+      envValue("VERCEL_GIT_COMMIT_SHA") ??
+      envValue("GIT_COMMIT_SHA"),
+    branch:
+      envValue("BRANCH") ??
+      envValue("HEAD") ??
+      envValue("VERCEL_GIT_COMMIT_REF"),
+    siteName: envValue("SITE_NAME") ?? envValue("NETLIFY_SITE_NAME"),
+    database: getDatabaseRuntimeFingerprint(),
+  };
+}
+
+function assertSafeIdentifier(value: string): void {
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(value)) {
+    throw new Error(`Unsafe schema identifier: ${value}`);
+  }
+}
+
+async function postgresTableColumns(
+  exec: DbExec,
+  table: string,
+): Promise<Set<string> | null> {
+  const result = await exec.execute({
+    sql: `SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = ?`,
+    args: [table],
+  });
+  if (!result.rows.length) return null;
+  return new Set(result.rows.map((row) => String(row.column_name)));
+}
+
+async function sqliteTableColumns(
+  exec: DbExec,
+  table: string,
+): Promise<Set<string> | null> {
+  assertSafeIdentifier(table);
+  const exists = await exec.execute({
+    sql: `SELECT name FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1`,
+    args: [table],
+  });
+  if (!exists.rows.length) return null;
+  const result = await exec.execute(`PRAGMA table_info(${table})`);
+  return new Set(result.rows.map((row) => String(row.name)));
+}
+
+async function tableColumns(
+  exec: DbExec,
+  dialect: Dialect,
+  table: string,
+): Promise<Set<string> | null> {
+  return dialect === "postgres"
+    ? postgresTableColumns(exec, table)
+    : sqliteTableColumns(exec, table);
+}
+
+export async function runDatabaseSchemaHealthCheck(
+  options: {
+    exec?: DbExec;
+    dialect?: Dialect;
+    required?: RequiredSchemaTable[];
+  } = {},
+): Promise<DatabaseSchemaHealthResult> {
+  const dialect = options.dialect ?? getDialect();
+  const exec = options.exec ?? getDbExec();
+  const required = options.required ?? DEFAULT_REQUIRED_SCHEMA;
+  const missingTables: string[] = [];
+  const missingColumns: Array<{ table: string; column: string }> = [];
+
+  try {
+    for (const requirement of required) {
+      const columns = await tableColumns(exec, dialect, requirement.table);
+      if (!columns) {
+        missingTables.push(requirement.table);
+        continue;
+      }
+      for (const column of requirement.columns) {
+        if (!columns.has(column)) {
+          missingColumns.push({ table: requirement.table, column });
+        }
+      }
+    }
+  } catch (err) {
+    return {
+      ok: false,
+      checked: false,
+      dialect,
+      missingTables,
+      missingColumns,
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+
+  return {
+    ok: missingTables.length === 0 && missingColumns.length === 0,
+    checked: true,
+    dialect,
+    missingTables,
+    missingColumns,
+  };
+}
+
+export function formatRuntimeDebugFingerprint(
+  fingerprint: RuntimeDebugFingerprint = getRuntimeDebugFingerprint(),
+): string {
+  const db = fingerprint.database;
+  return [
+    `app: ${fingerprint.app}`,
+    `environment: ${fingerprint.environment}`,
+    fingerprint.deployContext
+      ? `deploy_context: ${fingerprint.deployContext}`
+      : "",
+    fingerprint.deployId ? `deploy_id: ${fingerprint.deployId}` : "",
+    fingerprint.commitRef ? `commit_ref: ${fingerprint.commitRef}` : "",
+    fingerprint.branch ? `branch: ${fingerprint.branch}` : "",
+    fingerprint.siteName ? `site_name: ${fingerprint.siteName}` : "",
+    `db_configured: ${db.configured}`,
+    `db_source: ${db.source}`,
+    `db_dialect: ${db.dialect}`,
+    db.protocol ? `db_protocol: ${db.protocol}` : "",
+    db.host ? `db_host: ${db.host}` : "",
+    db.database ? `db_database: ${db.database}` : "",
+    db.urlHash ? `db_url_hash: ${db.urlHash}` : "",
+    db.neon?.endpointId ? `db_neon_endpoint: ${db.neon.endpointId}` : "",
+    db.neon ? `db_neon_pooled: ${db.neon.pooled}` : "",
+    db.neon?.projectHost ? `db_neon_project_host: ${db.neon.projectHost}` : "",
+    `db_auth_token_configured: ${db.authTokenConfigured}`,
+    `netlify_database_url_configured: ${db.netlifyDatabaseUrlConfigured}`,
+  ]
+    .filter(Boolean)
+    .join("\n");
+}