@agent-native/core 0.84.66 → 0.85.0

package/corpus/core/src/server/agent-chat-plugin.ts

@@ -8088,6 +8088,9 @@ Non-code requests are still fine on this surface: read data, navigate the UI, su
                 stages: m.RUN_DIAG_STAGE,
               }))
               .catch(() => null);
+            const runtimeDetail = await import("../db/runtime-diagnostics.js")
+              .then((m) => m.formatRuntimeDebugFingerprint())
+              .catch(() => "");
 
             // Record "the route handler was entered" against the run after auth
             // succeeds. This is the proof the bg-fn invocation actually reached
@@ -8097,14 +8100,74 @@ Non-code requests are still fine on this surface: read data, navigate the UI, su
                 .record(prepared.runId, diag.stages.routeEntered)
                 .catch(() => {});
               await diag
-                .record(prepared.runId, diag.stages.authPassed)
+                .record(prepared.runId, diag.stages.authPassed, runtimeDetail)
                 .catch(() => {});
             }
 
+            // PAYLOAD REHYDRATION: a `payloadRef` marker means the dispatch
+            // carried ONLY the marker (Netlify caps background-function request
+            // bodies at 256KB — a large chat history silently exceeded it) and
+            // the full request body is persisted on the run row. Rehydrate it
+            // here. The payload is NOT cleared on read: a Netlify at-least-once
+            // retry of a failed invocation must be able to rehydrate again (the
+            // claim CAS still dedupes execution); terminal status writes clear it.
+            let workerBody: Record<string, unknown> = prepared.body;
+            const preparedMarkerRecord =
+              preparedMarker && typeof preparedMarker === "object"
+                ? (preparedMarker as Record<string, unknown>)
+                : null;
+            if (preparedMarkerRecord?.payloadRef === true) {
+              const runStore = await import("../agent/run-store.js");
+              const rawPayload = await runStore
+                .readRunDispatchPayload(prepared.runId)
+                .catch(() => null);
+              let parsedPayload: Record<string, unknown> | null = null;
+              if (rawPayload) {
+                try {
+                  const candidate = JSON.parse(rawPayload);
+                  if (candidate && typeof candidate === "object") {
+                    parsedPayload = candidate as Record<string, unknown>;
+                  }
+                } catch {
+                  // Corrupt payload — treated as missing below.
+                }
+              }
+              if (!parsedPayload) {
+                // Row missing / reaped / already terminal — there is nothing to
+                // run. Fail the run loudly (if it is still running) and ack the
+                // dispatch with a 200 so Netlify does not retry a dead handoff.
+                if (diag) {
+                  await diag
+                    .record(
+                      prepared.runId,
+                      diag.stages.workerThrew,
+                      "dispatch payload missing — cannot rehydrate background run body",
+                    )
+                    .catch(() => {});
+                }
+                const statusUpdated = await runStore
+                  .updateRunStatusIfRunning(prepared.runId, "errored")
+                  .catch(() => false);
+                if (statusUpdated) {
+                  await runStore
+                    .setRunTerminalReason(
+                      prepared.runId,
+                      "dispatch_payload_missing",
+                    )
+                    .catch(() => {});
+                }
+                return { ok: false, skipped: "dispatch-payload-missing" };
+              }
+              workerBody = {
+                ...parsedPayload,
+                [AGENT_CHAT_BACKGROUND_RUN_FIELD]: preparedMarker,
+              };
+            }
+
             // Stash the verified+augmented body for the handler — the body stream
             // is already consumed, so the handler reads this instead.
             (event as any).context = (event as any).context ?? {};
-            (event as any).context.__agentChatBackgroundBody = prepared.body;
+            (event as any).context.__agentChatBackgroundBody = workerBody;
 
             // Durable owner context: this self-dispatch is cookieless (HMAC-only).
             // Resolve the owner from the persisted run row, never the request
@@ -8287,6 +8350,56 @@ Non-code requests are still fine on this surface: read data, navigate the UI, su
         }, 15_000); // Start 15s after init (after the scheduler)
       })();
 
+      // ─── Unclaimed background-run sweep ────────────────────────────────
+      // Backstop for LOST background handoffs. The foreground circuit-breaker
+      // covers the initial dispatch (a connected client is polling the claim),
+      // but a server-chained CONTINUATION handoff has no foreground watching
+      // it: if the dispatch is lost after the successor row was inserted, the
+      // row would sit at dispatch_mode='background' forever and the turn hangs
+      // silently. This sweep reaps such rows into a loud, attributable error
+      // (`background_worker_never_started`) that the client renders, instead
+      // of an idle spinner. Cheap: one indexed-ish query per 2-min window.
+      (() => {
+        let lastSweep = 0;
+        const SWEEP_INTERVAL_MS = 2 * 60 * 1000;
+
+        setTimeout(() => {
+          setInterval(() => {
+            const now = Date.now();
+            if (now - lastSweep < SWEEP_INTERVAL_MS) return;
+            lastSweep = now;
+
+            (async () => {
+              const {
+                listUnclaimedBackgroundRunIds,
+                reapUnclaimedBackgroundRun,
+              } = await import("../agent/run-store.js");
+              let runIds: string[];
+              try {
+                runIds = await listUnclaimedBackgroundRunIds();
+              } catch {
+                return; // Table may not exist yet on first boot
+              }
+              for (const staleRunId of runIds) {
+                try {
+                  const reaped = await reapUnclaimedBackgroundRun(staleRunId);
+                  if (reaped) {
+                    console.error(
+                      "[agent-chat] swept unclaimed background run (handoff lost):",
+                      staleRunId,
+                    );
+                  }
+                } catch {
+                  // best-effort per run
+                }
+              }
+            })().catch(() => {
+              // best-effort — never break the server
+            });
+          }, 30_000); // Check every 30s but only sweep once per 2min
+        }, 20_000); // Start 20s after init (after the agent-teams sweep)
+      })();
+
       // ─── Trigger Dispatcher (event-based automations) ─────────────────
       if (disableRecurringJobsRuntime) {
         if (process.env.DEBUG) {

package/corpus/core/src/server/analytics.ts

@@ -42,42 +42,9 @@ function getGaMeasurementId(): string | null {
   );
 }
 
-/**
- * Script hosts the injected GA loader pulls executable code from. Google Tag
- * Manager serves `gtag/js`, and GA4 can lazy-load additional collectors from
- * `www.google-analytics.com`. These must be listed in the document `script-src`
- * so the CSP reflects the code the framework itself injects (see
- * `applyDocumentCsp` in `ssr-handler.ts`).
- */
-export const GA_CSP_SCRIPT_HOSTS = [
-  "https://www.googletagmanager.com",
-  "https://www.google-analytics.com",
-] as const;
-
-/**
- * Network/image hosts used by the GA4 loader when it sends page-view and event
- * beacons. These are separate from `script-src`: a stricter deployment CSP with
- * `connect-src 'self'` or `img-src 'self'` can load gtag.js but still drop all
- * analytics events unless these hosts are present too.
- */
-export const GA_CSP_CONNECT_HOSTS = [
-  "https://www.google-analytics.com",
-  "https://analytics.google.com",
-  "https://stats.g.doubleclick.net",
-  "https://region1.google-analytics.com",
-] as const;
-
-export const GA_CSP_IMG_HOSTS = [
-  "https://www.google-analytics.com",
-  "https://www.googletagmanager.com",
-  "https://stats.g.doubleclick.net",
-] as const;
-
 /**
  * The exact JS body (no surrounding `<script>` tags) of the inline gtag config
- * block injected next to the gtag.js loader. Returned so the SSR handler can
- * hash it for the `script-src` CSP directive — the hash must be computed from
- * the identical string that `getGaScript()` embeds, so both call this helper.
+ * block injected next to the gtag.js loader.
  * Returns `null` when GA is not configured.
  */
 export function getGaInlineConfigScriptBody(): string | null {

package/corpus/core/src/server/core-routes-plugin.ts

@@ -45,6 +45,12 @@ import {
 import { mountBrowserSessionRoutes } from "../browser-sessions/routes.js";
 import { mountDbAdminRoutes } from "../db-admin/routes.js";
 import { getDbExec } from "../db/client.js";
+import {
+  getDatabaseRuntimeFingerprint,
+  getRuntimeDebugFingerprint,
+  runDatabaseSchemaHealthCheck,
+  type DatabaseSchemaHealthResult,
+} from "../db/runtime-diagnostics.js";
 import {
   uploadFile,
   getActiveFileUploadProviderForRequest,
@@ -143,6 +149,7 @@ import {
   DEFAULT_UPLOAD_MAX_FILE_BYTES,
   isAllowedUploadMimeType,
 } from "./h3-helpers.js";
+import { createHttpResponseTelemetryMiddleware } from "./http-response-telemetry.js";
 import { isIdentitySsoEnabled } from "./identity-sso-store.js";
 import { handleIdentitySso } from "./identity-sso.js";
 import { createOpenRouteHandler } from "./open-route.js";
@@ -204,10 +211,24 @@ export function getFrameworkEnvKeys(): EnvKeyConfig[] {
 export interface DbHealthProbeResult {
   /** The serverless function is live and served the request. */
   ok: true;
+  /** Database + optional schema readiness for stricter production monitors. */
+  ready: boolean;
   /** A trivial `SELECT 1` reached the database (false = no DB or unreachable). */
   db: boolean;
   /** Round-trip time of the probe in milliseconds. */
   ms: number;
+  /** Redacted database routing details useful for deploy/runtime checks. */
+  database: {
+    configured: boolean;
+    source: string;
+    dialect: string;
+    urlHash?: string;
+    appName?: string;
+    authTokenConfigured: boolean;
+    netlifyDatabaseUrlConfigured: boolean;
+  };
+  /** Optional metadata-only schema compatibility check. */
+  schema?: DatabaseSchemaHealthResult;
 }
 
 /**
@@ -222,16 +243,40 @@ export interface DbHealthProbeResult {
  */
 export async function runDbHealthProbe(
   exec: () => { execute: (sql: string) => Promise<unknown> } = getDbExec,
+  options: { schema?: boolean } = {},
 ): Promise<DbHealthProbeResult> {
   const startedAt = Date.now();
   let db = false;
+  let schema: DatabaseSchemaHealthResult | undefined;
+  const dbExec = exec();
   try {
-    await exec().execute("SELECT 1");
+    await dbExec.execute("SELECT 1");
     db = true;
   } catch {
     // Live even when the DB is unreachable or the app has no database.
   }
-  return { ok: true, db, ms: Date.now() - startedAt };
+  if (db && options.schema) {
+    schema = await runDatabaseSchemaHealthCheck({
+      exec: dbExec as ReturnType<typeof getDbExec>,
+    });
+  }
+  const database = getDatabaseRuntimeFingerprint();
+  return {
+    ok: true,
+    ready: db && (!schema || schema.ok),
+    db,
+    ms: Date.now() - startedAt,
+    database: {
+      configured: database.configured,
+      source: database.source,
+      dialect: database.dialect,
+      urlHash: database.urlHash,
+      appName: database.appName,
+      authTokenConfigured: database.authTokenConfigured,
+      netlifyDatabaseUrlConfigured: database.netlifyDatabaseUrlConfigured,
+    },
+    ...(schema ? { schema } : {}),
+  };
 }
 const DEFAULT_BUILDER_WAITLIST_FORM_ID = "DYTHuM0jlV";
 const DEFAULT_BUILDER_WAITLIST_FORMS_ORIGIN = "https://forms.agent-native.com";
@@ -863,6 +908,8 @@ export function createCoreRoutesPlugin(
 
       const P = FRAMEWORK_ROUTE_PREFIX;
 
+      getH3App(nitroApp).use(createHttpResponseTelemetryMiddleware());
+
       // Security response headers — emitted on every framework response.
       // Mounted before route handlers so 4xx/5xx error pages also carry the
       // headers. Routes that need to tighten a specific header override via
@@ -1071,17 +1118,57 @@ export function createCoreRoutesPlugin(
       // Health + DB warmup — liveness probe that touches the database so
       // uptime monitors and the keep-warm cron prevent a scale-to-zero
       // serverless DB (e.g. Neon) from cold-starting on the next real
-      // request. Public, side-effect free, and never cached.
+      // request. Public, side-effect free, and never cached. Add ?schema=1
+      // for metadata-only schema checks, and ?strict=1 to turn a not-ready
+      // DB/schema probe into a failing HTTP status for monitors.
       if (!options.disableHealth) {
         getH3App(nitroApp).use(
           `${P}/health`,
           defineEventHandler(async (event) => {
             setResponseHeader(event, "cache-control", "no-store");
-            return runDbHealthProbe();
+            const schema =
+              event.url?.searchParams.get("schema") === "1" ||
+              event.url?.searchParams.get("schema") === "true";
+            const strict =
+              event.url?.searchParams.get("strict") === "1" ||
+              event.url?.searchParams.get("strict") === "true" ||
+              process.env.AGENT_NATIVE_HEALTH_STRICT_SCHEMA === "true";
+            const result = await runDbHealthProbe(getDbExec, { schema });
+            if (strict && !result.ready) setResponseStatus(event, 503);
+            return result;
           }),
         );
       }
 
+      getH3App(nitroApp).use(
+        `${P}/debug/runtime`,
+        defineEventHandler(async (event) => {
+          setResponseHeader(event, "cache-control", "no-store");
+          const session = await getSession(event).catch(() => null);
+          const productionLike =
+            process.env.NODE_ENV === "production" ||
+            process.env.NETLIFY === "true" ||
+            process.env.VERCEL === "1";
+          if (!session?.email && productionLike) {
+            setResponseStatus(event, 401);
+            return { error: "Authentication required" };
+          }
+          const schema = await runDatabaseSchemaHealthCheck().catch((err) => ({
+            ok: false,
+            checked: false,
+            dialect: getDatabaseRuntimeFingerprint().dialect,
+            missingTables: [],
+            missingColumns: [],
+            error: err instanceof Error ? err.message : String(err),
+          }));
+          return {
+            ok: true,
+            runtime: getRuntimeDebugFingerprint(),
+            schema,
+          };
+        }),
+      );
+
       getH3App(nitroApp).use(
         `${P}/speculation-rules.json`,
         defineEventHandler((event) => {

package/corpus/core/src/server/http-response-telemetry.ts

@@ -0,0 +1,189 @@
+import { defineEventHandler, getHeader, getMethod } from "h3";
+import type { EventHandler, H3Event } from "h3";
+
+import { getDatabaseRuntimeFingerprint } from "../db/runtime-diagnostics.js";
+import { track } from "../tracking/index.js";
+import { getAppName } from "./app-name.js";
+
+const TELEMETRY_EVENT_NAME = "http.response";
+const TRACKING_INGEST_PATHS = new Set([
+  "/track",
+  "/api/analytics/track",
+  "/api/events/track",
+  "/_agent-native/track",
+]);
+
+function envValue(key: string): string | undefined {
+  const value = process.env[key]?.trim();
+  return value || undefined;
+}
+
+function boolEnv(key: string): boolean {
+  return ["1", "true", "yes", "on"].includes(
+    (process.env[key] ?? "").trim().toLowerCase(),
+  );
+}
+
+function sampleRate(): number {
+  const raw = envValue("AGENT_NATIVE_HTTP_TELEMETRY_SAMPLE_RATE");
+  if (!raw) return 1;
+  const parsed = Number.parseFloat(raw);
+  if (!Number.isFinite(parsed)) return 1;
+  return Math.max(0, Math.min(1, parsed));
+}
+
+function shouldDisableTelemetry(): boolean {
+  return boolEnv("AGENT_NATIVE_HTTP_TELEMETRY_DISABLED");
+}
+
+function requestPath(event: H3Event): string {
+  const raw =
+    event.url?.pathname ??
+    String(event.node?.req?.url ?? event.path ?? "/").split("?")[0] ??
+    "/";
+  return raw || "/";
+}
+
+function normalizeSegment(segment: string): string {
+  if (!segment) return segment;
+  if (/^[0-9]+$/.test(segment)) return ":id";
+  if (/^[0-9a-f]{8}-[0-9a-f-]{27}$/i.test(segment)) return ":id";
+  if (
+    /^(run|turn|thread|design|screen|file|msg|key|tok)_[a-z0-9_-]+$/i.test(
+      segment,
+    )
+  ) {
+    return ":id";
+  }
+  if (/^(run|turn)-[0-9]{10,}-[a-z0-9]+$/i.test(segment)) return ":id";
+  if (segment.length > 36 && /^[a-z0-9_-]+$/i.test(segment)) return ":id";
+  return segment;
+}
+
+export function normalizeHttpTelemetryPath(pathname: string): string {
+  const normalized = pathname.startsWith("/") ? pathname : `/${pathname}`;
+  return normalized
+    .split("/")
+    .map((segment, index) => (index === 0 ? "" : normalizeSegment(segment)))
+    .join("/");
+}
+
+function statusClass(statusCode: number): string {
+  if (!Number.isFinite(statusCode) || statusCode < 100) return "unknown";
+  return `${Math.floor(statusCode / 100)}xx`;
+}
+
+function routeKind(pathname: string): string {
+  if (pathname === "/_agent-native" || pathname.startsWith("/_agent-native/")) {
+    return "framework";
+  }
+  if (pathname === "/api" || pathname.startsWith("/api/")) return "api";
+  if (pathname.startsWith("/.well-known/")) return "well-known";
+  return "app";
+}
+
+function hostForEvent(event: H3Event): string | undefined {
+  return (
+    getHeader(event, "x-forwarded-host") ??
+    getHeader(event, "host") ??
+    undefined
+  );
+}
+
+function organizationForHost(host: string | undefined): string | undefined {
+  const configured =
+    envValue("AGENT_NATIVE_ANALYTICS_ORG_NAME") ??
+    envValue("AGENT_NATIVE_ORG_NAME");
+  if (configured) return configured;
+  const normalized = host?.split(":")[0]?.toLowerCase();
+  return normalized?.endsWith(".agent-native.com") ||
+    normalized === "agent-native.com"
+    ? "Builder.io"
+    : undefined;
+}
+
+function shouldTrack(pathname: string, statusCode: number): boolean {
+  if (shouldDisableTelemetry()) return false;
+  if (TRACKING_INGEST_PATHS.has(pathname)) return false;
+  if (pathname.startsWith("/api/analytics/replay")) return false;
+  if (statusCode >= 500) return true;
+  const rate = sampleRate();
+  if (rate <= 0) return false;
+  if (rate >= 1) return true;
+  return Math.random() < rate;
+}
+
+function responseStatusCode(event: H3Event): number {
+  const raw =
+    (event.node?.res as any)?.statusCode ??
+    (event.node?.res as any)?.status ??
+    (event as any).res?.status ??
+    200;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) ? parsed : 200;
+}
+
+function emitTelemetry(event: H3Event, startedAt: number): void {
+  const statusCode = responseStatusCode(event);
+  const pathname = requestPath(event);
+  if (!shouldTrack(pathname, statusCode)) return;
+
+  try {
+    const host = hostForEvent(event);
+    const db = getDatabaseRuntimeFingerprint();
+    track(TELEMETRY_EVENT_NAME, {
+      source: "server",
+      app: getAppName(),
+      template: envValue("AGENT_NATIVE_TEMPLATE") ?? getAppName(),
+      organization: organizationForHost(host),
+      method: getMethod(event),
+      path: normalizeHttpTelemetryPath(pathname),
+      route_kind: routeKind(pathname),
+      status_code: statusCode,
+      status_class: statusClass(statusCode),
+      duration_ms: Math.max(0, Date.now() - startedAt),
+      host,
+      environment: envValue("NODE_ENV"),
+      deploy_context: envValue("CONTEXT") ?? envValue("VERCEL_ENV"),
+      deploy_id: envValue("DEPLOY_ID") ?? envValue("VERCEL_DEPLOYMENT_ID"),
+      commit_ref:
+        envValue("COMMIT_REF") ??
+        envValue("NETLIFY_COMMIT_REF") ??
+        envValue("VERCEL_GIT_COMMIT_SHA") ??
+        envValue("GIT_COMMIT_SHA"),
+      db_source: db.source,
+      db_dialect: db.dialect,
+      db_url_hash: db.urlHash,
+      db_neon_endpoint: db.neon?.endpointId,
+      db_neon_pooled: db.neon?.pooled,
+    });
+  } catch {
+    // Response telemetry is best-effort. Never perturb request handling.
+  }
+}
+
+export function createHttpResponseTelemetryMiddleware(): EventHandler {
+  return defineEventHandler((event) => {
+    const startedAt = Date.now();
+    const res = event.node?.res as
+      | {
+          once?: (event: "finish" | "close", cb: () => void) => void;
+          on?: (event: "finish" | "close", cb: () => void) => void;
+        }
+      | undefined;
+    let emitted = false;
+    const emitOnce = () => {
+      if (emitted) return;
+      emitted = true;
+      emitTelemetry(event, startedAt);
+    };
+
+    if (typeof res?.once === "function") {
+      res.once("finish", emitOnce);
+      res.once("close", emitOnce);
+    } else if (typeof res?.on === "function") {
+      res.on("finish", emitOnce);
+      res.on("close", emitOnce);
+    }
+  });
+}

package/corpus/core/src/server/self-dispatch.ts

@@ -94,6 +94,23 @@ export interface FireInternalDispatchOptions {
   body?: Record<string, unknown>;
   /** Max ms to wait for the outbound request to leave the box. Default 250ms. */
   settleMs?: number;
+  /**
+   * Await the dispatch response fully instead of racing the settle timer.
+   *
+   * The 250ms settle race is correct for a synchronous handler that must
+   * respond to its own caller quickly — but it is WRONG for a handoff fired
+   * from a function that is about to finish (e.g. a background worker chaining
+   * its continuation chunk): once the handler's promise resolves, the Lambda
+   * freezes and a still-in-flight dispatch fetch is killed WITHOUT rejecting,
+   * so the handoff is lost silently — the error path never fires. With
+   * `awaitResponse: true` the call resolves only after the target confirmed
+   * receipt (Netlify background functions 202 on enqueue, normally well under
+   * a second) and throws on any network error or non-2xx, bounded by
+   * `responseTimeoutMs`.
+   */
+  awaitResponse?: boolean;
+  /** Max ms to await the dispatch response when `awaitResponse` is set. Default 15s. */
+  responseTimeoutMs?: number;
 }
 
 async function dispatchResponseError(
@@ -167,10 +184,14 @@ export async function fireInternalDispatch(
     }
   }
 
+  const awaitResponse = options.awaitResponse === true;
   const dispatchPromise = fetch(url, {
     method: "POST",
     headers,
     body: JSON.stringify({ taskId: options.taskId, ...(options.body ?? {}) }),
+    ...(awaitResponse
+      ? { signal: AbortSignal.timeout(options.responseTimeoutMs ?? 15_000) }
+      : {}),
   }).then(async (res) => {
     if (!res.ok) {
       throw await dispatchResponseError(options.path, res);
@@ -187,6 +208,14 @@ export async function fireInternalDispatch(
     );
   });
 
+  if (awaitResponse) {
+    // Confirmed handoff: resolve only once the target acknowledged the
+    // dispatch (throws on network error / timeout / non-2xx). Used by callers
+    // whose own invocation is about to end — see the option doc above.
+    await dispatchPromise;
+    return;
+  }
+
   const settleMs = options.settleMs ?? DEFAULT_DISPATCH_SETTLE_MS;
   await Promise.race([
     dispatchPromise,