@agent-native/core 0.109.2 → 0.109.4

package/corpus/core/src/integrations/remote-push-store.ts

@@ -4,7 +4,12 @@ import {
   intType,
   retryOnDdlRace,
 } from "../db/client.js";
-import { ensureTableExists, ensureIndexExists } from "../db/ddl-guard.js";
+import {
+  ensureColumnExists,
+  ensureIndexExists,
+  ensureTableExists,
+} from "../db/ddl-guard.js";
+import { isDuplicateColumnError } from "../db/migrations.js";
 import type {
   PublicRemotePushRegistration,
   RemotePushNotification,
@@ -46,6 +51,10 @@ function buildCreateNotificationsSql(): string {
     payload_json TEXT NOT NULL,
     status TEXT NOT NULL,
     attempts ${intType()} NOT NULL DEFAULT 0,
+    provider_ticket_id TEXT,
+    next_attempt_at ${intType()} NOT NULL DEFAULT 0,
+    last_error TEXT,
+    delivered_at ${intType()},
     created_at ${intType()} NOT NULL,
     updated_at ${intType()} NOT NULL
   )
@@ -80,6 +89,30 @@ async function ensureTables(): Promise<void> {
           "idx_remote_push_notifications_owner",
           `CREATE INDEX IF NOT EXISTS idx_remote_push_notifications_owner ON integration_remote_push_notifications(owner_email, org_id, status, created_at)`,
         );
+        await ensureColumnExists(
+          "integration_remote_push_notifications",
+          "provider_ticket_id",
+          `ALTER TABLE integration_remote_push_notifications ADD COLUMN IF NOT EXISTS provider_ticket_id TEXT`,
+        );
+        await ensureColumnExists(
+          "integration_remote_push_notifications",
+          "next_attempt_at",
+          `ALTER TABLE integration_remote_push_notifications ADD COLUMN IF NOT EXISTS next_attempt_at ${intType()} NOT NULL DEFAULT 0`,
+        );
+        await ensureColumnExists(
+          "integration_remote_push_notifications",
+          "last_error",
+          `ALTER TABLE integration_remote_push_notifications ADD COLUMN IF NOT EXISTS last_error TEXT`,
+        );
+        await ensureColumnExists(
+          "integration_remote_push_notifications",
+          "delivered_at",
+          `ALTER TABLE integration_remote_push_notifications ADD COLUMN IF NOT EXISTS delivered_at ${intType()}`,
+        );
+        await ensureIndexExists(
+          "idx_remote_push_notifications_delivery",
+          `CREATE INDEX IF NOT EXISTS idx_remote_push_notifications_delivery ON integration_remote_push_notifications(status, next_attempt_at, updated_at)`,
+        );
         return;
       }
       // SQLite (local dev): keep existing behavior
@@ -101,6 +134,18 @@ async function ensureTables(): Promise<void> {
           `CREATE INDEX IF NOT EXISTS idx_remote_push_notifications_owner ON integration_remote_push_notifications(owner_email, org_id, status, created_at)`,
         ),
       );
+      await addNotificationColumnIfMissing("provider_ticket_id", "TEXT");
+      await addNotificationColumnIfMissing(
+        "next_attempt_at",
+        `${intType()} NOT NULL DEFAULT 0`,
+      );
+      await addNotificationColumnIfMissing("last_error", "TEXT");
+      await addNotificationColumnIfMissing("delivered_at", intType());
+      await retryOnDdlRace(() =>
+        client.execute(
+          `CREATE INDEX IF NOT EXISTS idx_remote_push_notifications_delivery ON integration_remote_push_notifications(status, next_attempt_at, updated_at)`,
+        ),
+      );
     })().catch((err) => {
       // Retry init on the next call after a failed startup.
       _initPromise = undefined;
@@ -110,6 +155,21 @@ async function ensureTables(): Promise<void> {
   return _initPromise;
 }
 
+async function addNotificationColumnIfMissing(
+  name: string,
+  definition: string,
+): Promise<void> {
+  try {
+    await retryOnDdlRace(() =>
+      getDbExec().execute(
+        `ALTER TABLE integration_remote_push_notifications ADD COLUMN ${name} ${definition}`,
+      ),
+    );
+  } catch (error) {
+    if (!isDuplicateColumnError(error)) throw error;
+  }
+}
+
 function rowToRegistration(
   row: Record<string, unknown>,
 ): RemotePushRegistration {
@@ -134,13 +194,18 @@ function rowToRegistration(
 function rowToNotification(
   row: Record<string, unknown>,
 ): RemotePushNotification {
+  const storedStatus = String(row.status ?? "pending");
+  const status =
+    storedStatus === "delivered" || storedStatus === "failed"
+      ? storedStatus
+      : "pending";
   return {
     id: row.id as string,
     ownerEmail: row.owner_email as string,
     orgId: (row.org_id as string | null) ?? null,
     registrationId: row.registration_id as string,
     payload: parseJson(row.payload_json, null),
-    status: row.status as RemotePushNotification["status"],
+    status,
     attempts: Number(row.attempts ?? 0),
     createdAt: Number(row.created_at ?? 0),
     updatedAt: Number(row.updated_at ?? 0),
@@ -314,31 +379,34 @@ export async function queueRemotePushNotifications(input: {
     orgId: input.orgId ?? null,
     limit: 100,
   });
+  if (registrations.length === 0) return { queued: 0 };
+
   const client = getDbExec();
   const now = Date.now();
-  let queued = 0;
-  for (const registration of registrations) {
-    const id = `remote-push-notification-${now}-${randomHex(8)}`;
-    const result = await client.execute({
-      sql: `INSERT INTO integration_remote_push_notifications
+  const payload = JSON.stringify(input.payload ?? null);
+  const values = registrations.map(
+    () => "(?, ?, ?, ?, ?, 'pending', 0, ?, ?, ?)",
+  );
+  const args = registrations.flatMap((registration) => [
+    `remote-push-notification-${now}-${randomHex(8)}`,
+    input.ownerEmail,
+    input.orgId ?? null,
+    registration.id,
+    payload,
+    now,
+    now,
+    now,
+  ]);
+  const result = await client.execute({
+    sql: `INSERT INTO integration_remote_push_notifications
         (id, owner_email, org_id, registration_id, payload_json, status,
-         attempts, created_at, updated_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-      args: [
-        id,
-        input.ownerEmail,
-        input.orgId ?? null,
-        registration.id,
-        JSON.stringify(input.payload ?? null),
-        "pending",
-        0,
-        now,
-        now,
-      ],
-    });
-    queued += result.rowsAffected ?? (result as any).rowCount ?? 0;
-  }
-  return { queued };
+         attempts, next_attempt_at, created_at, updated_at)
+        VALUES ${values.join(", ")}`,
+    args,
+  });
+  return {
+    queued: result.rowsAffected ?? (result as any).rowCount ?? 0,
+  };
 }
 
 export async function listRemotePushNotificationsForOwner(input: {
@@ -349,13 +417,17 @@ export async function listRemotePushNotificationsForOwner(input: {
 }): Promise<RemotePushNotification[]> {
   await ensureTables();
   const limit = Math.max(1, Math.min(input.limit ?? 50, 100));
-  const statusClause = input.status ? " AND status = ?" : "";
+  const statusClause = input.status
+    ? input.status === "pending"
+      ? " AND status IN ('pending', 'sending', 'sent', 'checking')"
+      : " AND status = ?"
+    : "";
   const args: Array<string | number | null> = [
     input.ownerEmail,
     input.orgId ?? null,
     input.orgId ?? null,
   ];
-  if (input.status) args.push(input.status);
+  if (input.status && input.status !== "pending") args.push(input.status);
   args.push(limit);
   const { rows } = await getDbExec().execute({
     sql: `SELECT * FROM integration_remote_push_notifications
@@ -368,6 +440,183 @@ export async function listRemotePushNotificationsForOwner(input: {
   return rows.map((row) => rowToNotification(row as Record<string, unknown>));
 }
 
+export interface ClaimedRemotePushDelivery {
+  id: string;
+  registrationId: string;
+  provider: string;
+  token: string;
+  payload: unknown;
+  phase: "send" | "receipt";
+  providerTicketId: string | null;
+  attempts: number;
+}
+
+export async function claimNextRemotePushDelivery(input?: {
+  now?: number;
+  staleAfterMs?: number;
+}): Promise<ClaimedRemotePushDelivery | null> {
+  await ensureTables();
+  const client = getDbExec();
+  const now = input?.now ?? Date.now();
+  const staleBefore = now - Math.max(30_000, input?.staleAfterMs ?? 120_000);
+  const { rows } = await client.execute({
+    sql: `SELECT n.id, n.registration_id, n.payload_json, n.status, n.attempts,
+                 n.provider_ticket_id, n.updated_at, r.provider, r.token
+          FROM integration_remote_push_notifications n
+          INNER JOIN integration_remote_push_registrations r
+            ON r.id = n.registration_id
+          WHERE r.status = 'active'
+            AND (
+              (n.status IN ('pending', 'sent') AND n.next_attempt_at <= ?)
+              OR (n.status IN ('sending', 'checking') AND n.updated_at <= ?)
+            )
+          ORDER BY n.created_at ASC
+          LIMIT 1`,
+    args: [now, staleBefore],
+  });
+  const row = rows[0] as Record<string, unknown> | undefined;
+  if (!row) return null;
+
+  const previousStatus = String(row.status ?? "pending");
+  const providerTicketId = sanitizeString(
+    row.provider_ticket_id as string | null,
+    200,
+  );
+  const phase =
+    (previousStatus === "sent" || previousStatus === "checking") &&
+    providerTicketId
+      ? "receipt"
+      : "send";
+  const claimedStatus = phase === "receipt" ? "checking" : "sending";
+  const result = await client.execute({
+    sql: `UPDATE integration_remote_push_notifications
+          SET status = ?, attempts = attempts + 1, updated_at = ?
+          WHERE id = ? AND status = ? AND updated_at = ?`,
+    args: [
+      claimedStatus,
+      now,
+      row.id as string,
+      previousStatus,
+      Number(row.updated_at),
+    ],
+  });
+  if ((result.rowsAffected ?? (result as any).rowCount ?? 0) !== 1) return null;
+
+  return {
+    id: row.id as string,
+    registrationId: row.registration_id as string,
+    provider: String(row.provider ?? "unknown"),
+    token: row.token as string,
+    payload: parseJson(row.payload_json, null),
+    phase,
+    providerTicketId,
+    attempts: Number(row.attempts ?? 0) + 1,
+  };
+}
+
+export async function markRemotePushTicketAccepted(input: {
+  id: string;
+  providerTicketId: string;
+  checkAfter: number;
+}): Promise<boolean> {
+  await ensureTables();
+  const now = Date.now();
+  const result = await getDbExec().execute({
+    sql: `UPDATE integration_remote_push_notifications
+          SET status = 'sent', provider_ticket_id = ?, next_attempt_at = ?,
+              last_error = NULL, updated_at = ?
+          WHERE id = ? AND status = 'sending'`,
+    args: [input.providerTicketId, input.checkAfter, now, input.id],
+  });
+  return (result.rowsAffected ?? (result as any).rowCount ?? 0) === 1;
+}
+
+export async function markRemotePushDelivered(id: string): Promise<boolean> {
+  await ensureTables();
+  const now = Date.now();
+  const result = await getDbExec().execute({
+    sql: `UPDATE integration_remote_push_notifications
+          SET status = 'delivered', last_error = NULL, delivered_at = ?,
+              updated_at = ?
+          WHERE id = ? AND status = 'checking'`,
+    args: [now, now, id],
+  });
+  return (result.rowsAffected ?? (result as any).rowCount ?? 0) === 1;
+}
+
+export async function retryRemotePushDelivery(input: {
+  id: string;
+  phase: "send" | "receipt";
+  retryAt: number;
+  errorCode: string;
+  resend?: boolean;
+}): Promise<boolean> {
+  await ensureTables();
+  const currentStatus = input.phase === "receipt" ? "checking" : "sending";
+  const retryStatus =
+    input.phase === "receipt" && !input.resend ? "sent" : "pending";
+  const clearTicketClause = input.resend ? ", provider_ticket_id = NULL" : "";
+  const result = await getDbExec().execute({
+    sql: `UPDATE integration_remote_push_notifications
+          SET status = ?, next_attempt_at = ?, last_error = ?${clearTicketClause},
+              updated_at = ?
+          WHERE id = ? AND status = ?`,
+    args: [
+      retryStatus,
+      input.retryAt,
+      sanitizeString(input.errorCode, 160) ?? "temporary_error",
+      Date.now(),
+      input.id,
+      currentStatus,
+    ],
+  });
+  return (result.rowsAffected ?? (result as any).rowCount ?? 0) === 1;
+}
+
+export async function failRemotePushDelivery(input: {
+  id: string;
+  phase: "send" | "receipt";
+  errorCode: string;
+}): Promise<boolean> {
+  await ensureTables();
+  const currentStatus = input.phase === "receipt" ? "checking" : "sending";
+  const result = await getDbExec().execute({
+    sql: `UPDATE integration_remote_push_notifications
+          SET status = 'failed', last_error = ?, updated_at = ?
+          WHERE id = ? AND status = ?`,
+    args: [
+      sanitizeString(input.errorCode, 160) ?? "delivery_failed",
+      Date.now(),
+      input.id,
+      currentStatus,
+    ],
+  });
+  return (result.rowsAffected ?? (result as any).rowCount ?? 0) === 1;
+}
+
+export async function deactivateRemotePushRegistration(
+  registrationId: string,
+): Promise<boolean> {
+  await ensureTables();
+  const client = getDbExec();
+  const now = Date.now();
+  const result = await client.execute({
+    sql: `UPDATE integration_remote_push_registrations
+          SET status = 'inactive', updated_at = ?
+          WHERE id = ? AND status = 'active'`,
+    args: [now, registrationId],
+  });
+  await client.execute({
+    sql: `UPDATE integration_remote_push_notifications
+          SET status = 'failed', last_error = 'registration_inactive',
+              updated_at = ?
+          WHERE registration_id = ?
+            AND status IN ('pending', 'sending', 'sent', 'checking')`,
+    args: [now, registrationId],
+  });
+  return (result.rowsAffected ?? (result as any).rowCount ?? 0) === 1;
+}
+
 async function getRemotePushRegistrationByTokenHash(
   tokenHash: string,
 ): Promise<RemotePushRegistration | null> {

package/corpus/core/src/server/better-auth-instance.ts

@@ -1105,7 +1105,11 @@ async function buildDatabaseConfig(
 ): Promise<BetterAuthOptions["database"]> {
   if (dialect === "postgres") {
     const url = getDatabaseUrl();
-    const { isNeonUrl } = await import("../db/create-get-db.js");
+    const {
+      buildResilientNeonPool,
+      buildResilientPostgresJsClient,
+      isNeonUrl,
+    } = await import("../db/create-get-db.js");
 
     if (isPgliteUrl(url)) {
       const { drizzle } = await loadPgliteDrizzle();
@@ -1133,7 +1137,9 @@ async function buildDatabaseConfig(
       });
       attachNeonPoolErrorLogger(_neonAuthPool, "db/neon-auth");
       const { drizzle } = await import("drizzle-orm/neon-serverless");
-      const db = drizzle(_neonAuthPool, { schema: pgAuthSchema });
+      const db = drizzle(buildResilientNeonPool(_neonAuthPool), {
+        schema: pgAuthSchema,
+      });
       const { drizzleAdapter } = await import("better-auth/adapters/drizzle");
       return drizzleAdapter(db, {
         provider: "pg",
@@ -1149,7 +1155,9 @@ async function buildDatabaseConfig(
     const { default: postgres } = await import("postgres");
     const sql = postgres(url, pgPoolOptions(url));
     const { drizzle } = await import("drizzle-orm/postgres-js");
-    const db = drizzle(sql, { schema: pgAuthSchema });
+    const db = drizzle(buildResilientPostgresJsClient(sql), {
+      schema: pgAuthSchema,
+    });
     const { drizzleAdapter } = await import("better-auth/adapters/drizzle");
     return drizzleAdapter(db, {
       provider: "pg",

package/corpus/core/src/server/core-routes-plugin.ts

@@ -165,7 +165,6 @@ import {
   DEFAULT_UPLOAD_MAX_FILE_BYTES,
   isAllowedUploadMimeType,
 } from "./h3-helpers.js";
-import { createHttpResponseTelemetryMiddleware } from "./http-response-telemetry.js";
 import { isIdentitySsoEnabled } from "./identity-sso-store.js";
 import { handleIdentitySso } from "./identity-sso.js";
 import { createOpenRouteHandler } from "./open-route.js";
@@ -1069,6 +1068,35 @@ export interface CoreRoutesPluginOptions {
   anonymousOwner?: BuilderAnonymousOwnerResolver;
 }
 
+interface LegacyCoreRouteInitSettings {
+  persistedEnvVars: Record<string, string> | null;
+  builderDisconnected: { at?: number } | null;
+}
+
+type CoreRouteSettingReader = (
+  key: string,
+) => Promise<Record<string, unknown> | null>;
+
+export async function readLegacyCoreRouteInitSettings(
+  readSetting: CoreRouteSettingReader = getSetting,
+): Promise<LegacyCoreRouteInitSettings> {
+  const readOrNull = async (key: string) => {
+    try {
+      return await readSetting(key);
+    } catch {
+      return null;
+    }
+  };
+  const [persistedEnvVars, builderDisconnected] = await Promise.all([
+    readOrNull("persisted-env-vars"),
+    readOrNull("builder-disconnected"),
+  ]);
+  return {
+    persistedEnvVars: persistedEnvVars as Record<string, string> | null,
+    builderDisconnected: builderDisconnected as { at?: number } | null,
+  };
+}
+
 /**
  * Creates a Nitro plugin that mounts all standard agent-native framework routes.
  *
@@ -1110,18 +1138,17 @@ export function createCoreRoutesPlugin(
     try {
       await awaitBootstrap(nitroApp);
 
+      const { persistedEnvVars, builderDisconnected } =
+        await readLegacyCoreRouteInitSettings();
+
       // Legacy cleanup: key saves now go to scoped app_secrets rows. Do not
       // rehydrate the old deployment-global `persisted-env-vars` row into
       // process.env; keep only the Builder scrub so stale leaked keys self-heal.
       try {
-        const persisted = (await getSetting("persisted-env-vars")) as Record<
-          string,
-          string
-        > | null;
-        if (persisted) {
+        if (persistedEnvVars) {
           const builderKeys = new Set<string>(BUILDER_ENV_KEYS);
           let scrubbed = 0;
-          for (const k of Object.keys(persisted)) {
+          for (const k of Object.keys(persistedEnvVars)) {
             if (builderKeys.has(k)) {
               scrubbed++;
             }
@@ -1129,7 +1156,7 @@ export function createCoreRoutesPlugin(
           if (scrubbed > 0) {
             try {
               const cleaned: Record<string, string> = {};
-              for (const [k, v] of Object.entries(persisted)) {
+              for (const [k, v] of Object.entries(persistedEnvVars)) {
                 if (!builderKeys.has(k)) cleaned[k] = v;
               }
               await putSetting("persisted-env-vars", cleaned);
@@ -1154,10 +1181,7 @@ export function createCoreRoutesPlugin(
       // plugin init while the flag is set. The flag is cleared by the
       // Builder cli-auth callback when the user re-connects.
       try {
-        const disconnected = (await getSetting("builder-disconnected")) as {
-          at?: number;
-        } | null;
-        if (disconnected) {
+        if (builderDisconnected) {
           for (const key of BUILDER_ENV_KEYS) {
             delete process.env[key];
           }
@@ -1223,8 +1247,6 @@ export function createCoreRoutesPlugin(
         );
       }
 
-      getH3App(nitroApp).use(createHttpResponseTelemetryMiddleware());
-
       // Security response headers — emitted on every framework response.
       // Mounted before route handlers so 4xx/5xx error pages also carry the
       // headers. Routes that need to tighten a specific header override via

package/corpus/core/src/server/framework-request-handler.ts

@@ -19,6 +19,10 @@ import { MCP_PUBLIC_ROUTE_PREFIX } from "../mcp/route-paths.js";
 import { getConfiguredAppBasePath } from "./app-base-path.js";
 import { captureError } from "./capture-error.js";
 import { createCsrfMiddleware } from "./csrf.js";
+import {
+  installHttpResponseTelemetryHooks,
+  recordFrameworkReadyWait,
+} from "./http-response-telemetry.js";
 
 const BOOTSTRAPPED = new WeakSet<object>();
 const IN_BOOTSTRAP = new WeakSet<object>();
@@ -114,6 +118,7 @@ export function markDefaultPluginProvided(nitroApp: any, stem: string): void {
 export function getH3App(nitroApp: any): H3AppShim {
   if (!nitroApp) throw new Error("getH3App: nitroApp is required");
   ensureGlobalMiddlewareDispatch(nitroApp);
+  installHttpResponseTelemetryHooks(nitroApp);
 
   // Reuse the cached shim if we've wrapped this nitroApp before
   const cached = nitroApp[APP_SHIM_KEY] as H3AppShim | undefined;
@@ -204,7 +209,12 @@ export function getH3App(nitroApp: any): H3AppShim {
         resolveMountMatch(reqPath, WELL_KNOWN_PREFIX) ||
         resolveMountMatch(reqPath, MCP_PUBLIC_ROUTE_PREFIX)
       ) {
-        await awaitFrameworkRoutesReadyForRequest(nitroApp, reqPath);
+        const startedAt = Date.now();
+        try {
+          await awaitFrameworkRoutesReadyForRequest(nitroApp, reqPath);
+        } finally {
+          recordFrameworkReadyWait(event, Date.now() - startedAt);
+        }
       }
     });
   }