@agent-native/core 0.119.5 → 0.119.6

package/src/integrations/plugin.ts

@@ -9,6 +9,10 @@ import {
 import { getRequestHeader } from "h3";
 import { createRemoteJWKSet, jwtVerify } from "jose";
 
+import {
+  AGENT_BACKGROUND_PROCESSOR_FIELD,
+  AGENT_BACKGROUND_PROCESSOR_INTEGRATION,
+} from "../agent/durable-background.js";
 import { abortRun } from "../agent/run-manager.js";
 import { getOrgContext, resolveOrgIdForEmail } from "../org/context.js";
 import { loadResourcesForPrompt } from "../server/agent-chat-plugin.js";
@@ -65,22 +69,27 @@ import {
   updateIntegrationInstallation,
   upsertIntegrationInstallation,
 } from "./installations-store.js";
+import {
+  dispatchPendingIntegrationTask,
+  INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT,
+  integrationDispatchScopeValue,
+  isIntegrationDurableDispatchConfigured,
+} from "./integration-durable-dispatch.js";
 import {
   forgetIntegrationMemory,
   integrationMemoryActions,
   listIntegrationMemory,
   rememberForIntegrationScope,
 } from "./integration-memory.js";
+import { extractBearerToken, verifyInternalToken } from "./internal-token.js";
 import {
-  extractBearerToken,
-  signInternalToken,
-  verifyInternalToken,
-} from "./internal-token.js";
-import { startPendingTasksRetryJob } from "./pending-tasks-retry-job.js";
+  retryStuckPendingTasks,
+  startPendingTasksRetryJob,
+} from "./pending-tasks-retry-job.js";
 import {
   claimPendingTask,
   failTaskDeliveryTransition,
-  getNextPendingTaskIdForThread,
+  getNextPendingTaskForThread,
   insertPendingTask,
   isDuplicateEventError,
   MAX_PENDING_TASK_ATTEMPTS,
@@ -725,40 +734,29 @@ export function createIntegrationsPlugin(
           externalEventKey: opts?.dedupeKey
             ? systemNoticeEventKey(opts.dedupeKey, dedupeTtlMs)
             : undefined,
+          dispatchScope: integrationDispatchScopeValue({
+            platform: incoming.platform,
+            externalThreadId: noticeThreadId,
+            platformContext: incoming.platformContext,
+          }),
         });
       } catch (err) {
         if (isDuplicateEventError(err)) return;
         throw err;
       }
 
-      // The SQL row is the durable source of truth. This best-effort self-call
-      // only reduces latency; the recurring pending-task sweep retries a row
-      // if the serverless host freezes this webhook execution immediately.
-      let token: string | undefined;
-      try {
-        token = signInternalToken(taskId);
-      } catch (err) {
-        if (process.env.NODE_ENV === "production") throw err;
-      }
-      void fetch(`${getBaseUrl(event)}${P}/process-task`, {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          ...(token ? { Authorization: `Bearer ${token}` } : {}),
+      await dispatchPendingIntegrationTask({
+        taskId,
+        task: {
+          platform: incoming.platform,
+          externalThreadId: noticeThreadId,
+          platformContext: incoming.platformContext,
         },
-        body: JSON.stringify({ taskId }),
-      }).catch((err) => {
-        console.warn("[integrations] System notice dispatch failed:", err);
+        event,
+        baseUrl: getBaseUrl(event),
       });
     }
 
-    async function requireSession(event: any): Promise<boolean> {
-      const session = await getSession(event).catch(() => null);
-      if (session?.email) return true;
-      setResponseStatus(event, 401);
-      return false;
-    }
-
     async function requireSessionContext(
       event: any,
     ): Promise<{ ownerEmail: string; orgId: string | null } | null> {
@@ -903,9 +901,10 @@ export function createIntegrationsPlugin(
           setResponseStatus(event, 405);
           return { error: "Method not allowed" };
         }
-        if (!(await requireSession(event))) return { error: "unauthorized" };
+        const scope = await requireSessionContext(event);
+        if (!scope) return { error: "unauthorized" };
         try {
-          return await getTaskQueueStats();
+          return await getTaskQueueStats(scope);
         } catch (err: any) {
           setResponseStatus(event, 500);
           return { error: err?.message ?? String(err) };
@@ -1595,10 +1594,53 @@ export function createIntegrationsPlugin(
       }),
     );
 
+    // ─── Durable pending-task recovery sweep ─────────────────────
+    h3.use(
+      `${P}/retry-stuck-tasks`,
+      defineEventHandler(async (event) => {
+        if (getMethod(event) !== "POST") {
+          setResponseStatus(event, 405);
+          return { error: "Method not allowed" };
+        }
+        const body = (await readBody(event)) as {
+          taskId?: string;
+          [AGENT_BACKGROUND_PROCESSOR_FIELD]?: string;
+        };
+        if (body?.taskId !== INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT) {
+          setResponseStatus(event, 400);
+          return { error: "invalid sweep subject" };
+        }
+        if (!process.env.A2A_SECRET) {
+          setResponseStatus(event, 503);
+          return { error: "durable integration recovery is not configured" };
+        }
+        const token = extractBearerToken(
+          getRequestHeader(event, "authorization"),
+        );
+        if (
+          !token ||
+          !verifyInternalToken(INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT, token)
+        ) {
+          setResponseStatus(event, 401);
+          return { error: "Invalid or expired internal token" };
+        }
+        if (!isIntegrationDurableDispatchConfigured()) {
+          return { ok: true, disabled: true };
+        }
+        const result = await retryStuckPendingTasks({
+          webhookBaseUrl: getBaseUrl(event),
+          limit: 20,
+          durableOnly: true,
+        });
+        return { ok: true, ...result };
+      }),
+    );
+
     // ─── Process pending task (cross-platform task queue) ────────
     // POST /_agent-native/integrations/process-task
-    // Internal endpoint invoked via fire-and-forget self-webhook from the
-    // public webhook handler. Auth: HMAC bearer signed with A2A_SECRET.
+    // Internal endpoint invoked from the public webhook handler through either
+    // the portable self-dispatch path or an acknowledged background handoff.
+    // Auth: HMAC bearer signed with A2A_SECRET.
     // Each invocation runs the agent loop in a fresh function execution.
     h3.use(
       `${P}/process-task`,
@@ -1608,7 +1650,10 @@ export function createIntegrationsPlugin(
           return { error: "Method not allowed" };
         }
 
-        const body = (await readBody(event)) as { taskId?: string };
+        const body = (await readBody(event)) as {
+          taskId?: string;
+          [AGENT_BACKGROUND_PROCESSOR_FIELD]?: string;
+        };
         const taskId = body?.taskId;
         if (!taskId) {
           setResponseStatus(event, 400);
@@ -1645,7 +1690,12 @@ export function createIntegrationsPlugin(
         }
 
         // Atomic claim: only one invocation gets to process this task
-        const task = await claimPendingTask(taskId);
+        const dispatchOutcome =
+          body[AGENT_BACKGROUND_PROCESSOR_FIELD] ===
+          AGENT_BACKGROUND_PROCESSOR_INTEGRATION
+            ? "background-acknowledged"
+            : "portable-unconfirmed";
+        const task = await claimPendingTask(taskId, { dispatchOutcome });
         if (!task) {
           setResponseStatus(event, 200);
           return { ok: true, skipped: "already-claimed-or-missing" };
@@ -1779,24 +1829,22 @@ export function createIntegrationsPlugin(
             return { ok: true, taskId, retrying: "response-delivery" };
           }
           await markTaskCompleted(taskId);
-          const nextTaskId = await getNextPendingTaskIdForThread(
+          const nextTask = await getNextPendingTaskForThread(
             task.platform,
             task.externalThreadId,
           );
-          if (nextTaskId) {
-            const nextToken = signInternalToken(nextTaskId);
-            void fetch(`${getBaseUrl(event)}${P}/process-task`, {
-              method: "POST",
-              headers: {
-                "Content-Type": "application/json",
-                ...(nextToken ? { Authorization: `Bearer ${nextToken}` } : {}),
+          if (nextTask) {
+            await dispatchPendingIntegrationTask({
+              taskId: nextTask.id,
+              task: {
+                platform: task.platform,
+                externalThreadId: task.externalThreadId,
+                platformContext: nextTask.dispatchScope
+                  ? { channelId: nextTask.dispatchScope }
+                  : undefined,
               },
-              body: JSON.stringify({ taskId: nextTaskId }),
-            }).catch((err) => {
-              console.error(
-                "[integrations] Failed to dispatch queued thread turn:",
-                err,
-              );
+              event,
+              baseUrl: getBaseUrl(event),
             });
           }
           await processDueA2AContinuations({
@@ -2018,18 +2066,21 @@ export function createIntegrationsPlugin(
             ownerEmail: control.ownerEmail,
             orgId: control.orgId,
             externalEventKey: `control:${control.id}`,
+            dispatchScope: integrationDispatchScopeValue({
+              platform: incoming.platform,
+              externalThreadId: incoming.externalThreadId,
+              platformContext: incoming.platformContext,
+            }),
           });
-          const processUrl = `${getBaseUrl(event)}${P}/process-task`;
-          const token = signInternalToken(taskId);
-          void fetch(processUrl, {
-            method: "POST",
-            headers: {
-              "Content-Type": "application/json",
-              ...(token ? { Authorization: `Bearer ${token}` } : {}),
+          await dispatchPendingIntegrationTask({
+            taskId,
+            task: {
+              platform: incoming.platform,
+              externalThreadId: incoming.externalThreadId,
+              platformContext: incoming.platformContext,
             },
-            body: JSON.stringify({ taskId }),
-          }).catch((err) => {
-            console.error("[slack] Approval dispatch failed:", err);
+            event,
+            baseUrl: getBaseUrl(event),
           });
         } catch (err) {
           console.error("[slack] Interaction handling failed:", err);
@@ -2480,6 +2531,8 @@ export function createIntegrationsPlugin(
         if (parts[0] === "remote") return;
         // Already handled by the dedicated /process-task route above
         if (parts[0] === "process-task") return;
+        // Already handled by the signed durable recovery route above
+        if (parts[0] === "retry-stuck-tasks") return;
         // Already handled by the dedicated /process-a2a-continuation route above
         if (parts[0] === "process-a2a-continuation") return;
         // These are framework-owned control-plane routes, not integration

package/src/integrations/task-queue-stats.ts

@@ -3,11 +3,12 @@
  *
  * Lives in its own file so it stays out of `pending-tasks-store.ts`, which is
  * actively being edited by the agent that owns the queue itself. These
- * helpers only SELECT — they never write — and they degrade gracefully if
- * the `integration_pending_tasks` table doesn't exist yet (returning zeroed
- * stats instead of throwing).
+ * Queue reads never expose payloads or user text. The helper first runs the
+ * additive schema guard so older deployments gain the dispatch diagnostic
+ * columns before the SELECTs execute.
  */
 import { getDbExec } from "../db/client.js";
+import { ensurePendingTasksTable } from "./pending-tasks-store.js";
 
 export interface RecentFailure {
   id: string;
@@ -23,6 +24,20 @@ export interface TaskQueueStats {
   failed_last_hour: number;
   oldest_pending_age_seconds: number;
   recent_failures: RecentFailure[];
+  recent_tasks: Array<{
+    id: string;
+    platform: string;
+    status: string;
+    attempts: number;
+    dispatch_attempts: number;
+    last_dispatch_outcome: string | null;
+    age_seconds: number;
+  }>;
+}
+
+export interface TaskQueueStatsScope {
+  ownerEmail: string;
+  orgId: string | null;
 }
 
 const ZERO_STATS: TaskQueueStats = {
@@ -32,6 +47,7 @@ const ZERO_STATS: TaskQueueStats = {
   failed_last_hour: 0,
   oldest_pending_age_seconds: 0,
   recent_failures: [],
+  recent_tasks: [],
 };
 
 function isMissingTableError(err: unknown): boolean {
@@ -44,21 +60,27 @@ function isMissingTableError(err: unknown): boolean {
 /**
  * Get a snapshot of the integration task queue health.
  *
- * Returns zeros if the table doesn't exist yet — safe to call before the
- * pending-tasks store has initialised the schema.
+ * Safe to call before the pending-tasks store has initialized the schema.
  */
-export async function getTaskQueueStats(): Promise<TaskQueueStats> {
+export async function getTaskQueueStats(
+  scope: TaskQueueStatsScope,
+): Promise<TaskQueueStats> {
+  await ensurePendingTasksTable();
   const client = getDbExec();
   const now = Date.now();
   const oneHourAgo = now - 60 * 60 * 1000;
+  const scopeSql = `owner_email = ?
+              AND ((org_id IS NULL AND CAST(? AS TEXT) IS NULL) OR org_id = ?)`;
+  const scopeArgs = [scope.ownerEmail, scope.orgId, scope.orgId];
 
   try {
     // Status counts (pending, processing) — only need the live ones.
     const liveCounts = await client.execute({
       sql: `SELECT status, COUNT(*) AS c FROM integration_pending_tasks
-            WHERE status IN ('pending', 'processing')
+            WHERE ${scopeSql}
+              AND status IN ('pending', 'processing')
             GROUP BY status`,
-      args: [],
+      args: scopeArgs,
     });
 
     let pending = 0;
@@ -75,9 +97,10 @@ export async function getTaskQueueStats(): Promise<TaskQueueStats> {
     // miss tasks queued >1h ago that just finished now.
     const lastHourCounts = await client.execute({
       sql: `SELECT status, COUNT(*) AS c FROM integration_pending_tasks
-            WHERE status IN ('completed', 'failed') AND updated_at >= ?
+            WHERE ${scopeSql}
+              AND status IN ('completed', 'failed') AND updated_at >= ?
             GROUP BY status`,
-      args: [oneHourAgo],
+      args: [...scopeArgs, oneHourAgo],
     });
 
     let completedLastHour = 0;
@@ -94,10 +117,11 @@ export async function getTaskQueueStats(): Promise<TaskQueueStats> {
     if (pending > 0) {
       const oldest = await client.execute({
         sql: `SELECT created_at FROM integration_pending_tasks
-              WHERE status = 'pending'
+              WHERE ${scopeSql}
+                AND status = 'pending'
               ORDER BY created_at ASC
               LIMIT 1`,
-        args: [],
+        args: scopeArgs,
       });
       const oldestRow = oldest.rows[0] as Record<string, unknown> | undefined;
       if (oldestRow) {
@@ -113,10 +137,11 @@ export async function getTaskQueueStats(): Promise<TaskQueueStats> {
     // blowing up the response payload.
     const failures = await client.execute({
       sql: `SELECT id, platform, error_message, attempts FROM integration_pending_tasks
-            WHERE status = 'failed' AND updated_at >= ?
+            WHERE ${scopeSql}
+              AND status = 'failed' AND updated_at >= ?
             ORDER BY updated_at DESC
             LIMIT 5`,
-      args: [oneHourAgo],
+      args: [...scopeArgs, oneHourAgo],
     });
     const recentFailures: RecentFailure[] = (
       failures.rows as Array<Record<string, unknown>>
@@ -127,6 +152,33 @@ export async function getTaskQueueStats(): Promise<TaskQueueStats> {
       attempts: Number(row.attempts ?? 0),
     }));
 
+    const recent = await client.execute({
+      sql: `SELECT id, platform, status, attempts, dispatch_attempts,
+                   last_dispatch_outcome, created_at
+              FROM integration_pending_tasks
+             WHERE ${scopeSql}
+             ORDER BY created_at DESC
+             LIMIT 10`,
+      args: scopeArgs,
+    });
+    const recentTasks = (recent.rows as Array<Record<string, unknown>>).map(
+      (row) => ({
+        id: String(row.id ?? ""),
+        platform: String(row.platform ?? ""),
+        status: String(row.status ?? ""),
+        attempts: Number(row.attempts ?? 0),
+        dispatch_attempts: Number(row.dispatch_attempts ?? 0),
+        last_dispatch_outcome:
+          row.last_dispatch_outcome == null
+            ? null
+            : String(row.last_dispatch_outcome),
+        age_seconds: Math.max(
+          0,
+          Math.floor((now - Number(row.created_at ?? now)) / 1000),
+        ),
+      }),
+    );
+
     return {
       pending,
       processing,
@@ -134,10 +186,11 @@ export async function getTaskQueueStats(): Promise<TaskQueueStats> {
       failed_last_hour: failedLastHour,
       oldest_pending_age_seconds: oldestPendingAgeSeconds,
       recent_failures: recentFailures,
+      recent_tasks: recentTasks,
     };
   } catch (err) {
     if (isMissingTableError(err)) {
-      return { ...ZERO_STATS, recent_failures: [] };
+      return { ...ZERO_STATS, recent_failures: [], recent_tasks: [] };
     }
     throw err;
   }

package/src/integrations/webhook-handler.ts

@@ -46,7 +46,6 @@ import { updateThreadData } from "../chat-threads/store.js";
 import { isLocalDatabase } from "../db/client.js";
 import { getOrgA2ASecret, resolveOrgIdForEmail } from "../org/context.js";
 import { withConfiguredAppBasePath } from "../server/app-base-path.js";
-import { FRAMEWORK_ROUTE_PREFIX } from "../server/core-routes-plugin.js";
 import {
   canUseDeployCredentialFallbackForRequest,
   readDeployCredentialEnv,
@@ -58,8 +57,11 @@ import {
   clearIntegrationAwaitingInput,
   setIntegrationAwaitingInput,
 } from "./awaiting-input-store.js";
+import {
+  dispatchPendingIntegrationTask,
+  integrationDispatchScopeValue,
+} from "./integration-durable-dispatch.js";
 import { loadIntegrationMemoryPrompt } from "./integration-memory.js";
-import { signInternalToken } from "./internal-token.js";
 import {
   insertPendingTask,
   isDuplicateEventError,
@@ -307,7 +309,7 @@ export async function resolveIntegrationApiKey(
  * 2. Verify webhook signature
  * 3. Parse incoming message (null = ignored event)
  * 4. Persist task to SQL
- * 5. Fire-and-forget POST to /_agent-native/integrations/process-task
+ * 5. Dispatch the queued task through the configured processor handoff
  *    (a fresh function execution with its own timeout budget)
  * 6. Return HTTP 200 immediately (within Slack's 3s SLA)
  *
@@ -477,48 +479,14 @@ async function enqueueAndDispatch(
     // platform produce the same key, so the unique index rejects the
     // second insert (H3 in the webhook security audit).
     externalEventKey: buildEventDedupKey(incoming),
+    dispatchScope: integrationDispatchScopeValue({
+      platform: incoming.platform,
+      externalThreadId: incoming.externalThreadId,
+      platformContext: incoming.platformContext,
+    }),
   });
 
   const baseUrl = resolveBaseUrl(event);
-  const processUrl = `${baseUrl}${FRAMEWORK_ROUTE_PREFIX}/integrations/process-task`;
-
-  // Sign the dispatch with an HMAC token so the processor endpoint can
-  // verify the request came from us and not the public internet. The
-  // processor refuses unsigned requests in production (C3 in the webhook
-  // security audit). In dev, dispatching unsigned is allowed and falls
-  // through to the SQL atomic claim for double-processing protection.
-  const headers: Record<string, string> = {
-    "Content-Type": "application/json",
-  };
-  try {
-    headers["Authorization"] = `Bearer ${signInternalToken(taskId)}`;
-  } catch (err) {
-    // Distinguish "secret not configured" (the documented dev path) from
-    // a real signing failure — silently swallowing both made malformed
-    // secrets fail invisibly (L5 in the audit).
-    if (err instanceof Error && !/A2A_SECRET/i.test(err.message)) {
-      console.error(
-        `[integrations] signInternalToken failed unexpectedly for ${taskId}:`,
-        err,
-      );
-    }
-  }
-
-  // Fire-and-forget: do NOT await the full response (the processor's run
-  // takes minutes — we don't want to block the caller). BUT on Netlify
-  // Lambda, when we return immediately, the runtime can freeze the function
-  // before the outbound TCP handshake even starts, which leaves the dispatch
-  // request stuck waiting for the 60s retry-sweep job. Race the fetch
-  // against a short timer so the request gets a reasonable chance to leave
-  // the box; the trade-off is at most a couple seconds of added webhook
-  // latency, still inside Slack's timeout window.
-  const dispatchPromise = fetch(processUrl, {
-    method: "POST",
-    headers,
-    body: JSON.stringify({ taskId }),
-  }).catch((err) => {
-    console.error("[integrations] Failed to dispatch processor request:", err);
-  });
   const settleWaitMs = options.adapter.capabilities?.deferredWebhookResponse
     ? Math.min(
         DEFERRED_RESPONSE_DISPATCH_SETTLE_WAIT_MS,
@@ -528,10 +496,17 @@ async function enqueueAndDispatch(
         ),
       )
     : PROCESSOR_DISPATCH_SETTLE_WAIT_MS;
-  await Promise.race([
-    dispatchPromise,
-    new Promise<void>((resolve) => setTimeout(resolve, settleWaitMs)),
-  ]);
+  await dispatchPendingIntegrationTask({
+    taskId,
+    task: {
+      platform: incoming.platform,
+      externalThreadId: incoming.externalThreadId,
+      platformContext: incoming.platformContext,
+    },
+    event,
+    baseUrl,
+    portableSettleMs: settleWaitMs,
+  });
 }
 
 /**

package/src/server/auth.ts

@@ -1740,6 +1740,12 @@ function createAuthGuardFn(): (
       return;
     }
 
+    // External durable-recovery scheduler. The route verifies a short-lived
+    // HMAC token bound to its fixed sweep subject before touching the queue.
+    if (p === "/_agent-native/integrations/retry-stuck-tasks") {
+      return;
+    }
+
     // Internal processor endpoint for deferred A2A continuations created by
     // integration tasks. It uses the same HMAC internal-token scheme as the
     // primary integration processor, so it must bypass cookie/session auth.