@agent-native/core 0.119.5 → 0.120.0

package/corpus/core/src/integrations/integration-durable-dispatch.ts

@@ -0,0 +1,190 @@
+import { hasConfiguredA2ASecret } from "../a2a/auth-policy.js";
+import {
+  AGENT_BACKGROUND_PROCESSOR_FIELD,
+  AGENT_BACKGROUND_PROCESSOR_INTEGRATION,
+  dispatchPathTargetsNetlifyBackgroundFunction,
+  resolveDurableBackgroundDispatchPath,
+} from "../agent/durable-background.js";
+import { fireInternalDispatch } from "../server/self-dispatch.js";
+import {
+  INTEGRATION_DURABLE_DISPATCH_ENV,
+  INTEGRATION_DURABLE_DISPATCH_SCOPES_ENV,
+  INTEGRATION_PROCESS_TASK_PATH,
+  INTEGRATION_RETRY_SWEEP_PATH,
+  INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT,
+  isIntegrationDurableDispatchConfigured,
+} from "./integration-durable-dispatch-config.js";
+import { recordPendingTaskDispatchAttempt } from "./pending-tasks-store.js";
+
+export {
+  INTEGRATION_DURABLE_DISPATCH_ENV,
+  INTEGRATION_DURABLE_DISPATCH_SCOPES_ENV,
+  INTEGRATION_PROCESS_TASK_PATH,
+  INTEGRATION_RETRY_SWEEP_PATH,
+  INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT,
+  isIntegrationDurableDispatchConfigured,
+};
+
+export type IntegrationDispatchOutcome =
+  | "background-acknowledged"
+  | "portable-unconfirmed"
+  | "failed";
+
+export interface IntegrationDispatchTaskScope {
+  platform: string;
+  externalThreadId: string;
+  platformContext?: Record<string, unknown>;
+}
+
+export interface IntegrationDurableDispatchScope {
+  platform: string;
+  value: string;
+}
+
+export function integrationDispatchScopeValue(
+  task: IntegrationDispatchTaskScope,
+): string | null {
+  const value = task.platformContext?.channelId;
+  return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+export function configuredIntegrationDurableDispatchScopes():
+  | IntegrationDurableDispatchScope[]
+  | null {
+  const raw = process.env.AGENT_INTEGRATION_DURABLE_DISPATCH_SCOPES;
+  if (!raw?.trim()) return null;
+  return raw
+    .split(",")
+    .map((value) => value.trim())
+    .filter(Boolean)
+    .map((scope) => {
+      const separator = scope.indexOf(":");
+      return {
+        platform: separator > 0 ? scope.slice(0, separator) : "",
+        value: separator > 0 ? scope.slice(separator + 1) : "",
+      };
+    })
+    .filter((scope) => scope.platform && scope.value);
+}
+
+function taskScopeCandidates(task: IntegrationDispatchTaskScope): string[] {
+  const candidates = [
+    `${task.platform}:*`,
+    `${task.platform}:${task.externalThreadId}`,
+  ];
+  const explicitScope = integrationDispatchScopeValue(task);
+  if (explicitScope) {
+    candidates.push(`${task.platform}:${explicitScope}`);
+  } else if (task.platform === "slack") {
+    const channelId = task.externalThreadId.split(":")[2];
+    if (channelId) candidates.push(`slack:${channelId}`);
+  }
+  return candidates;
+}
+
+export function isIntegrationDurableDispatchEnabledForTask(
+  task: IntegrationDispatchTaskScope,
+): boolean {
+  if (!isIntegrationDurableDispatchConfigured()) return false;
+  const scopes = configuredIntegrationDurableDispatchScopes();
+  if (
+    scopes &&
+    !taskScopeCandidates(task).some((candidate) =>
+      scopes.some((scope) => `${scope.platform}:${scope.value}` === candidate),
+    )
+  ) {
+    return false;
+  }
+  const path = resolveDurableBackgroundDispatchPath(
+    INTEGRATION_PROCESS_TASK_PATH,
+  );
+  return (
+    dispatchPathTargetsNetlifyBackgroundFunction(path) &&
+    hasConfiguredA2ASecret()
+  );
+}
+
+async function recordDispatch(
+  taskId: string,
+  outcome: IntegrationDispatchOutcome,
+): Promise<void> {
+  try {
+    await recordPendingTaskDispatchAttempt(taskId, outcome);
+  } catch (error) {
+    console.error(
+      `[integrations] Failed to record dispatch outcome for ${taskId}:`,
+      error,
+    );
+  }
+}
+
+function logDispatch(
+  taskId: string,
+  outcome: IntegrationDispatchOutcome,
+  startedAt: number,
+): void {
+  console.info("[integrations] pending task dispatch", {
+    taskId,
+    outcome,
+    durationMs: Date.now() - startedAt,
+  });
+}
+
+export async function dispatchPendingIntegrationTask(input: {
+  taskId: string;
+  task: IntegrationDispatchTaskScope;
+  event?: unknown;
+  baseUrl?: string;
+  portableSettleMs?: number;
+}): Promise<IntegrationDispatchOutcome> {
+  const startedAt = Date.now();
+  const durable = isIntegrationDurableDispatchEnabledForTask(input.task);
+  if (durable) {
+    const backgroundPath = resolveDurableBackgroundDispatchPath(
+      INTEGRATION_PROCESS_TASK_PATH,
+    );
+    try {
+      await fireInternalDispatch({
+        event: input.event,
+        baseUrl: input.baseUrl,
+        path: backgroundPath,
+        taskId: input.taskId,
+        body: {
+          [AGENT_BACKGROUND_PROCESSOR_FIELD]:
+            AGENT_BACKGROUND_PROCESSOR_INTEGRATION,
+        },
+        awaitResponse: true,
+        responseTimeoutMs: 2_000,
+      });
+      await recordDispatch(input.taskId, "background-acknowledged");
+      logDispatch(input.taskId, "background-acknowledged", startedAt);
+      return "background-acknowledged";
+    } catch (error) {
+      console.error(
+        `[integrations] Background dispatch failed for ${input.taskId}; trying the portable processor:`,
+        error,
+      );
+    }
+  }
+
+  try {
+    await fireInternalDispatch({
+      event: input.event,
+      baseUrl: input.baseUrl,
+      path: INTEGRATION_PROCESS_TASK_PATH,
+      taskId: input.taskId,
+      settleMs: input.portableSettleMs,
+    });
+    await recordDispatch(input.taskId, "portable-unconfirmed");
+    logDispatch(input.taskId, "portable-unconfirmed", startedAt);
+    return "portable-unconfirmed";
+  } catch (error) {
+    console.error(
+      `[integrations] Portable dispatch failed for ${input.taskId}:`,
+      error,
+    );
+    await recordDispatch(input.taskId, "failed");
+    logDispatch(input.taskId, "failed", startedAt);
+    return "failed";
+  }
+}

package/corpus/core/src/integrations/pending-tasks-retry-job.ts

@@ -1,8 +1,13 @@
 import { getDbExec } from "../db/client.js";
-import { withConfiguredAppBasePath } from "../server/app-base-path.js";
-import { FRAMEWORK_ROUTE_PREFIX } from "../server/core-routes-plugin.js";
-import { signInternalToken } from "./internal-token.js";
-import { MAX_PENDING_TASK_ATTEMPTS } from "./pending-tasks-store.js";
+import {
+  configuredIntegrationDurableDispatchScopes,
+  dispatchPendingIntegrationTask,
+  isIntegrationDurableDispatchEnabledForTask,
+} from "./integration-durable-dispatch.js";
+import {
+  ensurePendingTasksTable,
+  MAX_PENDING_TASK_ATTEMPTS,
+} from "./pending-tasks-store.js";
 
 /**
  * Retries stuck integration webhook tasks.
@@ -10,12 +15,16 @@ import { MAX_PENDING_TASK_ATTEMPTS } from "./pending-tasks-store.js";
  * The integration webhook flow enqueues work into `integration_pending_tasks`
  * (see `pending-tasks-store.ts`) and then fires a self-webhook to the
  * `/_agent-native/integrations/process-task` endpoint to drain the queue.
- * If that fire-and-forget dispatch fails (e.g. transient network blip), the
+ * If that initial dispatch is lost (e.g. transient network blip), the
  * row stays in `pending` forever. Likewise, if the processor is killed mid-
  * processing (function timeout, container shutdown), a row can remain in
  * `processing` forever.
  *
- * This job runs every 60s and re-fires the processor endpoint for tasks that
+ * The in-process fallback runs every 60s. Durable deployments also invoke the
+ * same bounded sweep from an external scheduler so recovery does not depend on
+ * a serverless process remaining alive.
+ *
+ * Each sweep re-fires the processor endpoint for tasks that
  * look stuck:
  *   - status='pending' AND created_at older than 90s (initial dispatch lost)
  *   - status='processing' AND updated_at older than the host-specific
@@ -35,8 +44,9 @@ const PENDING_STUCK_AFTER_MS = 90_000;
 /** Tasks "processing" longer than this are considered killed mid-flight. */
 const DEFAULT_PROCESSING_STUCK_AFTER_MS = 5 * 60 * 1000;
 const SERVERLESS_PROCESSING_STUCK_AFTER_MS = 75_000;
+const DURABLE_BACKGROUND_PROCESSING_STUCK_AFTER_MS = 16 * 60 * 1000;
 /** After this many attempts we give up and mark the task failed */
-const PROCESSOR_PATH = `${FRAMEWORK_ROUTE_PREFIX}/integrations/process-task`;
+const DEFAULT_SWEEP_LIMIT = 100;
 
 let retryInterval: ReturnType<typeof setInterval> | null = null;
 let initialTimer: ReturnType<typeof setTimeout> | null = null;
@@ -49,8 +59,74 @@ let tableExists: boolean | null = null;
 
 interface StuckTaskRow {
   id: string;
+  platform: string;
+  externalThreadId: string;
   status: string;
   attempts: number;
+  updatedAt: number;
+  dispatchScope: string | null;
+}
+
+export interface PendingTasksSweepResult {
+  selected: number;
+  dispatched: number;
+  markedFailed: number;
+  skipped: number;
+  dispatchFailed: number;
+}
+
+export interface PendingTasksSweepOptions {
+  webhookBaseUrl?: string;
+  limit?: number;
+  durableOnly?: boolean;
+}
+
+function affectedRows(result: unknown): number {
+  return Number(
+    (result as { rowsAffected?: number }).rowsAffected ??
+      (result as { rowCount?: number }).rowCount ??
+      0,
+  );
+}
+
+function isMissingPendingTasksTableError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error ?? "");
+  return /no such table|relation .* does not exist|undefined_table/i.test(
+    message,
+  );
+}
+
+function durableScopeSql(): { clause: string; args: string[] } {
+  const scopes = configuredIntegrationDurableDispatchScopes();
+  if (scopes === null) return { clause: "", args: [] };
+  if (scopes.length === 0) return { clause: " AND 1 = 0", args: [] };
+
+  const clauses: string[] = [];
+  const args: string[] = [];
+  for (const scope of scopes) {
+    if (scope.value === "*") {
+      clauses.push("platform = ?");
+      args.push(scope.platform);
+      continue;
+    }
+    if (scope.platform === "slack" && /^[A-Za-z0-9]+$/.test(scope.value)) {
+      clauses.push(
+        "(platform = ? AND (external_thread_id = ? OR dispatch_scope = ? OR external_thread_id LIKE ?))",
+      );
+      args.push(
+        scope.platform,
+        scope.value,
+        scope.value,
+        `%:%:${scope.value}:%`,
+      );
+      continue;
+    }
+    clauses.push(
+      "(platform = ? AND (external_thread_id = ? OR dispatch_scope = ?))",
+    );
+    args.push(scope.platform, scope.value, scope.value);
+  }
+  return { clause: ` AND (${clauses.join(" OR ")})`, args };
 }
 
 /**
@@ -58,39 +134,76 @@ interface StuckTaskRow {
  * Exported for tests and for manual triggers.
  */
 export async function retryStuckPendingTasks(
-  webhookBaseUrl?: string,
-): Promise<void> {
-  const baseUrl = webhookBaseUrl ?? activeWebhookBaseUrl;
+  input?: string | PendingTasksSweepOptions,
+): Promise<PendingTasksSweepResult> {
+  const options: PendingTasksSweepOptions =
+    typeof input === "string" ? { webhookBaseUrl: input } : (input ?? {});
+  const baseUrl = options.webhookBaseUrl ?? activeWebhookBaseUrl;
+  const limit = Math.max(
+    1,
+    Math.min(500, Math.floor(options.limit ?? DEFAULT_SWEEP_LIMIT)),
+  );
+  const scopeSql = options.durableOnly
+    ? durableScopeSql()
+    : { clause: "", args: [] as string[] };
+  const result: PendingTasksSweepResult = {
+    selected: 0,
+    dispatched: 0,
+    markedFailed: 0,
+    skipped: 0,
+    dispatchFailed: 0,
+  };
+  await ensurePendingTasksTable();
   const client = getDbExec();
   const now = Date.now();
   const pendingCutoff = now - PENDING_STUCK_AFTER_MS;
   const processingCutoff = now - getProcessingStuckAfterMs();
+  const durableProcessingCutoff =
+    now - DURABLE_BACKGROUND_PROCESSING_STUCK_AFTER_MS;
 
   let stuckRows: StuckTaskRow[];
   try {
     const { rows } = await client.execute({
       sql: `
-        SELECT id, status, attempts
+        SELECT id, platform, external_thread_id, dispatch_scope, status, attempts, updated_at
           FROM integration_pending_tasks
-         WHERE (status = 'pending' AND created_at <= ? AND updated_at <= ?)
-            OR (status = 'processing' AND updated_at <= ?)
+         WHERE ((status = 'pending' AND created_at <= ? AND updated_at <= ?)
+            OR (status = 'processing' AND (
+              (last_dispatch_outcome = 'background-acknowledged'
+                AND updated_at <= ?)
+              OR ((last_dispatch_outcome IS NULL
+                    OR last_dispatch_outcome <> 'background-acknowledged')
+                AND updated_at <= ?)
+            )))
+         ${scopeSql.clause}
+         ORDER BY updated_at ASC
+         LIMIT ?
       `,
       // `updated_at` is initialized to `created_at` on insert, so a genuinely
       // stuck pending row still matches on the first sweep. The retry path
       // below touches `updated_at`, which (with this predicate) keeps the row
       // from being re-selected — and re-firing the processor — on every tick.
-      args: [pendingCutoff, pendingCutoff, processingCutoff],
+      args: [
+        pendingCutoff,
+        pendingCutoff,
+        durableProcessingCutoff,
+        processingCutoff,
+        ...scopeSql.args,
+        limit,
+      ],
     });
     stuckRows = rows.map((r) => ({
       id: r.id as string,
+      platform: r.platform as string,
+      externalThreadId: r.external_thread_id as string,
       status: r.status as string,
       attempts: Number(r.attempts ?? 0),
+      updatedAt: Number(r.updated_at ?? 0),
+      dispatchScope: (r.dispatch_scope as string | null) ?? null,
     }));
     tableExists = true;
   } catch (err) {
-    // Most common case: the table hasn't been created yet because no inbound
-    // integration webhook has been processed on this deployment. Silently
-    // no-op until the table appears.
+    if (!isMissingPendingTasksTableError(err)) throw err;
     if (tableExists !== false) {
       tableExists = false;
       if (process.env.DEBUG) {
@@ -99,17 +212,29 @@ export async function retryStuckPendingTasks(
         );
       }
     }
-    return;
+    return result;
   }
 
-  if (stuckRows.length === 0) return;
+  stuckRows = stuckRows.filter((row) => {
+    const durable = isIntegrationDurableDispatchEnabledForTask({
+      platform: row.platform,
+      externalThreadId: row.externalThreadId,
+      platformContext: row.dispatchScope
+        ? { channelId: row.dispatchScope }
+        : undefined,
+    });
+    if (options.durableOnly && !durable) return false;
+    return true;
+  });
+  result.selected = stuckRows.length;
+  if (stuckRows.length === 0) return result;
 
   for (const row of stuckRows) {
     try {
       // Cap retries — mark failed and move on so the row stops bouncing
       // between pending and processing forever.
       if (row.attempts >= MAX_PENDING_TASK_ATTEMPTS) {
-        await client.execute({
+        const update = await client.execute({
           sql: `
             UPDATE integration_pending_tasks
                SET status = 'failed',
@@ -119,14 +244,21 @@ export async function retryStuckPendingTasks(
                    external_event_key = NULL
              WHERE id = ?
                AND status = ?
+               AND updated_at = ?
           `,
           args: [
             Date.now(),
             `Retry job: exceeded ${MAX_PENDING_TASK_ATTEMPTS} attempts`,
             row.id,
             row.status,
+            row.updatedAt,
           ],
         });
+        if (affectedRows(update) === 0) {
+          result.skipped += 1;
+          continue;
+        }
+        result.markedFailed += 1;
         console.warn(
           `[integrations] Pending task ${row.id} exceeded ${MAX_PENDING_TASK_ATTEMPTS} attempts — marking failed`,
         );
@@ -138,24 +270,44 @@ export async function retryStuckPendingTasks(
       // Without this, processing rows stay stuck forever.
       // For pending rows, just touch updated_at to avoid re-firing every tick.
       const newStatus = row.status === "processing" ? "pending" : row.status;
-      await client.execute({
+      const update = await client.execute({
         sql: `
           UPDATE integration_pending_tasks
              SET status = ?, updated_at = ?
            WHERE id = ?
              AND status = ?
+             AND updated_at = ?
         `,
-        args: [newStatus, Date.now(), row.id, row.status],
+        args: [newStatus, Date.now(), row.id, row.status, row.updatedAt],
       });
+      if (affectedRows(update) === 0) {
+        result.skipped += 1;
+        continue;
+      }
 
-      await refireProcessor(row.id, baseUrl);
+      const outcome = await dispatchPendingIntegrationTask({
+        taskId: row.id,
+        task: {
+          platform: row.platform,
+          externalThreadId: row.externalThreadId,
+          platformContext: row.dispatchScope
+            ? { channelId: row.dispatchScope }
+            : undefined,
+        },
+        baseUrl,
+        portableSettleMs: 1_000,
+      });
+      if (outcome === "failed") result.dispatchFailed += 1;
+      else result.dispatched += 1;
     } catch (err) {
+      result.dispatchFailed += 1;
       console.error(
         `[integrations] Failed to retry pending task ${row.id}:`,
         err,
       );
     }
   }
+  return result;
 }
 
 function getProcessingStuckAfterMs(): number {
@@ -170,70 +322,6 @@ function getProcessingStuckAfterMs(): number {
   return DEFAULT_PROCESSING_STUCK_AFTER_MS;
 }
 
-/**
- * Fire-and-forget POST to the processor endpoint for a single task id.
- * Mirrors the original dispatch from the webhook handler, including the
- * short-lived HMAC bearer token bound to this taskId.
- */
-async function refireProcessor(
-  taskId: string,
-  webhookBaseUrl: string | undefined,
-): Promise<void> {
-  const baseUrl =
-    webhookBaseUrl ||
-    process.env.WEBHOOK_BASE_URL ||
-    process.env.APP_URL ||
-    process.env.URL ||
-    process.env.DEPLOY_URL ||
-    `http://localhost:${process.env.PORT || 3000}`;
-
-  const url = `${withConfiguredAppBasePath(baseUrl)}${PROCESSOR_PATH}`;
-
-  // Sign with HMAC if A2A_SECRET is configured. In production we MUST sign —
-  // an unsigned dispatch in production lets attackers re-trigger any queued
-  // task with a guessable id (C3 in the webhook security audit). In dev we
-  // fall back to unsigned so contributors can iterate without configuring
-  // A2A_SECRET locally.
-  const headers: Record<string, string> = {
-    "Content-Type": "application/json",
-  };
-  try {
-    headers["Authorization"] = `Bearer ${signInternalToken(taskId)}`;
-  } catch (err) {
-    if (process.env.NODE_ENV === "production") {
-      console.error(
-        `[integrations] Refusing to dispatch task ${taskId} — A2A_SECRET not configured. ` +
-          "Set A2A_SECRET to enable signed retry dispatches.",
-      );
-      return;
-    }
-    // Dev: proceed unsigned. Log the underlying error path so a malformed
-    // secret (different from "not set") doesn't fail silently (L5 in the audit).
-    if (err instanceof Error && !/A2A_SECRET/i.test(err.message)) {
-      console.error(
-        `[integrations] signInternalToken failed unexpectedly for ${taskId}:`,
-        err,
-      );
-    }
-  }
-
-  // Don't await the body — we just want the request to leave the box.
-  // A short timeout avoids tying up the retry loop on a hung processor.
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), 5_000);
-
-  try {
-    await fetch(url, {
-      method: "POST",
-      headers,
-      body: JSON.stringify({ taskId }),
-      signal: controller.signal,
-    });
-  } finally {
-    clearTimeout(timer);
-  }
-}
-
 /**
  * Start the periodic retry loop. Safe to call multiple times — second call
  * is a no-op.