@agent-native/core 0.130.0 → 0.130.1

package/src/integrations/adapters/slack.ts

@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto";
+
 import type { H3Event } from "h3";
 import { createError, getHeader, readRawBody } from "h3";
 
@@ -24,6 +26,7 @@ import type {
   PlatformRunProgress,
   PlatformRunProgressRef,
   PlatformDeliveryReceipt,
+  PlatformDeliveryOptions,
   IntegrationContextMessage,
   IntegrationFileReference,
 } from "../types.js";
@@ -47,6 +50,9 @@ const SLACK_IDENTITY_NEGATIVE_CACHE_TTL_MS = 30 * 1_000;
 const SLACK_IDENTITY_CACHE_MAX_ENTRIES = 1_000;
 const SLACK_TOKEN_IDENTITY_CACHE_TTL_MS = 10 * 60 * 1_000;
 const SLACK_TOKEN_IDENTITY_NEGATIVE_CACHE_TTL_MS = 30 * 1_000;
+const SLACK_DELIVERY_RECONCILIATION_PAGE_LIMIT = 200;
+const SLACK_DELIVERY_RECONCILIATION_MAX_PAGES = 3;
+const SLACK_DELIVERY_MARKER_PREFIX = "agent_native_terminal_";
 
 type SlackTokenIdentity = {
   teamId: string | null;
@@ -441,7 +447,7 @@ export function slackAdapter(
     async sendResponse(
       message: OutgoingMessage,
       context: IncomingMessage,
-      opts?: { placeholderRef?: string },
+      opts?: PlatformDeliveryOptions,
     ): Promise<void | PlatformDeliveryReceipt> {
       const token = await resolveBotToken(context);
       if (!token) {
@@ -471,6 +477,22 @@ export function slackAdapter(
       }
       const restChunks = chunks.slice(1);
       const messageRefs: string[] = [];
+      const freshChunkIndexes = [
+        ...(!placeholderRef ? [0] : []),
+        ...restChunks.map((_, index) => index + 1),
+      ];
+      const reconciledRefs =
+        opts?.idempotencyKey && freshChunkIndexes.length > 0
+          ? await reconcileSlackDeliveryChunks(
+              token,
+              channelId,
+              threadTs,
+              opts.idempotencyKey,
+              freshChunkIndexes,
+              opts.reconcileAfter,
+              opts.signal,
+            )
+          : new Map<number, string>();
 
       const finalBlocks =
         blocks ??
@@ -491,40 +513,57 @@ export function slackAdapter(
       try {
         if (placeholderRef) {
           // Replace the "thinking…" placeholder in place.
-          const res = await slackApiFetch("https://slack.com/api/chat.update", {
-            method: "POST",
-            headers: {
-              Authorization: `Bearer ${token}`,
-              "Content-Type": "application/json",
+          const data = (await slackApiJson(
+            "https://slack.com/api/chat.update",
+            {
+              method: "POST",
+              headers: {
+                Authorization: `Bearer ${token}`,
+                "Content-Type": "application/json",
+              },
+              body: JSON.stringify({ ...baseBody, ts: placeholderRef }),
+              signal: opts?.signal,
             },
-            body: JSON.stringify({ ...baseBody, ts: placeholderRef }),
-          });
-          const data = (await res.json()) as {
+          )) as {
             ok: boolean;
             error?: string;
             ts?: string;
           };
           if (!data.ok) {
             console.error("[slack] chat.update error:", data.error);
+            if (opts?.strictTargetRef) {
+              throw new Error(data.error || "chat.update failed");
+            }
             // Fall back to a fresh post so the user still sees a reply
             const postedTs = await postFresh(
               token,
               channelId,
               threadTs,
-              baseBody,
+              opts?.idempotencyKey
+                ? withSlackDeliveryMarker(baseBody, opts.idempotencyKey, 0)
+                : baseBody,
+              opts?.signal,
             );
             if (postedTs) messageRefs.push(postedTs);
           } else {
             messageRefs.push(data.ts || placeholderRef);
           }
         } else {
-          const postedTs = await postFresh(
-            token,
-            channelId,
-            threadTs,
-            baseBody,
-          );
-          if (postedTs) messageRefs.push(postedTs);
+          const reconciledRef = reconciledRefs.get(0);
+          if (reconciledRef) {
+            messageRefs.push(reconciledRef);
+          } else {
+            const postedTs = await postFresh(
+              token,
+              channelId,
+              threadTs,
+              opts?.idempotencyKey
+                ? withSlackDeliveryMarker(baseBody, opts.idempotencyKey, 0)
+                : baseBody,
+              opts?.signal,
+            );
+            if (postedTs) messageRefs.push(postedTs);
+          }
         }
 
         // Clear the AI-assistant "is thinking…" status now that we've
@@ -534,14 +573,33 @@ export function slackAdapter(
         }
 
         // Overflow chunks (rare) — post as plain follow-ups in the same thread
-        for (const chunk of restChunks) {
-          const postedTs = await postFresh(token, channelId, threadTs, {
+        for (const [index, chunk] of restChunks.entries()) {
+          const chunkIndex = index + 1;
+          const reconciledRef = reconciledRefs.get(chunkIndex);
+          if (reconciledRef) {
+            messageRefs.push(reconciledRef);
+            continue;
+          }
+          const overflowBody: Record<string, unknown> = {
             channel: channelId,
             text: chunk,
             unfurl_links: false,
             unfurl_media: false,
             mrkdwn: true,
-          });
+          };
+          const postedTs = await postFresh(
+            token,
+            channelId,
+            threadTs,
+            opts?.idempotencyKey
+              ? withSlackDeliveryMarker(
+                  overflowBody,
+                  opts.idempotencyKey,
+                  chunkIndex,
+                )
+              : overflowBody,
+            opts?.signal,
+          );
           if (postedTs) messageRefs.push(postedTs);
         }
         return {
@@ -628,7 +686,7 @@ export function slackAdapter(
         if (target.threadRef) body.thread_ts = target.threadRef;
 
         try {
-          const res = await slackApiFetch(
+          const data = (await slackApiJson(
             "https://slack.com/api/chat.postMessage",
             {
               method: "POST",
@@ -638,8 +696,7 @@ export function slackAdapter(
               },
               body: JSON.stringify(body),
             },
-          );
-          const data = (await res.json()) as { ok: boolean; error?: string };
+          )) as { ok: boolean; error?: string };
           if (!data.ok) {
             throw new Error(data.error || "chat.postMessage failed");
           }
@@ -1037,7 +1094,7 @@ function setSlackAssistantStatus(
   threadTs: string,
   status: string,
 ): void {
-  slackApiFetch("https://slack.com/api/assistant.threads.setStatus", {
+  slackApiJson("https://slack.com/api/assistant.threads.setStatus", {
     method: "POST",
     headers: {
       Authorization: `Bearer ${token}`,
@@ -1094,6 +1151,7 @@ async function postFresh(
   channelId: string,
   threadTs: string | undefined,
   body: Record<string, unknown>,
+  signal?: AbortSignal,
 ): Promise<string | undefined> {
   const hasBlocks =
     Array.isArray(body.blocks) && (body.blocks as unknown[]).length > 0;
@@ -1110,15 +1168,15 @@ async function postFresh(
     channel: channelId,
   };
   if (threadTs && !payload.thread_ts) payload.thread_ts = threadTs;
-  const res = await slackApiFetch("https://slack.com/api/chat.postMessage", {
+  const data = (await slackApiJson("https://slack.com/api/chat.postMessage", {
     method: "POST",
     headers: {
       Authorization: `Bearer ${token}`,
       "Content-Type": "application/json",
     },
     body: JSON.stringify(payload),
-  });
-  const data = (await res.json()) as {
+    signal,
+  })) as {
     ok: boolean;
     error?: string;
     ts?: string;
@@ -1130,23 +1188,145 @@ async function postFresh(
   return data.ts;
 }
 
-async function slackApiFetch(
+function slackDeliveryMarker(key: string, chunkIndex: number): string {
+  const digest = createHash("sha256")
+    .update(`${key}:${chunkIndex}`)
+    .digest("hex")
+    .slice(0, 32);
+  return `${SLACK_DELIVERY_MARKER_PREFIX}${digest}`;
+}
+
+function withSlackDeliveryMarker(
+  body: Record<string, unknown>,
+  key: string,
+  chunkIndex: number,
+): Record<string, unknown> {
+  const marker = slackDeliveryMarker(key, chunkIndex);
+  const blocks = Array.isArray(body.blocks) ? body.blocks : [];
+  if (blocks.length > 0) {
+    const [first, ...rest] = blocks;
+    if (first && typeof first === "object" && !Array.isArray(first)) {
+      return {
+        ...body,
+        blocks: [
+          { ...(first as Record<string, unknown>), block_id: marker },
+          ...rest,
+        ],
+      };
+    }
+  }
+  return {
+    ...body,
+    blocks: [
+      {
+        type: "section",
+        block_id: marker,
+        text: { type: "mrkdwn", text: String(body.text ?? "") },
+      },
+    ],
+  };
+}
+
+async function reconcileSlackDeliveryChunks(
+  token: string,
+  channelId: string,
+  threadTs: string | undefined,
+  key: string,
+  chunkIndexes: number[],
+  reconcileAfter?: number,
+  signal?: AbortSignal,
+): Promise<Map<number, string>> {
+  if (!threadTs) {
+    throw new Error(
+      "Cannot reconcile an idempotent Slack delivery without a thread timestamp",
+    );
+  }
+  const refs = new Map<number, string>();
+  const expectedMarkers = new Map(
+    chunkIndexes.map((chunkIndex) => [
+      slackDeliveryMarker(key, chunkIndex),
+      chunkIndex,
+    ]),
+  );
+  let cursor = "";
+  for (
+    let page = 0;
+    page < SLACK_DELIVERY_RECONCILIATION_MAX_PAGES;
+    page += 1
+  ) {
+    const url = new URL("https://slack.com/api/conversations.replies");
+    url.searchParams.set("channel", channelId);
+    url.searchParams.set("ts", threadTs);
+    url.searchParams.set(
+      "limit",
+      String(SLACK_DELIVERY_RECONCILIATION_PAGE_LIMIT),
+    );
+    if (cursor) url.searchParams.set("cursor", cursor);
+    if (typeof reconcileAfter === "number" && reconcileAfter > 0) {
+      url.searchParams.set("oldest", String(Math.floor(reconcileAfter / 1000)));
+      url.searchParams.set("inclusive", "true");
+    }
+    const body = (await slackApiJson(url.toString(), {
+      headers: { Authorization: `Bearer ${token}` },
+      signal,
+    })) as {
+      ok: boolean;
+      error?: string;
+      messages?: Array<{
+        ts?: string;
+        blocks?: Array<{ block_id?: string }>;
+      }>;
+      response_metadata?: { next_cursor?: string };
+    };
+    if (!body.ok) {
+      throw new Error(body.error || "conversations.replies failed");
+    }
+    for (const message of body.messages ?? []) {
+      if (typeof message.ts !== "string") continue;
+      for (const block of message.blocks ?? []) {
+        if (typeof block.block_id !== "string") continue;
+        const chunkIndex = expectedMarkers.get(block.block_id);
+        if (chunkIndex !== undefined) refs.set(chunkIndex, message.ts);
+      }
+    }
+    if (refs.size === expectedMarkers.size) return refs;
+    cursor = body.response_metadata?.next_cursor?.trim() ?? "";
+    if (!cursor) return refs;
+  }
+  throw new Error(
+    "Slack delivery reconciliation exceeded the bounded thread scan",
+  );
+}
+
+async function slackApiJson(
   url: string,
   init: RequestInit,
   timeoutMs = SLACK_API_TIMEOUT_MS,
-): Promise<Response> {
+): Promise<Record<string, any>> {
   const controller =
     typeof AbortController !== "undefined" ? new AbortController() : undefined;
+  const externalSignal = init.signal;
+  const abortFromExternal = () => controller?.abort(externalSignal?.reason);
+  if (controller && externalSignal) {
+    if (externalSignal.aborted) abortFromExternal();
+    else {
+      externalSignal.addEventListener("abort", abortFromExternal, {
+        once: true,
+      });
+    }
+  }
   const timer = controller
     ? setTimeout(() => controller.abort(), timeoutMs)
     : undefined;
   try {
-    return await fetch(url, {
+    const response = await fetch(url, {
       ...init,
       signal: controller?.signal ?? init.signal,
     });
+    return (await response.json()) as Record<string, any>;
   } finally {
     if (timer) clearTimeout(timer);
+    externalSignal?.removeEventListener("abort", abortFromExternal);
   }
 }
 
@@ -1165,12 +1345,11 @@ async function resolveSlackThreadPermalink(
     const url = new URL("https://slack.com/api/chat.getPermalink");
     url.searchParams.set("channel", channelId);
     url.searchParams.set("message_ts", threadTs);
-    const res = await slackApiFetch(
+    const data = (await slackApiJson(
       url.toString(),
       { headers: { Authorization: `Bearer ${token}` } },
       SLACK_PERMALINK_TIMEOUT_MS,
-    );
-    const data = (await res.json()) as {
+    )) as {
       ok?: boolean;
       permalink?: unknown;
     };
@@ -1260,12 +1439,11 @@ async function slackJson(
     for (const [key, value] of Object.entries(params)) {
       url.searchParams.set(key, value);
     }
-    const response = await slackApiFetch(
+    const body = await slackApiJson(
       url.toString(),
       { headers: { Authorization: `Bearer ${token}` } },
       timeoutMs,
     );
-    const body = (await response.json()) as Record<string, any>;
     return body.ok ? body : null;
   } catch {
     return null;
@@ -1618,16 +1796,17 @@ async function postSlackJson(
   token: string,
   method: string,
   body: Record<string, unknown>,
+  signal?: AbortSignal,
 ): Promise<Record<string, any>> {
-  const response = await slackApiFetch(`https://slack.com/api/${method}`, {
+  const data = await slackApiJson(`https://slack.com/api/${method}`, {
     method: "POST",
     headers: {
       Authorization: `Bearer ${token}`,
       "Content-Type": "application/json",
     },
     body: JSON.stringify(body),
+    signal,
   });
-  const data = (await response.json()) as Record<string, any>;
   if (!data.ok) throw new Error(data.error || `${method} failed`);
   return data;
 }
@@ -1775,6 +1954,7 @@ function createSlackRunProgress(
 
   return {
     ref: { kind: "slack-stream", streamTs },
+    responseTargetRef: streamTs,
     async onEvent(event) {
       if (!cancelControl) {
         const context = getIntegrationRequestContext();
@@ -1933,7 +2113,7 @@ function createSlackRunProgress(
         });
       }
     },
-    async complete(message) {
+    async complete(message, opts) {
       if (pendingTimer) clearTimeout(pendingTimer);
       const finalChunks = [...tasks.entries()].map(([id, task]) => ({
         type: "task_update",
@@ -1968,25 +2148,35 @@ function createSlackRunProgress(
             },
           ]
         : [];
-      await postSlackJson(token, "chat.stopStream", {
-        channel,
-        ts: streamTs,
-        markdown_text: message.text || "Done.",
-        ...(finalChunks.length ? { chunks: finalChunks } : {}),
-        ...(messageBlocks.length || controlBlocks.length
-          ? { blocks: [...messageBlocks, ...controlBlocks].slice(0, 50) }
-          : {}),
-      });
+      await postSlackJson(
+        token,
+        "chat.stopStream",
+        {
+          channel,
+          ts: streamTs,
+          markdown_text: message.text || "Done.",
+          ...(finalChunks.length ? { chunks: finalChunks } : {}),
+          ...(messageBlocks.length || controlBlocks.length
+            ? { blocks: [...messageBlocks, ...controlBlocks].slice(0, 50) }
+            : {}),
+        },
+        opts?.signal,
+      );
       setSlackAssistantStatus(token, channel, threadTs, "");
       return { status: "delivered", messageRefs: [streamTs] };
     },
-    async fail(message) {
+    async fail(message, opts) {
       if (pendingTimer) clearTimeout(pendingTimer);
-      await postSlackJson(token, "chat.stopStream", {
-        channel,
-        ts: streamTs,
-        markdown_text: message.slice(0, SLACK_MAX_LENGTH),
-      }).catch(() => {});
+      await postSlackJson(
+        token,
+        "chat.stopStream",
+        {
+          channel,
+          ts: streamTs,
+          markdown_text: message.slice(0, SLACK_MAX_LENGTH),
+        },
+        opts?.signal,
+      ).catch(() => {});
       setSlackAssistantStatus(token, channel, threadTs, "");
     },
   };

package/src/integrations/integration-campaign-recovery.ts

@@ -1,3 +1,4 @@
+import { getA2AContinuationTaskOutcome } from "./a2a-continuations-store.js";
 import {
   getIntegrationCampaign,
   failDisabledIntegrationCampaignTask,
@@ -72,7 +73,9 @@ export async function recoverDueIntegrationCampaigns(options: {
             : undefined,
         },
       );
-      const confirmedReceipt = hasConfirmedDeliveryReceipt(task.payload);
+      const confirmedReceipt =
+        hasConfirmedDeliveryReceipt(task.payload) ||
+        (await getA2AContinuationTaskOutcome(task.id)) === "terminal-delivered";
       if (!durableDispatchEnabled && !confirmedReceipt) {
         await failDisabledIntegrationCampaignTask(task.id);
         const nextTask = await getNextPendingTaskForThread(

package/src/integrations/plugin.ts

@@ -36,12 +36,11 @@ import { runWithRequestContext } from "../server/request-context.js";
 import {
   processA2AContinuationById,
   processDueA2AContinuations,
+  reconcileTerminalA2AParentIfDisabled,
+  recoverA2AContinuationAfterProcessorFailure,
   recoverDueA2AContinuations,
 } from "./a2a-continuation-processor.js";
-import {
-  failA2AContinuation,
-  hasActiveA2AContinuationsForIntegrationTask,
-} from "./a2a-continuations-store.js";
+import { getA2AContinuationTaskOutcome } from "./a2a-continuations-store.js";
 import { mergeIntegrationAdapters } from "./adapter-overrides.js";
 import { discordAdapter } from "./adapters/discord.js";
 import { emailAdapter } from "./adapters/email.js";
@@ -1908,13 +1907,18 @@ export function createIntegrationsPlugin(
               ? { channelId: task.dispatchScope }
               : undefined,
           });
+        const a2aOutcome = campaignContinuation
+          ? await getA2AContinuationTaskOutcome(task.id)
+          : null;
         const confirmedDeliveryReceipt =
           taskPayload.kind === "response-delivery" &&
           taskPayload.deliveryReceipt?.status === "delivered";
+        const confirmedDeliveryProof =
+          confirmedDeliveryReceipt || a2aOutcome === "terminal-delivered";
         if (
           campaignContinuation &&
           !durableCampaignEnabled &&
-          !confirmedDeliveryReceipt
+          !confirmedDeliveryProof
         ) {
           await failDisabledIntegrationCampaignTask(task.id);
           const nextTask = await getNextPendingTaskForThread(
@@ -1995,6 +1999,17 @@ export function createIntegrationsPlugin(
                 return;
               }
               if (taskPayload.kind === "response-delivery") {
+                if (
+                  campaignContinuation &&
+                  taskPayload.awaitingA2ACompletion &&
+                  a2aOutcome === "terminal-delivered"
+                ) {
+                  const completed =
+                    await completeIntegrationCampaignTaskAfterA2A(task.id);
+                  return completed
+                    ? ("completed" as const)
+                    : ("campaign-active" as const);
+                }
                 let receipt: void | PlatformDeliveryReceipt =
                   taskPayload.deliveryReceipt;
                 let deliveryLease:
@@ -2114,15 +2129,15 @@ export function createIntegrationsPlugin(
                     if (!waiting) return "campaign-active" as const;
                   }
                   if (
-                    !(await hasActiveA2AContinuationsForIntegrationTask(
-                      task.id,
-                    ))
+                    a2aOutcome === "terminal-without-delivery" &&
+                    (await reconcileTerminalA2AParentIfDisabled(task.id))
                   ) {
-                    const completed =
-                      await completeIntegrationCampaignTaskAfterA2A(task.id);
-                    return completed
-                      ? ("completed" as const)
-                      : ("campaign-active" as const);
+                    return "campaign-failed" as const;
+                  }
+                  if (a2aOutcome !== "active") {
+                    console.warn(
+                      `[integrations] Waiting campaign ${task.id} has A2A outcome ${a2aOutcome} without terminal delivery proof`,
+                    );
                   }
                   return "campaign-active" as const;
                 }
@@ -2387,15 +2402,18 @@ export function createIntegrationsPlugin(
             adapters: adapterMap,
           });
         } catch (err: any) {
-          // Mark the continuation failed so it isn't left dangling, and surface
-          // a 500 to the caller instead of leaking an unhandled rejection.
-          await failA2AContinuation(
-            continuationId,
-            err?.message?.slice(0, 500) || "continuation processing failed",
-          ).catch(() => {});
+          const reason =
+            err?.message?.slice(0, 500) || "continuation processing failed";
+          await recoverA2AContinuationAfterProcessorFailure(continuationId, {
+            adapters: adapterMap,
+            reason,
+          }).catch(() => {
+            console.error(
+              `[integrations] A2A continuation ${continuationId} recovery scheduling failed`,
+            );
+          });
           console.error(
-            "[integrations] process-a2a-continuation failure:",
-            err,
+            `[integrations] process-a2a-continuation failure for ${continuationId}; durable recovery requested`,
           );
           setResponseStatus(event, 500);
           return { error: "Failed to process A2A continuation" };