@agent-native/core 0.122.2 → 0.122.4

package/dist/integrations/plugin.js

@@ -13,8 +13,8 @@ import { getH3App, markDefaultPluginProvided, } from "../server/framework-reques
 import { decodeOAuthState, encodeOAuthState, oauthCallbackResponse, oauthErrorPage, resolveOAuthRedirectUri, } from "../server/google-oauth.js";
 import { readBody } from "../server/h3-helpers.js";
 import { runWithRequestContext } from "../server/request-context.js";
-import { processA2AContinuationById, processDueA2AContinuations, } from "./a2a-continuation-processor.js";
-import { failA2AContinuation } from "./a2a-continuations-store.js";
+import { processA2AContinuationById, processDueA2AContinuations, recoverDueA2AContinuations, } from "./a2a-continuation-processor.js";
+import { failA2AContinuation, hasActiveA2AContinuationsForIntegrationTask, } from "./a2a-continuations-store.js";
 import { mergeIntegrationAdapters } from "./adapter-overrides.js";
 import { discordAdapter } from "./adapters/discord.js";
 import { emailAdapter } from "./adapters/email.js";
@@ -30,11 +30,13 @@ import { claimIntegrationControl } from "./controls-store.js";
 import { startGoogleDocsPoller, handlePushNotification, } from "./google-docs-poller.js";
 import { IntegrationIdentityDeclinedError, resolveDefaultIntegrationExecutionContext, } from "./identity.js";
 import { disconnectIntegrationInstallation, listIntegrationInstallations, resolveIntegrationTokenBundle, updateIntegrationInstallation, upsertIntegrationInstallation, } from "./installations-store.js";
-import { dispatchPendingIntegrationTask, INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT, integrationDispatchScopeValue, isIntegrationDurableDispatchConfigured, } from "./integration-durable-dispatch.js";
+import { recoverDueIntegrationCampaigns } from "./integration-campaign-recovery.js";
+import { claimIntegrationCampaignDeliveryForTask, completeIntegrationCampaignTaskAfterA2A, completeIntegrationCampaignTask, failIntegrationCampaignTaskDeliveryContainment, failDisabledIntegrationCampaignTask, failIntegrationCampaign, refreshIntegrationCampaignTaskA2AReceiptRetry, terminalizeIntegrationCampaignForTask, transitionIntegrationCampaignTaskToA2AReceiptRetry, transitionIntegrationCampaignTaskToDeliveryRetry, waitForA2AIntegrationCampaign, } from "./integration-campaigns-store.js";
+import { dispatchPendingIntegrationTask, INTEGRATION_CAMPAIGN_PROCESSOR_FIELD, INTEGRATION_RETRY_SWEEP_TOKEN_SUBJECT, integrationDispatchScopeValue, isIntegrationDurableDispatchConfigured, isIntegrationDurableDispatchEnabledForTask, } from "./integration-durable-dispatch.js";
 import { forgetIntegrationMemory, integrationMemoryActions, listIntegrationMemory, rememberForIntegrationScope, } from "./integration-memory.js";
 import { extractBearerToken, verifyInternalToken } from "./internal-token.js";
 import { retryStuckPendingTasks, startPendingTasksRetryJob, } from "./pending-tasks-retry-job.js";
-import { claimPendingTask, failTaskDeliveryTransition, getNextPendingTaskForThread, insertPendingTask, isDuplicateEventError, MAX_PENDING_TASK_ATTEMPTS, markTaskCompleted, markTaskDeliveryRetryable, markTaskFailed, markTaskRetryable, stageTaskDeliveryPayload, } from "./pending-tasks-store.js";
+import { claimPendingTask, getNextPendingTaskForThread, getPendingTask, insertPendingTask, isDuplicateEventError, MAX_PENDING_TASK_ATTEMPTS, markTaskCompleted, markTaskDeliveryRetryable, markTaskFailed, markTaskRetryable, stageTaskDeliveryPayload, } from "./pending-tasks-store.js";
 import { claimNextComputerCommand, claimNextRemoteCommand, enqueueComputerCommand, enqueueRemoteCommand as enqueueRemoteCommandRow, isRemoteCommandKind, listRemoteCommandsForOwner, updateRemoteCommandResult, } from "./remote-commands-store.js";
 import { authenticateRemoteDeviceToken, createRemoteDevice, getRemoteComputerCapabilities, getRemoteDeviceForOwner, listRemoteDevicesForOwner, revokeRemoteDeviceForOwner, toPublicRemoteDevice, unregisterRemoteDevice, updateRemoteDeviceDetails, } from "./remote-devices-store.js";
 import { startRemotePushDeliveryJob } from "./remote-push-delivery-job.js";
@@ -48,6 +50,110 @@ import { getTaskQueueStats } from "./task-queue-stats.js";
 import { listIntegrationUsageBudgets, saveIntegrationUsageBudget, } from "./usage-budget-store.js";
 import { handleWebhook, processIntegrationTask, recordIntegrationResponseDelivery, } from "./webhook-handler.js";
 let a2aContinuationRetryInterval = null;
+const INTEGRATION_DELIVERY_LEASE_MS = 2 * 60_000;
+async function checkpointIntegrationDeliveryRetry(task, payload, errorMessage, event, campaignLease) {
+    let terminalStatus;
+    let confirmedReceipt = false;
+    let awaitingA2ACompletion = false;
+    try {
+        const parsed = JSON.parse(payload);
+        terminalStatus = parsed.campaignTerminalStatus;
+        confirmedReceipt = parsed.deliveryReceipt?.status === "delivered";
+        awaitingA2ACompletion = parsed.awaitingA2ACompletion === true;
+    }
+    catch { }
+    if (awaitingA2ACompletion) {
+        if (campaignLease && campaignLease.campaignStatus !== "waiting-a2a") {
+            throw new Error("A2A receipt retry lease has the wrong custody mode");
+        }
+        const transitioned = campaignLease
+            ? await transitionIntegrationCampaignTaskToA2AReceiptRetry(task.id, {
+                payload,
+                errorMessage,
+                campaignId: campaignLease.campaignId,
+                runId: campaignLease.runId,
+                leaseToken: campaignLease.leaseToken,
+                nextRunAt: Date.now() + 15_000,
+            })
+            : await refreshIntegrationCampaignTaskA2AReceiptRetry(task.id, {
+                payload,
+                errorMessage,
+            });
+        if (!transitioned)
+            return "superseded";
+        await dispatchPendingIntegrationTask({
+            taskId: task.id,
+            task: {
+                platform: task.platform,
+                externalThreadId: task.externalThreadId,
+                platformContext: task.dispatchScope
+                    ? { channelId: task.dispatchScope }
+                    : undefined,
+            },
+            event,
+            baseUrl: getBaseUrl(event),
+            campaignContinuation: true,
+            allowPortableConfirmedReceiptReconciliation: confirmedReceipt,
+        });
+        return "requeued";
+    }
+    if (terminalStatus && !campaignLease && !confirmedReceipt) {
+        throw new Error("Campaign delivery retry is missing its lease");
+    }
+    if (terminalStatus && campaignLease) {
+        if (terminalStatus !== campaignLease.campaignStatus) {
+            throw new Error("Campaign delivery retry status does not match its lease");
+        }
+        const transitioned = await transitionIntegrationCampaignTaskToDeliveryRetry(task.id, {
+            payload,
+            errorMessage,
+            campaignStatus: campaignLease.campaignStatus,
+            campaignId: campaignLease.campaignId,
+            runId: campaignLease.runId,
+            leaseToken: campaignLease.leaseToken,
+        });
+        if (!transitioned) {
+            return "superseded";
+        }
+    }
+    else {
+        await markTaskDeliveryRetryable(task.id, payload, errorMessage);
+    }
+    await dispatchPendingIntegrationTask({
+        taskId: task.id,
+        task: {
+            platform: task.platform,
+            externalThreadId: task.externalThreadId,
+            platformContext: task.dispatchScope
+                ? { channelId: task.dispatchScope }
+                : undefined,
+        },
+        event,
+        baseUrl: getBaseUrl(event),
+    });
+    return "requeued";
+}
+async function containFailedDeliveryTransition(task, errorMessage, event) {
+    const contained = await failIntegrationCampaignTaskDeliveryContainment(task.id, errorMessage);
+    if (!contained) {
+        throw new Error("Delivery containment lost pending-task custody");
+    }
+    const nextTask = await getNextPendingTaskForThread(task.platform, task.externalThreadId);
+    if (!nextTask)
+        return;
+    await dispatchPendingIntegrationTask({
+        taskId: nextTask.id,
+        task: {
+            platform: task.platform,
+            externalThreadId: task.externalThreadId,
+            platformContext: nextTask.dispatchScope
+                ? { channelId: nextTask.dispatchScope }
+                : undefined,
+        },
+        event,
+        baseUrl: getBaseUrl(event),
+    });
+}
 function startA2AContinuationRetryJob(adapters) {
     if (a2aContinuationRetryInterval)
         return;
@@ -1262,12 +1368,38 @@ export function createIntegrationsPlugin(options) {
             if (!isIntegrationDurableDispatchConfigured()) {
                 return { ok: true, disabled: true };
             }
-            const result = await retryStuckPendingTasks({
-                webhookBaseUrl: getBaseUrl(event),
-                limit: 20,
-                durableOnly: true,
-            });
-            return { ok: true, ...result };
+            const webhookBaseUrl = getBaseUrl(event);
+            const [pendingTasks, campaigns, a2aContinuations] = await Promise.all([
+                retryStuckPendingTasks({
+                    webhookBaseUrl,
+                    limit: 20,
+                    durableOnly: true,
+                }).catch((error) => {
+                    console.error("[integrations] Pending-task recovery failed:", error);
+                    return { error: "pending-task-recovery-failed" };
+                }),
+                recoverDueIntegrationCampaigns({
+                    event,
+                    webhookBaseUrl,
+                    limit: 20,
+                }).catch((error) => {
+                    console.error("[integrations] Campaign recovery failed:", error);
+                    return { error: "campaign-recovery-failed" };
+                }),
+                recoverDueA2AContinuations({
+                    webhookBaseUrl,
+                    limit: 10,
+                }).catch((error) => {
+                    console.error("[integrations] A2A recovery failed:", error);
+                    return { error: "a2a-recovery-failed" };
+                }),
+            ]);
+            return {
+                ok: true,
+                pendingTasks,
+                campaigns,
+                a2aContinuations,
+            };
         }));
         // ─── Process pending task (cross-platform task queue) ────────
         // POST /_agent-native/integrations/process-task
@@ -1317,14 +1449,62 @@ export function createIntegrationsPlugin(options) {
                 AGENT_BACKGROUND_PROCESSOR_INTEGRATION
                 ? "background-acknowledged"
                 : "portable-unconfirmed";
-            const task = await claimPendingTask(taskId, { dispatchOutcome });
+            const campaignContinuation = body[INTEGRATION_CAMPAIGN_PROCESSOR_FIELD] === true;
+            const task = campaignContinuation
+                ? await getPendingTask(taskId)
+                : await claimPendingTask(taskId, { dispatchOutcome });
             if (!task) {
                 setResponseStatus(event, 200);
                 return { ok: true, skipped: "already-claimed-or-missing" };
             }
+            if (campaignContinuation && task.status !== "processing") {
+                setResponseStatus(event, 200);
+                return { ok: true, skipped: "campaign-task-not-processing" };
+            }
+            let taskPayload;
+            try {
+                taskPayload = JSON.parse(task.payload);
+            }
+            catch {
+                await markTaskFailed(task.id, "Invalid integration task payload");
+                setResponseStatus(event, 400);
+                return { error: "Invalid integration task payload" };
+            }
+            const durableCampaignEnabled = isIntegrationDurableDispatchEnabledForTask({
+                platform: task.platform,
+                externalThreadId: task.externalThreadId,
+                platformContext: task.dispatchScope
+                    ? { channelId: task.dispatchScope }
+                    : undefined,
+            });
+            const confirmedDeliveryReceipt = taskPayload.kind === "response-delivery" &&
+                taskPayload.deliveryReceipt?.status === "delivered";
+            if (campaignContinuation &&
+                !durableCampaignEnabled &&
+                !confirmedDeliveryReceipt) {
+                await failDisabledIntegrationCampaignTask(task.id);
+                const nextTask = await getNextPendingTaskForThread(task.platform, task.externalThreadId);
+                if (nextTask) {
+                    await dispatchPendingIntegrationTask({
+                        taskId: nextTask.id,
+                        task: {
+                            platform: task.platform,
+                            externalThreadId: task.externalThreadId,
+                            platformContext: nextTask.dispatchScope
+                                ? { channelId: nextTask.dispatchScope }
+                                : undefined,
+                        },
+                        event,
+                        baseUrl: getBaseUrl(event),
+                    });
+                }
+                setResponseStatus(event, 200);
+                return { ok: true, failed: "campaign-disabled" };
+            }
             let deliveryRetryTransitionStarted = false;
             let deliveryRetryRecovery;
             let confirmedDeliveryRetryPayload;
+            let campaignDeliveryLease;
             try {
                 const adapter = adapterMap.get(task.platform);
                 if (!adapter) {
@@ -1337,8 +1517,8 @@ export function createIntegrationsPlugin(options) {
                     ...(task.orgId ? { orgId: task.orgId } : {}),
                     isIntegrationCaller: true,
                 }, async () => {
-                    const taskPayload = JSON.parse(task.payload);
-                    if (taskPayload.kind === "system-notice") {
+                    if (!campaignContinuation &&
+                        taskPayload.kind === "system-notice") {
                         if (!adapter.sendSystemNotice) {
                             throw new Error(`Platform ${task.platform} cannot deliver system notices`);
                         }
@@ -1356,6 +1536,30 @@ export function createIntegrationsPlugin(options) {
                     }
                     if (taskPayload.kind === "response-delivery") {
                         let receipt = taskPayload.deliveryReceipt;
+                        let deliveryLease;
+                        if (campaignContinuation && !receipt) {
+                            const runId = `integration-delivery-${crypto.randomUUID()}`;
+                            const leaseToken = crypto.randomUUID();
+                            const deliveryClaim = await claimIntegrationCampaignDeliveryForTask(task.id, {
+                                runId,
+                                leaseToken,
+                                leaseDurationMs: INTEGRATION_DELIVERY_LEASE_MS,
+                            });
+                            if (!deliveryClaim)
+                                return "campaign-active";
+                            deliveryLease = {
+                                campaignId: deliveryClaim.id,
+                                runId,
+                                leaseToken,
+                            };
+                            campaignDeliveryLease = {
+                                ...deliveryLease,
+                                campaignStatus: taskPayload.campaignTerminalStatus ??
+                                    (taskPayload.awaitingA2ACompletion
+                                        ? "waiting-a2a"
+                                        : "completed"),
+                            };
+                        }
                         if (!receipt) {
                             receipt = await adapter.sendResponse(taskPayload.message, taskPayload.incoming, {
                                 ...(taskPayload.placeholderRef
@@ -1384,6 +1588,55 @@ export function createIntegrationsPlugin(options) {
                             deliveryRetryRecovery = undefined;
                         }
                         await recordIntegrationResponseDelivery(deliveredPayload, receipt);
+                        const campaignTerminalStatus = taskPayload.campaignTerminalStatus;
+                        if (campaignTerminalStatus) {
+                            const errorMessage = "Integration campaign exhausted its continuation limit";
+                            const terminalized = deliveryLease
+                                ? campaignTerminalStatus === "failed"
+                                    ? await failIntegrationCampaign(deliveryLease.campaignId, {
+                                        runId: deliveryLease.runId,
+                                        leaseToken: deliveryLease.leaseToken,
+                                        errorMessage,
+                                    })
+                                    : await completeIntegrationCampaignTask(deliveryLease.campaignId, {
+                                        integrationTaskId: task.id,
+                                        runId: deliveryLease.runId,
+                                        leaseToken: deliveryLease.leaseToken,
+                                    })
+                                : await terminalizeIntegrationCampaignForTask(task.id, {
+                                    status: campaignTerminalStatus,
+                                    ...(campaignTerminalStatus === "failed"
+                                        ? { errorMessage }
+                                        : {}),
+                                });
+                            if (deliveryLease && !terminalized) {
+                                return "campaign-active";
+                            }
+                            return campaignTerminalStatus === "failed"
+                                ? "campaign-failed"
+                                : "completed";
+                        }
+                        if (taskPayload.awaitingA2ACompletion) {
+                            if (deliveryLease) {
+                                const waiting = await waitForA2AIntegrationCampaign(deliveryLease.campaignId, {
+                                    runId: deliveryLease.runId,
+                                    leaseToken: deliveryLease.leaseToken,
+                                    nextRunAt: Date.now() + 15_000,
+                                });
+                                if (!waiting)
+                                    return "campaign-active";
+                            }
+                            if (!(await hasActiveA2AContinuationsForIntegrationTask(task.id))) {
+                                const completed = await completeIntegrationCampaignTaskAfterA2A(task.id);
+                                return completed
+                                    ? "completed"
+                                    : "campaign-active";
+                            }
+                            return "campaign-active";
+                        }
+                        if (campaignContinuation) {
+                            return "campaign-active";
+                        }
                         return;
                     }
                     const resources = await loadResourcesForPrompt(task.ownerEmail, true, options?.appId, task.orgId);
@@ -1397,18 +1650,55 @@ export function createIntegrationsPlugin(options) {
                         engine: options?.engine,
                         ownerEmail: task.ownerEmail,
                         appId: options?.appId,
+                    }, {
+                        enabled: durableCampaignEnabled,
+                        continuationInvocation: campaignContinuation,
                     });
                     if (result?.status === "delivery-pending") {
                         deliveryRetryTransitionStarted = true;
-                        await markTaskDeliveryRetryable(task.id, JSON.stringify(result.payload), result.errorMessage);
+                        const checkpoint = await checkpointIntegrationDeliveryRetry(task, JSON.stringify(result.payload), result.errorMessage, event, result.campaignLease);
+                        if (checkpoint === "superseded") {
+                            return "campaign-active";
+                        }
                         return "delivery-retry";
                     }
+                    if (result?.status === "campaign-pending" ||
+                        result?.status === "campaign-active") {
+                        return "campaign-active";
+                    }
+                    if (result?.status === "campaign-failed") {
+                        return "campaign-failed";
+                    }
                     return "completed";
                 });
                 if (processingResult === "delivery-retry") {
                     setResponseStatus(event, 202);
                     return { ok: true, taskId, retrying: "response-delivery" };
                 }
+                if (processingResult === "campaign-active") {
+                    setResponseStatus(event, 202);
+                    return { ok: true, taskId, continuing: true };
+                }
+                if (processingResult === "campaign-failed") {
+                    await markTaskFailed(taskId, "Integration campaign exhausted its continuation limit");
+                    const nextTask = await getNextPendingTaskForThread(task.platform, task.externalThreadId);
+                    if (nextTask) {
+                        await dispatchPendingIntegrationTask({
+                            taskId: nextTask.id,
+                            task: {
+                                platform: task.platform,
+                                externalThreadId: task.externalThreadId,
+                                platformContext: nextTask.dispatchScope
+                                    ? { channelId: nextTask.dispatchScope }
+                                    : undefined,
+                            },
+                            event,
+                            baseUrl: getBaseUrl(event),
+                        });
+                    }
+                    setResponseStatus(event, 200);
+                    return { ok: true, taskId, failed: "campaign-exhausted" };
+                }
                 await markTaskCompleted(taskId);
                 const nextTask = await getNextPendingTaskForThread(task.platform, task.externalThreadId);
                 if (nextTask) {
@@ -1439,7 +1729,11 @@ export function createIntegrationsPlugin(options) {
                     : "processor failed";
                 if (deliveryRetryRecovery) {
                     try {
-                        await markTaskDeliveryRetryable(taskId, deliveryRetryRecovery.payload, `${deliveryRetryRecovery.errorMessage}: ${errorMessage}`);
+                        const checkpoint = await checkpointIntegrationDeliveryRetry(task, deliveryRetryRecovery.payload, `${deliveryRetryRecovery.errorMessage}: ${errorMessage}`, event, campaignDeliveryLease);
+                        if (checkpoint === "superseded") {
+                            setResponseStatus(event, 202);
+                            return { ok: true, taskId, continuing: true };
+                        }
                         setResponseStatus(event, 202);
                         return { ok: true, taskId, retrying: "response-delivery" };
                     }
@@ -1447,24 +1741,37 @@ export function createIntegrationsPlugin(options) {
                         const transitionMessage = transitionError instanceof Error
                             ? transitionError.message
                             : String(transitionError);
-                        await failTaskDeliveryTransition(taskId, `Could not safely checkpoint the delivery receipt: ${transitionMessage}`).catch((failureTransitionError) => {
+                        await containFailedDeliveryTransition(task, `Could not safely checkpoint the delivery receipt: ${transitionMessage}`, event).catch((failureTransitionError) => {
                             console.error("[integrations] Failed to contain delivery receipt transition failure:", failureTransitionError);
                         });
                     }
                 }
                 else if (confirmedDeliveryRetryPayload) {
                     try {
-                        await markTaskDeliveryRetryable(taskId, confirmedDeliveryRetryPayload, `Provider delivery was confirmed but history persistence failed: ${errorMessage}`);
+                        const checkpoint = await checkpointIntegrationDeliveryRetry(task, confirmedDeliveryRetryPayload, `Provider delivery was confirmed but history persistence failed: ${errorMessage}`, event, campaignDeliveryLease);
+                        if (checkpoint === "superseded") {
+                            setResponseStatus(event, 202);
+                            return { ok: true, taskId, continuing: true };
+                        }
                         console.error("[integrations] process-task failure:", err);
                         setResponseStatus(event, 202);
                         return { ok: true, taskId, retrying: "response-delivery" };
                     }
                     catch (transitionError) {
+                        const transitionMessage = transitionError instanceof Error
+                            ? transitionError.message
+                            : String(transitionError);
                         console.error("[integrations] Failed to requeue confirmed delivery history:", transitionError);
+                        await containFailedDeliveryTransition(task, `Could not safely checkpoint confirmed delivery history: ${transitionMessage}`, event).catch((failureTransitionError) => {
+                            console.error("[integrations] Failed to contain confirmed delivery history transition failure:", failureTransitionError);
+                        });
+                        console.error("[integrations] process-task failure:", err);
+                        setResponseStatus(event, 500);
+                        return { error: "Internal task failed" };
                     }
                 }
                 else if (deliveryRetryTransitionStarted) {
-                    await failTaskDeliveryTransition(taskId, `Could not safely checkpoint the delivery retry: ${errorMessage}`).catch((transitionError) => {
+                    await containFailedDeliveryTransition(task, `Could not safely checkpoint the delivery retry: ${errorMessage}`, event).catch((transitionError) => {
                         console.error("[integrations] Failed to contain delivery retry transition failure:", transitionError);
                     });
                 }