@agent-native/core 0.122.2 → 0.122.4

package/corpus/core/src/cli/telemetry.ts

@@ -198,7 +198,7 @@ export function createCliTelemetry(options: CliTelemetryOptions): CliTelemetry {
   const publicKey = resolvePublicKey();
   const disabled = telemetryDisabled() || !publicKey;
   const endpoint = resolveEndpoint();
-  const installId = disabled ? "" : resolveInstallId();
+  let installId: string | undefined = disabled ? "" : undefined;
   const runId = crypto.randomUUID();
   const inFlight = new Set<Promise<void>>();
 
@@ -211,18 +211,18 @@ export function createCliTelemetry(options: CliTelemetryOptions): CliTelemetry {
     ci: process.env.CI === "true",
     interactive: options.interactive,
     runId,
-    installId,
   };
 
   function track(event: string, properties?: Record<string, unknown>): void {
     if (disabled) return;
+    installId ??= resolveInstallId();
     const body = JSON.stringify({
       publicKey,
       event,
       anonymousId: installId,
       sessionId: runId,
       timestamp: new Date().toISOString(),
-      properties: { ...base, ...properties },
+      properties: { ...base, installId, ...properties },
     });
     const promise = fetch(endpoint, {
       method: "POST",

package/corpus/core/src/integrations/a2a-continuation-processor.ts

@@ -19,12 +19,30 @@ import {
   claimA2AContinuationDelivery,
   claimDueA2AContinuations,
   completeA2AContinuation,
+  failA2AContinuationsForIntegrationTask,
   failA2AContinuation,
   getA2AContinuation,
+  hasActiveA2AContinuationsForIntegrationTask,
+  listRecoverableA2AIntegrationTasks,
+  recoverDueA2AContinuationIds,
   rescheduleA2AContinuation,
   type A2AContinuation,
+  type RecoverableA2AIntegrationTask,
 } from "./a2a-continuations-store.js";
+import {
+  completeIntegrationCampaignTaskAfterA2A,
+  failDisabledIntegrationCampaignTask,
+  getIntegrationCampaignForTask,
+} from "./integration-campaigns-store.js";
+import {
+  dispatchPendingIntegrationTask,
+  isIntegrationDurableDispatchEnabledForTask,
+} from "./integration-durable-dispatch.js";
 import { signInternalToken } from "./internal-token.js";
+import {
+  getNextPendingTaskForThread,
+  getPendingTask,
+} from "./pending-tasks-store.js";
 import { getThreadMapping } from "./thread-mapping-store.js";
 import type {
   OutgoingMessage,
@@ -150,10 +168,62 @@ export async function processDueA2AContinuations(options: {
   }
 }
 
+/**
+ * Durable scheduler wake-up only: make a bounded set of due/stale rows
+ * eligible, then invoke their normal processors. It never polls remote A2A
+ * tasks or runs a mutation itself, keeping the scheduled route within its
+ * short execution budget. Duplicate wake-ups are safe because each processor
+ * still takes the store's atomic claim before it can progress or deliver.
+ */
+export async function recoverDueA2AContinuations(options?: {
+  limit?: number;
+  webhookBaseUrl?: string;
+}): Promise<{ dispatched: number; failed: number }> {
+  const limit = options?.limit ?? 5;
+  const candidateTasks = await listRecoverableA2AIntegrationTasks(200);
+  const eligibleTaskIds: string[] = [];
+  for (const task of candidateTasks) {
+    const enabled = isIntegrationDurableDispatchEnabledForTask({
+      platform: task.platform,
+      externalThreadId: task.externalThreadId,
+      platformContext: task.dispatchScope
+        ? { channelId: task.dispatchScope }
+        : undefined,
+    });
+    if (enabled) {
+      eligibleTaskIds.push(task.id);
+      if (eligibleTaskIds.length >= limit) break;
+    } else {
+      await failDisabledDurableA2ATask(task);
+    }
+  }
+  const ids = await recoverDueA2AContinuationIds(limit, eligibleTaskIds);
+  let dispatched = 0;
+  let failed = 0;
+
+  await Promise.all(
+    ids.map(async (id) => {
+      try {
+        await dispatchA2AContinuation(id, options?.webhookBaseUrl);
+        dispatched += 1;
+      } catch (err) {
+        failed += 1;
+        console.error(
+          `[integrations] Failed to recover A2A continuation ${id}:`,
+          err,
+        );
+      }
+    }),
+  );
+
+  return { dispatched, failed };
+}
+
 async function processClaimedContinuation(
   continuation: A2AContinuation,
   options: { adapters: Map<string, PlatformAdapter> },
 ): Promise<void> {
+  if (!(await durableContinuationScopeStillEnabled(continuation))) return;
   const adapter = options.adapters.get(continuation.platform);
   if (!adapter) {
     await failA2AContinuation(
@@ -300,6 +370,66 @@ async function processClaimedContinuation(
   );
 }
 
+async function durableContinuationScopeStillEnabled(
+  continuation: A2AContinuation,
+): Promise<boolean> {
+  const campaign = await getIntegrationCampaignForTask(
+    continuation.integrationTaskId,
+  );
+  if (!campaign) return true;
+  if (campaign.status === "completed" || campaign.status === "failed") {
+    await failA2AContinuation(
+      continuation.id,
+      "Owning integration campaign is already terminal",
+    );
+    return false;
+  }
+  const task = await getPendingTask(continuation.integrationTaskId);
+  const enabled =
+    task?.status === "processing" &&
+    isIntegrationDurableDispatchEnabledForTask({
+      platform: task.platform,
+      externalThreadId: task.externalThreadId,
+      platformContext: task.dispatchScope
+        ? { channelId: task.dispatchScope }
+        : undefined,
+    });
+  if (enabled) return true;
+
+  await failDisabledDurableA2ATask({
+    id: continuation.integrationTaskId,
+    platform: task?.platform ?? continuation.platform,
+    externalThreadId: task?.externalThreadId ?? continuation.externalThreadId,
+    dispatchScope: task?.dispatchScope ?? null,
+    status: task?.status ?? "missing",
+  });
+  return false;
+}
+
+async function failDisabledDurableA2ATask(
+  task: RecoverableA2AIntegrationTask,
+): Promise<void> {
+  const message = "Durable integration campaign was disabled for this scope";
+  await failA2AContinuationsForIntegrationTask(task.id, message);
+  await failDisabledIntegrationCampaignTask(task.id, message);
+  const nextTask = await getNextPendingTaskForThread(
+    task.platform,
+    task.externalThreadId,
+  );
+  if (nextTask) {
+    await dispatchPendingIntegrationTask({
+      taskId: nextTask.id,
+      task: {
+        platform: task.platform,
+        externalThreadId: task.externalThreadId,
+        platformContext: nextTask.dispatchScope
+          ? { channelId: nextTask.dispatchScope }
+          : undefined,
+      },
+    });
+  }
+}
+
 async function resumeA2AContinuationProgress(
   continuation: A2AContinuation,
   adapter: PlatformAdapter,
@@ -372,6 +502,7 @@ async function notifyAndFailA2AContinuation(
   reason: string,
   progress: PlatformRunProgress | null = null,
 ): Promise<void> {
+  if (!(await durableContinuationScopeStillEnabled(continuation))) return;
   const deliveryContinuation = await claimA2AContinuationDelivery(
     continuation.id,
   );
@@ -399,9 +530,17 @@ async function notifyAndFailA2AContinuation(
       `[integrations] Failed to notify ${deliveryContinuation.platform} about failed A2A continuation ${deliveryContinuation.id}:`,
       err,
     );
+    if (deliveryContinuation.attempts >= MAX_ATTEMPTS) {
+      await failA2AContinuation(deliveryContinuation.id, reason);
+      await completeParentCampaignAfterTerminalA2A(deliveryContinuation);
+      return;
+    }
+    await rescheduleAndRedispatchA2AContinuation(deliveryContinuation.id);
+    return;
   }
 
   await failA2AContinuation(deliveryContinuation.id, reason);
+  await completeParentCampaignAfterTerminalA2A(deliveryContinuation);
 }
 
 async function deliverAndCompleteA2AContinuation(
@@ -410,6 +549,7 @@ async function deliverAndCompleteA2AContinuation(
   text: string,
   progress: PlatformRunProgress | null = null,
 ): Promise<void> {
+  if (!(await durableContinuationScopeStillEnabled(continuation))) return;
   const deliveryContinuation = await claimA2AContinuationDelivery(
     continuation.id,
   );
@@ -589,6 +729,7 @@ async function completeAfterSuccessfulDelivery(
   for (let attempt = 0; attempt < COMPLETE_AFTER_DELIVERY_ATTEMPTS; attempt++) {
     try {
       await completeA2AContinuation(continuation.id);
+      await completeParentCampaignAfterTerminalA2A(continuation);
       return;
     } catch (err) {
       lastError = err;
@@ -602,6 +743,47 @@ async function completeAfterSuccessfulDelivery(
   );
 }
 
+async function completeParentCampaignAfterTerminalA2A(
+  continuation: A2AContinuation,
+): Promise<void> {
+  const campaign = await getIntegrationCampaignForTask(
+    continuation.integrationTaskId,
+  );
+  if (!campaign) return;
+  if (
+    await hasActiveA2AContinuationsForIntegrationTask(
+      continuation.integrationTaskId,
+    )
+  ) {
+    return;
+  }
+  const completed = await completeIntegrationCampaignTaskAfterA2A(
+    continuation.integrationTaskId,
+  );
+  if (!completed) return;
+
+  const nextTask = await getNextPendingTaskForThread(
+    continuation.platform,
+    continuation.externalThreadId,
+  );
+  if (!nextTask) return;
+  await dispatchPendingIntegrationTask({
+    taskId: nextTask.id,
+    task: {
+      platform: continuation.platform,
+      externalThreadId: continuation.externalThreadId,
+      platformContext: nextTask.dispatchScope
+        ? { channelId: nextTask.dispatchScope }
+        : undefined,
+    },
+  }).catch((err) => {
+    console.error(
+      `[integrations] Failed to wake successor ${nextTask.id} after A2A parent completion:`,
+      err,
+    );
+  });
+}
+
 function formatContinuationFailureMessage(
   continuation: A2AContinuation,
   reason: string,

package/corpus/core/src/integrations/a2a-continuations-store.ts

@@ -418,6 +418,35 @@ export async function getA2AContinuationForIntegrationTask(
   return rows[0] ? rowToContinuation(rows[0] as Record<string, unknown>) : null;
 }
 
+export async function hasActiveA2AContinuationsForIntegrationTask(
+  integrationTaskId: string,
+): Promise<boolean> {
+  await ensureTable();
+  const { rows } = await getDbExec().execute({
+    sql: `SELECT 1 AS active FROM integration_a2a_continuations
+          WHERE integration_task_id = ?
+            AND status IN ('pending', 'processing', 'delivering')
+          LIMIT 1`,
+    args: [integrationTaskId],
+  });
+  return rows.length > 0;
+}
+
+export async function failA2AContinuationsForIntegrationTask(
+  integrationTaskId: string,
+  errorMessage: string,
+): Promise<void> {
+  await ensureTable();
+  const now = Date.now();
+  await getDbExec().execute({
+    sql: `UPDATE integration_a2a_continuations
+          SET status = 'failed', error_message = ?, updated_at = ?, completed_at = ?
+          WHERE integration_task_id = ?
+            AND status IN ('pending', 'processing', 'delivering')`,
+    args: [errorMessage.slice(0, 2000), now, now, integrationTaskId],
+  });
+}
+
 export async function getA2AContinuationsForIntegrationTaskAgent(
   integrationTaskId: string,
   agentUrl: string,
@@ -532,40 +561,121 @@ export async function claimA2AContinuation(
 export async function claimDueA2AContinuations(
   limit = 5,
 ): Promise<A2AContinuation[]> {
+  const ids = await recoverDueA2AContinuationIds(limit);
+  const claimed: A2AContinuation[] = [];
+  for (const id of ids) {
+    const continuation = await claimA2AContinuation(id);
+    if (continuation) claimed.push(continuation);
+  }
+  return claimed;
+}
+
+/**
+ * Makes stale leases eligible again and returns a bounded set of due ids.
+ *
+ * This intentionally does not claim anything. Durable schedulers use it only
+ * to wake the normal processor, whose atomic claim remains the sole progress
+ * and delivery owner under overlapping scheduler/self-dispatch executions.
+ */
+export async function recoverDueA2AContinuationIds(
+  limit = 5,
+  integrationTaskIds?: string[],
+): Promise<string[]> {
   await ensureTable();
   const client = getDbExec();
   const now = Date.now();
   const processingCutoff = now - PROCESSING_STUCK_AFTER_MS;
   const staleNextCheckCutoff = now - PROCESSING_NEXT_CHECK_STALE_AFTER_MS;
+  if (integrationTaskIds && integrationTaskIds.length === 0) return [];
+  const taskFilter = integrationTaskIds?.length
+    ? ` AND integration_task_id IN (${integrationTaskIds.map(() => "?").join(", ")})`
+    : "";
+  const taskArgs = integrationTaskIds ?? [];
   // If a processor dies while holding a delivery claim, retry the final send.
   // The stale cutoff preserves the in-flight delivery guard while keeping
   // final integration replies at-least-once.
   await client.execute({
     sql: `UPDATE integration_a2a_continuations
           SET status = ?, next_check_at = ?, updated_at = ?
-          WHERE status = 'delivering' AND updated_at <= ?`,
-    args: ["pending", now, now, now - 5 * 60 * 1000],
+          WHERE status = 'delivering' AND updated_at <= ?${taskFilter}`,
+    args: ["pending", now, now, now - 5 * 60 * 1000, ...taskArgs],
   });
   await client.execute({
     sql: `UPDATE integration_a2a_continuations
           SET status = ?, next_check_at = ?, updated_at = ?
           WHERE status = 'processing'
-            AND (updated_at <= ? OR next_check_at <= ?)`,
-    args: ["pending", now, now, processingCutoff, staleNextCheckCutoff],
+            AND (updated_at <= ? OR next_check_at <= ?)${taskFilter}`,
+    args: [
+      "pending",
+      now,
+      now,
+      processingCutoff,
+      staleNextCheckCutoff,
+      ...taskArgs,
+    ],
   });
   const { rows } = await client.execute({
     sql: `SELECT id FROM integration_a2a_continuations
-          WHERE status = 'pending' AND next_check_at <= ?
+          WHERE status = 'pending' AND next_check_at <= ?${taskFilter}
           ORDER BY next_check_at ASC
           LIMIT ?`,
-    args: [now, limit],
+    args: [now, ...taskArgs, limit],
   });
-  const claimed: A2AContinuation[] = [];
-  for (const row of rows) {
-    const continuation = await claimA2AContinuation(row.id as string);
-    if (continuation) claimed.push(continuation);
-  }
-  return claimed;
+  return rows.map((row) => row.id as string);
+}
+
+export async function listRecoverableA2AIntegrationTaskIds(
+  limit = 50,
+): Promise<string[]> {
+  const tasks = await listRecoverableA2AIntegrationTasks(limit);
+  return tasks.map((task) => task.id);
+}
+
+export interface RecoverableA2AIntegrationTask {
+  id: string;
+  platform: string;
+  externalThreadId: string;
+  dispatchScope: string | null;
+  status: string;
+}
+
+/**
+ * Read due continuation owners and their rollout scope in one query. Recovery
+ * can filter the canary in memory without an N+1 pending-task lookup loop.
+ */
+export async function listRecoverableA2AIntegrationTasks(
+  limit = 50,
+): Promise<RecoverableA2AIntegrationTask[]> {
+  await ensureTable();
+  const now = Date.now();
+  const { rows } = await getDbExec().execute({
+    sql: `SELECT DISTINCT c.integration_task_id, t.platform,
+                 t.external_thread_id, t.dispatch_scope, t.status
+          FROM integration_a2a_continuations c
+          INNER JOIN integration_pending_tasks t
+             ON t.id = c.integration_task_id
+          WHERE t.status = 'processing'
+            AND ((c.status = 'pending' AND c.next_check_at <= ?)
+             OR (c.status = 'processing' AND
+                 (c.updated_at <= ? OR c.next_check_at <= ?))
+             OR (c.status = 'delivering' AND c.updated_at <= ?))
+          ORDER BY c.integration_task_id ASC
+          LIMIT ?`,
+    args: [
+      now,
+      now - PROCESSING_STUCK_AFTER_MS,
+      now - PROCESSING_NEXT_CHECK_STALE_AFTER_MS,
+      now - 5 * 60 * 1000,
+      Math.max(1, Math.min(Math.floor(limit), 200)),
+    ],
+  });
+  return rows.map((row) => ({
+    id: String(row.integration_task_id),
+    platform: String(row.platform),
+    externalThreadId: String(row.external_thread_id),
+    dispatchScope: (row.dispatch_scope as string | null) ?? null,
+    status: String(row.status),
+  }));
 }
 
 export async function claimA2AContinuationDelivery(

package/corpus/core/src/integrations/integration-campaign-recovery.ts

@@ -0,0 +1,127 @@
+import {
+  getIntegrationCampaign,
+  failDisabledIntegrationCampaignTask,
+  listDueIntegrationCampaignIds,
+} from "./integration-campaigns-store.js";
+import {
+  dispatchPendingIntegrationTask,
+  isIntegrationDurableDispatchEnabledForTask,
+} from "./integration-durable-dispatch.js";
+import {
+  getNextPendingTaskForThread,
+  getPendingTask,
+} from "./pending-tasks-store.js";
+
+export interface IntegrationCampaignRecoveryResult {
+  selected: number;
+  dispatched: number;
+  skipped: number;
+  failed: number;
+}
+
+function hasConfirmedDeliveryReceipt(payload: string): boolean {
+  try {
+    const parsed = JSON.parse(payload) as {
+      kind?: unknown;
+      deliveryReceipt?: { status?: unknown };
+    };
+    return (
+      parsed.kind === "response-delivery" &&
+      parsed.deliveryReceipt?.status === "delivered"
+    );
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Wake due campaign chunks without claiming or executing them in the sweep.
+ * The signed process-task endpoint owns the lease and all mutations.
+ */
+export async function recoverDueIntegrationCampaigns(options: {
+  limit?: number;
+  event?: unknown;
+  webhookBaseUrl?: string;
+}): Promise<IntegrationCampaignRecoveryResult> {
+  const ids = await listDueIntegrationCampaignIds(options.limit ?? 20);
+  const result: IntegrationCampaignRecoveryResult = {
+    selected: ids.length,
+    dispatched: 0,
+    skipped: 0,
+    failed: 0,
+  };
+
+  for (const id of ids) {
+    try {
+      const campaign = await getIntegrationCampaign(id);
+      if (!campaign) {
+        result.skipped += 1;
+        continue;
+      }
+      const task = await getPendingTask(campaign.integrationTaskId);
+      if (!task || task.status !== "processing") {
+        result.skipped += 1;
+        continue;
+      }
+      const durableDispatchEnabled = isIntegrationDurableDispatchEnabledForTask(
+        {
+          platform: task.platform,
+          externalThreadId: task.externalThreadId,
+          platformContext: task.dispatchScope
+            ? { channelId: task.dispatchScope }
+            : undefined,
+        },
+      );
+      const confirmedReceipt = hasConfirmedDeliveryReceipt(task.payload);
+      if (!durableDispatchEnabled && !confirmedReceipt) {
+        await failDisabledIntegrationCampaignTask(task.id);
+        const nextTask = await getNextPendingTaskForThread(
+          task.platform,
+          task.externalThreadId,
+        );
+        if (nextTask) {
+          await dispatchPendingIntegrationTask({
+            taskId: nextTask.id,
+            task: {
+              platform: task.platform,
+              externalThreadId: task.externalThreadId,
+              platformContext: nextTask.dispatchScope
+                ? { channelId: nextTask.dispatchScope }
+                : undefined,
+            },
+            event: options.event,
+            baseUrl: options.webhookBaseUrl,
+          });
+        }
+        result.skipped += 1;
+        continue;
+      }
+      const outcome = await dispatchPendingIntegrationTask({
+        taskId: task.id,
+        task: {
+          platform: task.platform,
+          externalThreadId: task.externalThreadId,
+          platformContext: task.dispatchScope
+            ? { channelId: task.dispatchScope }
+            : undefined,
+        },
+        event: options.event,
+        baseUrl: options.webhookBaseUrl,
+        campaignContinuation: true,
+        ...(confirmedReceipt && !durableDispatchEnabled
+          ? { allowPortableConfirmedReceiptReconciliation: true }
+          : {}),
+      });
+      if (outcome === "failed") result.failed += 1;
+      else result.dispatched += 1;
+    } catch (error) {
+      result.failed += 1;
+      console.error(
+        `[integrations] Failed to wake integration campaign ${id}:`,
+        error,
+      );
+    }
+  }
+
+  return result;
+}