@shipfox/api-integration-core 12.1.1 → 12.3.0

package/drizzle/0001_durable_secret_cleanup.sql

@@ -0,0 +1,22 @@
+CREATE TABLE "integrations_secret_cleanups" (
+	"id" uuid PRIMARY KEY DEFAULT uuidv7() NOT NULL,
+	"workspace_id" uuid NOT NULL,
+	"provider" text NOT NULL,
+	"connection_id" uuid NOT NULL,
+	"external_account_id" text NOT NULL,
+	"slug" text NOT NULL,
+	"display_name" text NOT NULL,
+	"lifecycle_status" text NOT NULL,
+	"connection_created_at" timestamp with time zone NOT NULL,
+	"connection_updated_at" timestamp with time zone NOT NULL,
+	"attempt_count" integer DEFAULT 0 NOT NULL,
+	"next_attempt_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"lease_token" uuid,
+	"lease_expires_at" timestamp with time zone,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX "integrations_secret_cleanups_provider_connection_unique" ON "integrations_secret_cleanups" USING btree ("provider","connection_id");--> statement-breakpoint
+CREATE INDEX "integrations_secret_cleanups_connection_id_idx" ON "integrations_secret_cleanups" USING btree ("connection_id");--> statement-breakpoint
+CREATE INDEX "integrations_secret_cleanups_pending_idx" ON "integrations_secret_cleanups" USING btree ("next_attempt_at","created_at","id");

package/drizzle/meta/0001_snapshot.json

@@ -1,6 +1,6 @@
 {
-  "id": "a808bfbe-960e-418d-a72a-d88ddca95096",
-  "prevId": "e3544ba8-0307-4ffd-a646-3851b89bcbfb",
+  "id": "baeb90ab-7ec4-4a2f-8875-37d820097a48",
+  "prevId": "a808bfbe-960e-418d-a72a-d88ddca95096",
   "version": "7",
   "dialect": "postgresql",
   "tables": {
@@ -267,6 +267,184 @@
       "checkConstraints": {},
       "isRLSEnabled": false
     },
+    "public.integrations_secret_cleanups": {
+      "name": "integrations_secret_cleanups",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "primaryKey": true,
+          "notNull": true,
+          "default": "uuidv7()"
+        },
+        "workspace_id": {
+          "name": "workspace_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "provider": {
+          "name": "provider",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "connection_id": {
+          "name": "connection_id",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "external_account_id": {
+          "name": "external_account_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "slug": {
+          "name": "slug",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "display_name": {
+          "name": "display_name",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "lifecycle_status": {
+          "name": "lifecycle_status",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "connection_created_at": {
+          "name": "connection_created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "connection_updated_at": {
+          "name": "connection_updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "attempt_count": {
+          "name": "attempt_count",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "default": 0
+        },
+        "next_attempt_at": {
+          "name": "next_attempt_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "lease_token": {
+          "name": "lease_token",
+          "type": "uuid",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "lease_expires_at": {
+          "name": "lease_expires_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamp with time zone",
+          "primaryKey": false,
+          "notNull": true,
+          "default": "now()"
+        }
+      },
+      "indexes": {
+        "integrations_secret_cleanups_provider_connection_unique": {
+          "name": "integrations_secret_cleanups_provider_connection_unique",
+          "columns": [
+            {
+              "expression": "provider",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "connection_id",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": true,
+          "concurrently": false,
+          "method": "btree",
+          "with": {}
+        },
+        "integrations_secret_cleanups_connection_id_idx": {
+          "name": "integrations_secret_cleanups_connection_id_idx",
+          "columns": [
+            {
+              "expression": "connection_id",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": false,
+          "concurrently": false,
+          "method": "btree",
+          "with": {}
+        },
+        "integrations_secret_cleanups_pending_idx": {
+          "name": "integrations_secret_cleanups_pending_idx",
+          "columns": [
+            {
+              "expression": "next_attempt_at",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "created_at",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            },
+            {
+              "expression": "id",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last"
+            }
+          ],
+          "isUnique": false,
+          "concurrently": false,
+          "method": "btree",
+          "with": {}
+        }
+      },
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "policies": {},
+      "checkConstraints": {},
+      "isRLSEnabled": false
+    },
     "public.integrations_webhook_deliveries": {
       "name": "integrations_webhook_deliveries",
       "schema": "",

package/drizzle/meta/_journal.json

@@ -8,6 +8,13 @@
       "when": 1777467600000,
       "tag": "0000_initial",
       "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "7",
+      "when": 1786020534190,
+      "tag": "0001_durable_secret_cleanup",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@shipfox/api-integration-core",
   "license": "MIT",
-  "version": "12.1.1",
+  "version": "12.3.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/ShipfoxHQ/shipfox.git",
@@ -19,33 +19,34 @@
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "1.29.0",
+    "@temporalio/activity": "1.18.1",
     "@temporalio/workflow": "1.18.1",
     "ajv": "^8.20.0",
     "drizzle-orm": "^0.45.2",
     "zod": "^4.4.3",
-    "@shipfox/api-agent-dto": "12.0.0",
-    "@shipfox/api-auth-context": "12.0.0",
-    "@shipfox/api-workflows-dto": "12.1.0",
-    "@shipfox/api-integration-core-dto": "12.0.0",
-    "@shipfox/api-integration-spi": "1.0.0",
+    "@shipfox/api-agent-dto": "12.2.0",
+    "@shipfox/api-auth-context": "12.2.0",
+    "@shipfox/api-workflows-dto": "12.3.0",
+    "@shipfox/api-integration-core-dto": "12.2.0",
+    "@shipfox/api-integration-spi": "1.1.0",
     "@shipfox/api-workspaces-dto": "12.0.0",
-    "@shipfox/api-integration-gitea": "12.0.0",
-    "@shipfox/api-integration-github": "12.0.0",
-    "@shipfox/api-integration-jira": "12.1.1",
-    "@shipfox/api-integration-linear": "12.0.0",
-    "@shipfox/api-integration-sentry": "12.0.0",
-    "@shipfox/api-integration-slack": "12.0.0",
-    "@shipfox/api-integration-webhook": "12.0.0",
+    "@shipfox/api-integration-gitea": "12.3.0",
+    "@shipfox/api-integration-github": "12.3.0",
+    "@shipfox/api-integration-jira": "12.3.0",
+    "@shipfox/api-integration-linear": "12.3.0",
+    "@shipfox/api-integration-sentry": "12.3.0",
+    "@shipfox/api-integration-slack": "12.3.0",
+    "@shipfox/api-integration-webhook": "12.3.0",
     "@shipfox/config": "1.2.4",
     "@shipfox/inter-module": "0.2.3",
     "@shipfox/node-drizzle": "0.3.5",
-    "@shipfox/node-fastify": "0.4.1",
-    "@shipfox/node-module": "1.0.5",
-    "@shipfox/node-opentelemetry": "0.6.3",
+    "@shipfox/node-fastify": "0.4.2",
+    "@shipfox/node-module": "1.0.6",
+    "@shipfox/node-opentelemetry": "0.6.4",
     "@shipfox/node-error-monitoring": "0.3.0",
     "@shipfox/node-outbox": "0.2.6",
     "@shipfox/node-postgres": "0.5.0",
-    "@shipfox/node-temporal": "0.4.4",
+    "@shipfox/node-temporal": "0.4.5",
     "@shipfox/regex": "0.2.4",
     "@shipfox/redact": "0.2.6"
   },

package/src/core/secret-cleanup.test.ts

@@ -0,0 +1,129 @@
+import {afterEach} from '@shipfox/vitest/vi';
+import {sql} from 'drizzle-orm';
+import {createIntegrationProviderRegistry} from '#core/providers/registry.js';
+import {upsertIntegrationConnection} from '#db/connections.js';
+import {db} from '#db/db.js';
+import {
+  enqueueIntegrationSecretCleanup,
+  listIntegrationSecretCleanups,
+} from '#db/secret-cleanups.js';
+import {processIntegrationSecretCleanups} from './secret-cleanup.js';
+
+afterEach(async () => {
+  await db().execute(sql`TRUNCATE integrations_secret_cleanups CASCADE`);
+});
+
+describe('processIntegrationSecretCleanups', () => {
+  it('retries rows for providers that are not loaded', async () => {
+    const connection = await createCleanupConnection();
+    const now = cleanupNow();
+
+    await expect(
+      processIntegrationSecretCleanups({
+        registry: createIntegrationProviderRegistry([]),
+        now,
+      }),
+    ).resolves.toEqual({claimed: 1, completed: 0, failed: 0, unavailable: 1, unacknowledged: 0});
+
+    const [pending] = await listIntegrationSecretCleanups({connectionId: connection.id});
+    expect(pending).toMatchObject({
+      attemptCount: 1,
+      leaseToken: null,
+      leaseExpiresAt: null,
+    });
+    if (!pending) throw new Error('Expected the cleanup to remain pending');
+    expect(pending.nextAttemptAt.getTime()).toBeGreaterThan(now.getTime());
+  });
+
+  it('counts a retry whose lease is lost before acknowledgement', async () => {
+    const connection = await createCleanupConnection();
+    const provider = {
+      provider: 'slack',
+      displayName: 'Slack',
+      deleteConnectionSecrets: vi.fn(async () => {
+        await db().execute(sql`
+          UPDATE integrations_secret_cleanups
+          SET lease_expires_at = now() - interval '1 second'
+          WHERE connection_id = ${connection.id}
+        `);
+        throw new Error('transient failure');
+      }),
+    };
+
+    await expect(
+      processIntegrationSecretCleanups({
+        registry: createIntegrationProviderRegistry([provider]),
+        now: cleanupNow(),
+      }),
+    ).resolves.toEqual({claimed: 1, completed: 0, failed: 1, unavailable: 0, unacknowledged: 1});
+  });
+
+  it('acknowledges a cleanup when a loaded provider has no secret hook', async () => {
+    const connection = await createCleanupConnection({provider: 'gitea'});
+    const provider = {provider: 'gitea', displayName: 'Gitea'};
+
+    await expect(
+      processIntegrationSecretCleanups({
+        registry: createIntegrationProviderRegistry([provider]),
+        now: cleanupNow(),
+      }),
+    ).resolves.toEqual({claimed: 1, completed: 1, failed: 0, unavailable: 0, unacknowledged: 0});
+    await expect(listIntegrationSecretCleanups({connectionId: connection.id})).resolves.toEqual([]);
+  });
+
+  it('continues a batch after one provider cleanup fails', async () => {
+    const failedConnection = await createCleanupConnection({externalAccountId: 'failed'});
+    const completedConnection = await createCleanupConnection({externalAccountId: 'completed'});
+    const deleteConnectionSecrets = vi.fn((connection: {id: string}) => {
+      if (connection.id === failedConnection.id) {
+        return Promise.reject(new Error('transient failure'));
+      }
+      return Promise.resolve();
+    });
+    const provider = {
+      provider: 'slack',
+      displayName: 'Slack',
+      deleteConnectionSecrets,
+    };
+
+    await expect(
+      processIntegrationSecretCleanups({
+        registry: createIntegrationProviderRegistry([provider]),
+        limit: 2,
+        now: cleanupNow(),
+      }),
+    ).resolves.toEqual({claimed: 2, completed: 1, failed: 1, unavailable: 0, unacknowledged: 0});
+    expect(deleteConnectionSecrets).toHaveBeenCalledWith({
+      id: completedConnection.id,
+      workspaceId: completedConnection.workspaceId,
+      provider: completedConnection.provider,
+      externalAccountId: completedConnection.externalAccountId,
+      slug: completedConnection.slug,
+      displayName: completedConnection.displayName,
+      lifecycleStatus: completedConnection.lifecycleStatus,
+      createdAt: completedConnection.createdAt,
+      updatedAt: completedConnection.updatedAt,
+    });
+    await expect(
+      listIntegrationSecretCleanups({connectionId: failedConnection.id}),
+    ).resolves.toHaveLength(1);
+  });
+});
+
+async function createCleanupConnection(
+  overrides: {externalAccountId?: string; provider?: string} = {},
+) {
+  const connection = await upsertIntegrationConnection({
+    workspaceId: crypto.randomUUID(),
+    provider: overrides.provider ?? 'slack',
+    externalAccountId: overrides.externalAccountId ?? crypto.randomUUID(),
+    slug: `${overrides.provider ?? 'slack'}_${crypto.randomUUID()}`,
+    displayName: overrides.provider ?? 'Slack',
+  });
+  await enqueueIntegrationSecretCleanup({connection});
+  return connection;
+}
+
+function cleanupNow(): Date {
+  return new Date(Date.now() + 60 * 1_000);
+}

package/src/core/secret-cleanup.ts

@@ -0,0 +1,229 @@
+import {reportError} from '@shipfox/node-error-monitoring';
+import {logger} from '@shipfox/node-opentelemetry';
+import type {IntegrationConnection} from '#core/entities/connection.js';
+import type {IntegrationProviderRegistry} from '#core/providers/registry.js';
+import {
+  claimIntegrationSecretCleanups,
+  completeIntegrationSecretCleanup,
+  type IntegrationSecretCleanup,
+  retryIntegrationSecretCleanup,
+} from '#db/secret-cleanups.js';
+
+const RETRY_BASE_DELAY_MS = 60 * 1_000;
+const RETRY_MAX_DELAY_MS = 60 * 60 * 1_000;
+const DEFAULT_BATCH_SIZE = 100;
+const HEARTBEAT_INTERVAL_MS = 15_000;
+const STUCK_CLEANUP_ALERT_CACHE_LIMIT = 10_000;
+// Attempts, not hours: the backoff caps at RETRY_MAX_DELAY_MS, so a stuck row is retried
+// hourly and this threshold lands about a week out. Revisit it alongside that cap.
+const STUCK_CLEANUP_ATTEMPT_COUNT = 7 * 24;
+const reportedStuckCleanupIds = new Set<string>();
+
+export interface ProcessIntegrationSecretCleanupsOptions {
+  registry: IntegrationProviderRegistry;
+  connectionId?: string | undefined;
+  connection?: IntegrationConnection | undefined;
+  limit?: number | undefined;
+  now?: Date | undefined;
+  heartbeat?: (() => void) | undefined;
+}
+
+export interface ProcessIntegrationSecretCleanupsResult {
+  claimed: number;
+  completed: number;
+  failed: number;
+  unavailable: number;
+  /** Rows whose lease was taken over by another sweep before this one acknowledged them. */
+  unacknowledged: number;
+}
+
+type CleanupRetryReason = 'provider-unavailable' | 'cleanup-failed';
+
+export async function processIntegrationSecretCleanups(
+  options: ProcessIntegrationSecretCleanupsOptions,
+): Promise<ProcessIntegrationSecretCleanupsResult> {
+  const cleanups = await claimIntegrationSecretCleanups({
+    connectionId: options.connectionId,
+    limit: options.limit ?? DEFAULT_BATCH_SIZE,
+    now: options.now ?? new Date(),
+  });
+  const result: ProcessIntegrationSecretCleanupsResult = {
+    claimed: cleanups.length,
+    completed: 0,
+    failed: 0,
+    unavailable: 0,
+    unacknowledged: 0,
+  };
+
+  for (const cleanup of cleanups) {
+    // Keep heartbeating through the provider call itself. Beating only between rows would
+    // let one slow deletion trip the activity's heartbeat timeout and kill the whole sweep.
+    options.heartbeat?.();
+    const heartbeatInterval = setInterval(() => options.heartbeat?.(), HEARTBEAT_INTERVAL_MS);
+    // A single row must never strand the rest of the claimed batch under its lease.
+    try {
+      const provider = options.registry
+        .list()
+        .find((candidate) => candidate.provider === cleanup.provider);
+      if (!provider) {
+        result.unavailable += 1;
+        if (!(await scheduleRetry(cleanup, 'provider-unavailable'))) {
+          result.unacknowledged += 1;
+        }
+        continue;
+      }
+
+      // A loaded provider without the hook has no secrets to delete, so the intent is
+      // already satisfied and the row can retire.
+      await provider.deleteConnectionSecrets?.(getCleanupConnection(cleanup, options.connection));
+      const acknowledged = await completeIntegrationSecretCleanup({
+        id: cleanup.id,
+        leaseToken: requireLeaseToken(cleanup),
+      });
+      if (acknowledged) {
+        result.completed += 1;
+      } else {
+        result.unacknowledged += 1;
+        logLeaseLost(cleanup, 'complete');
+      }
+    } catch (error) {
+      result.failed += 1;
+      if (!(await scheduleRetry(cleanup, 'cleanup-failed', error))) {
+        result.unacknowledged += 1;
+      }
+    } finally {
+      clearInterval(heartbeatInterval);
+      options.heartbeat?.();
+    }
+  }
+
+  return result;
+}
+
+function getCleanupConnection(
+  cleanup: IntegrationSecretCleanup,
+  connection: IntegrationConnection | undefined,
+): IntegrationConnection {
+  if (connection?.id === cleanup.connectionId) return connection;
+  return {
+    id: cleanup.connectionId,
+    workspaceId: cleanup.workspaceId,
+    provider: cleanup.provider,
+    externalAccountId: cleanup.externalAccountId,
+    slug: cleanup.slug,
+    displayName: cleanup.displayName,
+    lifecycleStatus: cleanup.lifecycleStatus,
+    createdAt: cleanup.connectionCreatedAt,
+    updatedAt: cleanup.connectionUpdatedAt,
+  };
+}
+
+async function scheduleRetry(
+  cleanup: IntegrationSecretCleanup,
+  reason: CleanupRetryReason,
+  error?: unknown,
+): Promise<boolean> {
+  const delayMs = retryDelayMs(cleanup.attemptCount);
+  logCleanupRetry(cleanup, reason, delayMs, error);
+  if (error !== undefined) {
+    reportError(error, {
+      boundary: 'integration.secret-cleanup',
+      operation: 'delete-connection-secrets',
+      tags: {provider: cleanup.provider},
+    });
+  }
+  // Keep escalating past the threshold. A stuck row means secrets may still exist, and
+  // one missed alert would hide that forever. The bounded per-cleanup cache suppresses
+  // duplicate reports in this worker while allowing a fresh process to re-alert.
+  if (cleanup.attemptCount >= STUCK_CLEANUP_ATTEMPT_COUNT) {
+    const stuckErrorMessage = 'Integration connection secret cleanup exceeded its retry threshold';
+    logger().error(
+      {
+        provider: cleanup.provider,
+        connectionId: cleanup.connectionId,
+        attempt: cleanup.attemptCount,
+      },
+      stuckErrorMessage,
+    );
+    if (!reportedStuckCleanupIds.has(cleanup.id)) {
+      const eventId = reportError(new Error(stuckErrorMessage), {
+        boundary: 'integration.secret-cleanup',
+        operation: 'stuck-cleanup',
+        tags: {provider: cleanup.provider},
+        extra: {cleanupId: cleanup.id},
+      });
+      if (eventId) {
+        reportedStuckCleanupIds.add(cleanup.id);
+        if (reportedStuckCleanupIds.size > STUCK_CLEANUP_ALERT_CACHE_LIMIT) {
+          const oldestCleanupId = reportedStuckCleanupIds.values().next().value;
+          if (oldestCleanupId !== undefined) reportedStuckCleanupIds.delete(oldestCleanupId);
+        }
+      }
+    }
+  }
+
+  try {
+    const rescheduled = await retryIntegrationSecretCleanup({
+      id: cleanup.id,
+      leaseToken: requireLeaseToken(cleanup),
+      delayMs,
+    });
+    if (!rescheduled) {
+      logLeaseLost(cleanup, 'retry');
+      return false;
+    }
+    return true;
+  } catch (retryError) {
+    // The lease expires on its own, so a later sweep still picks this row up.
+    logger().error(
+      {provider: cleanup.provider, connectionId: cleanup.connectionId, err: retryError},
+      'Failed to reschedule integration connection secret cleanup',
+    );
+    reportError(retryError, {
+      boundary: 'integration.secret-cleanup',
+      operation: 'reschedule-cleanup',
+      tags: {provider: cleanup.provider},
+    });
+    return false;
+  }
+}
+
+function retryDelayMs(attemptCount: number): number {
+  return Math.min(RETRY_BASE_DELAY_MS * 2 ** Math.max(0, attemptCount - 1), RETRY_MAX_DELAY_MS);
+}
+
+function requireLeaseToken(cleanup: IntegrationSecretCleanup): string {
+  if (!cleanup.leaseToken) throw new Error('Claimed integration secret cleanup has no lease token');
+  return cleanup.leaseToken;
+}
+
+function logCleanupRetry(
+  cleanup: IntegrationSecretCleanup,
+  reason: CleanupRetryReason,
+  delayMs: number,
+  error: unknown,
+): void {
+  logger().warn(
+    {
+      provider: cleanup.provider,
+      connectionId: cleanup.connectionId,
+      attempt: cleanup.attemptCount,
+      reason,
+      delayMs,
+      ...(error === undefined ? {} : {err: error}),
+    },
+    'Integration connection secret cleanup will be retried',
+  );
+}
+
+function logLeaseLost(cleanup: IntegrationSecretCleanup, operation: 'complete' | 'retry'): void {
+  logger().warn(
+    {
+      provider: cleanup.provider,
+      connectionId: cleanup.connectionId,
+      attempt: cleanup.attemptCount,
+      operation,
+    },
+    'Integration connection secret cleanup lease was lost before acknowledgement',
+  );
+}