@shipfox/api-integration-core 12.2.0 → 12.4.0

package/src/db/secret-cleanups.test.ts

@@ -0,0 +1,113 @@
+import {afterEach} from '@shipfox/vitest/vi';
+import {sql} from 'drizzle-orm';
+import {CLEANUP_SECRETS_ACTIVITY_TIMEOUT_MS} from '#temporal/constants.js';
+import {upsertIntegrationConnection} from './connections.js';
+import {db} from './db.js';
+import {
+  claimIntegrationSecretCleanups,
+  completeIntegrationSecretCleanup,
+  enqueueIntegrationSecretCleanup,
+  retryIntegrationSecretCleanup,
+} from './secret-cleanups.js';
+
+const now = new Date(Date.now() + 60 * 1_000);
+
+afterEach(async () => {
+  await db().execute(sql`TRUNCATE integrations_secret_cleanups CASCADE`);
+});
+
+describe('integration secret cleanup persistence', () => {
+  it('keeps the default lease longer than the cleanup activity timeout', async () => {
+    await createCleanupConnection();
+
+    const [claimed] = await claimIntegrationSecretCleanups({limit: 1, now});
+
+    if (!claimed?.leaseExpiresAt) throw new Error('Expected the cleanup to have a lease');
+    expect(claimed.leaseExpiresAt.getTime() - now.getTime()).toBeGreaterThan(
+      CLEANUP_SECRETS_ACTIVITY_TIMEOUT_MS,
+    );
+  });
+
+  it('claims a due row once and reclaims it after its lease expires', async () => {
+    const connection = await createCleanupConnection();
+    const [queued] = await claimIntegrationSecretCleanups({limit: 1, now, leaseDurationMs: 1_000});
+
+    expect(queued).toMatchObject({
+      connectionId: connection.id,
+      attemptCount: 1,
+      leaseToken: expect.any(String),
+    });
+    await expect(claimIntegrationSecretCleanups({limit: 1, now})).resolves.toEqual([]);
+
+    const wrongLeaseToken = crypto.randomUUID();
+    await expect(
+      completeIntegrationSecretCleanup({
+        id: queued?.id ?? '',
+        leaseToken: wrongLeaseToken,
+        now,
+      }),
+    ).resolves.toBe(false);
+    await expect(
+      retryIntegrationSecretCleanup({
+        id: queued?.id ?? '',
+        leaseToken: wrongLeaseToken,
+        delayMs: 1_000,
+        now,
+      }),
+    ).resolves.toBe(false);
+
+    const [reclaimed] = await claimIntegrationSecretCleanups({
+      limit: 1,
+      now: new Date(now.getTime() + 1_001),
+      leaseDurationMs: 1_000,
+    });
+    expect(reclaimed).toMatchObject({
+      id: queued?.id,
+      attemptCount: 2,
+      leaseToken: expect.any(String),
+    });
+
+    await expect(
+      completeIntegrationSecretCleanup({
+        id: reclaimed?.id ?? '',
+        leaseToken: reclaimed?.leaseToken ?? '',
+        now: new Date(now.getTime() + 1_001),
+      }),
+    ).resolves.toBe(true);
+  });
+
+  it('honors the batch limit and retry backoff', async () => {
+    await createCleanupConnection({externalAccountId: 'first'});
+    await createCleanupConnection({externalAccountId: 'second'});
+
+    const firstClaim = await claimIntegrationSecretCleanups({limit: 1, now});
+    expect(firstClaim).toHaveLength(1);
+    const first = firstClaim[0];
+    if (!first?.leaseToken) throw new Error('Expected the first cleanup to have a lease');
+
+    await expect(
+      retryIntegrationSecretCleanup({
+        id: first.id,
+        leaseToken: first.leaseToken,
+        delayMs: 5_000,
+        now,
+      }),
+    ).resolves.toBe(true);
+
+    const secondClaim = await claimIntegrationSecretCleanups({limit: 2, now});
+    expect(secondClaim).toHaveLength(1);
+    expect(secondClaim[0]?.id).not.toBe(first.id);
+  });
+});
+
+async function createCleanupConnection(overrides: {externalAccountId?: string} = {}) {
+  const connection = await upsertIntegrationConnection({
+    workspaceId: crypto.randomUUID(),
+    provider: 'slack',
+    externalAccountId: overrides.externalAccountId ?? crypto.randomUUID(),
+    slug: `slack_${crypto.randomUUID()}`,
+    displayName: 'Slack',
+  });
+  await enqueueIntegrationSecretCleanup({connection});
+  return connection;
+}

package/src/db/secret-cleanups.ts

@@ -0,0 +1,209 @@
+import {and, asc, eq, gt, inArray, isNull, lte, or, sql} from 'drizzle-orm';
+import type {IntegrationConnection} from '#core/entities/connection.js';
+import {db} from './db.js';
+import {
+  type IntegrationSecretCleanupDb,
+  integrationSecretCleanups,
+} from './schema/secret-cleanups.js';
+
+// Keep the lease longer than cleanupIntegrationSecretsCron's 5-minute activity timeout so
+// a timed-out sweep cannot have its claimed rows reclaimed while it is still running.
+const DEFAULT_LEASE_DURATION_MS = 10 * 60 * 1_000;
+
+export interface EnqueueIntegrationSecretCleanupParams {
+  connection: IntegrationConnection;
+  now?: Date | undefined;
+}
+
+export interface IntegrationSecretCleanup {
+  id: string;
+  workspaceId: string;
+  provider: string;
+  connectionId: string;
+  externalAccountId: string;
+  slug: string;
+  displayName: string;
+  lifecycleStatus: IntegrationConnection['lifecycleStatus'];
+  connectionCreatedAt: Date;
+  connectionUpdatedAt: Date;
+  attemptCount: number;
+  nextAttemptAt: Date;
+  leaseToken: string | null;
+  leaseExpiresAt: Date | null;
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+type IntegrationDb = ReturnType<typeof db>;
+type IntegrationTx = Parameters<Parameters<IntegrationDb['transaction']>[0]>[0];
+
+export async function enqueueIntegrationSecretCleanup(
+  params: EnqueueIntegrationSecretCleanupParams,
+  options: {tx?: IntegrationDb | IntegrationTx | undefined} = {},
+): Promise<void> {
+  const executor = options.tx ?? db();
+  await executor
+    .insert(integrationSecretCleanups)
+    .values({
+      workspaceId: params.connection.workspaceId,
+      provider: params.connection.provider,
+      connectionId: params.connection.id,
+      externalAccountId: params.connection.externalAccountId,
+      slug: params.connection.slug,
+      displayName: params.connection.displayName,
+      lifecycleStatus: params.connection.lifecycleStatus,
+      connectionCreatedAt: params.connection.createdAt,
+      connectionUpdatedAt: params.connection.updatedAt,
+      // Claims compare against the application clock, so the first due time must come
+      // from that clock too. A database clock even milliseconds ahead would otherwise
+      // hide the row from the sweep that runs right after this insert commits.
+      nextAttemptAt: params.now ?? new Date(),
+    })
+    .onConflictDoNothing({
+      target: [integrationSecretCleanups.provider, integrationSecretCleanups.connectionId],
+    });
+}
+
+export interface ClaimIntegrationSecretCleanupsParams {
+  limit: number;
+  connectionId?: string | undefined;
+  now?: Date | undefined;
+  leaseDurationMs?: number | undefined;
+}
+
+export async function claimIntegrationSecretCleanups(
+  params: ClaimIntegrationSecretCleanupsParams,
+): Promise<IntegrationSecretCleanup[]> {
+  if (!Number.isInteger(params.limit) || params.limit <= 0) {
+    throw new Error('Secret cleanup claim limit must be a positive integer');
+  }
+  const now = params.now ?? new Date();
+  const leaseDurationMs = params.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
+  if (!Number.isInteger(leaseDurationMs) || leaseDurationMs <= 0) {
+    throw new Error('Secret cleanup lease duration must be a positive integer');
+  }
+
+  return await db().transaction(async (tx) => {
+    const conditions = [
+      lte(integrationSecretCleanups.nextAttemptAt, now),
+      or(
+        isNull(integrationSecretCleanups.leaseExpiresAt),
+        lte(integrationSecretCleanups.leaseExpiresAt, now),
+      ),
+      ...(params.connectionId
+        ? [eq(integrationSecretCleanups.connectionId, params.connectionId)]
+        : []),
+    ];
+    const candidates = await tx
+      .select({id: integrationSecretCleanups.id})
+      .from(integrationSecretCleanups)
+      .where(and(...conditions))
+      .orderBy(
+        asc(integrationSecretCleanups.nextAttemptAt),
+        asc(integrationSecretCleanups.createdAt),
+        asc(integrationSecretCleanups.id),
+      )
+      .limit(params.limit)
+      .for('update', {skipLocked: true});
+    if (candidates.length === 0) return [];
+
+    const rows = await tx
+      .update(integrationSecretCleanups)
+      .set({
+        attemptCount: sql`${integrationSecretCleanups.attemptCount} + 1`,
+        leaseToken: sql`gen_random_uuid()`,
+        leaseExpiresAt: new Date(now.getTime() + leaseDurationMs),
+        updatedAt: now,
+      })
+      .where(
+        inArray(
+          integrationSecretCleanups.id,
+          candidates.map((candidate) => candidate.id),
+        ),
+      )
+      .returning();
+    return rows.map(toIntegrationSecretCleanup);
+  });
+}
+
+export async function completeIntegrationSecretCleanup(params: {
+  id: string;
+  leaseToken: string;
+  now?: Date | undefined;
+}): Promise<boolean> {
+  const now = params.now ?? new Date();
+  const result = await db()
+    .delete(integrationSecretCleanups)
+    .where(
+      and(
+        eq(integrationSecretCleanups.id, params.id),
+        eq(integrationSecretCleanups.leaseToken, params.leaseToken),
+        gt(integrationSecretCleanups.leaseExpiresAt, now),
+      ),
+    );
+  return (result.rowCount ?? 0) > 0;
+}
+
+export async function retryIntegrationSecretCleanup(params: {
+  id: string;
+  leaseToken: string;
+  delayMs: number;
+  now?: Date | undefined;
+}): Promise<boolean> {
+  if (!Number.isInteger(params.delayMs) || params.delayMs < 0) {
+    throw new Error('Secret cleanup retry delay must be a non-negative integer');
+  }
+  const now = params.now ?? new Date();
+  const result = await db()
+    .update(integrationSecretCleanups)
+    .set({
+      nextAttemptAt: new Date(now.getTime() + params.delayMs),
+      leaseToken: null,
+      leaseExpiresAt: null,
+      updatedAt: now,
+    })
+    .where(
+      and(
+        eq(integrationSecretCleanups.id, params.id),
+        eq(integrationSecretCleanups.leaseToken, params.leaseToken),
+        gt(integrationSecretCleanups.leaseExpiresAt, now),
+      ),
+    );
+  return (result.rowCount ?? 0) > 0;
+}
+
+export async function listIntegrationSecretCleanups(
+  params: {connectionId?: string | undefined} = {},
+): Promise<IntegrationSecretCleanup[]> {
+  const rows = await db()
+    .select()
+    .from(integrationSecretCleanups)
+    .where(
+      params.connectionId
+        ? eq(integrationSecretCleanups.connectionId, params.connectionId)
+        : undefined,
+    )
+    .orderBy(asc(integrationSecretCleanups.createdAt), asc(integrationSecretCleanups.id));
+  return rows.map(toIntegrationSecretCleanup);
+}
+
+function toIntegrationSecretCleanup(row: IntegrationSecretCleanupDb): IntegrationSecretCleanup {
+  return {
+    id: row.id,
+    workspaceId: row.workspaceId,
+    provider: row.provider,
+    connectionId: row.connectionId,
+    externalAccountId: row.externalAccountId,
+    slug: row.slug,
+    displayName: row.displayName,
+    lifecycleStatus: row.lifecycleStatus,
+    connectionCreatedAt: row.connectionCreatedAt,
+    connectionUpdatedAt: row.connectionUpdatedAt,
+    attemptCount: row.attemptCount,
+    nextAttemptAt: row.nextAttemptAt,
+    leaseToken: row.leaseToken,
+    leaseExpiresAt: row.leaseExpiresAt,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+  };
+}

package/src/index.ts

@@ -267,13 +267,18 @@ export async function createIntegrationsContext(
       {
         taskQueue: INTEGRATIONS_MAINTENANCE_TASK_QUEUE,
         workflowsPath: maintenanceWorkflowsPath,
-        activities: createIntegrationsMaintenanceActivities,
+        activities: () => createIntegrationsMaintenanceActivities({registry}),
         workflows: [
           {
             name: 'pruneWebhookDeliveriesCron',
             id: 'integrations-prune-webhook-deliveries',
             cronSchedule: '0 3 * * *',
           },
+          {
+            name: 'cleanupIntegrationSecretsCron',
+            id: 'integrations-cleanup-secret-namespaces',
+            cronSchedule: '*/5 * * * *',
+          },
         ],
       },
       ...parts.flatMap((part) => part.workers ?? []),

package/src/presentation/routes/manage-connections.test.ts

@@ -1,4 +1,7 @@
+import {createIntegrationProviderRegistry} from '#core/providers/registry.js';
+import {processIntegrationSecretCleanups} from '#core/secret-cleanup.js';
 import {getIntegrationConnectionById, upsertIntegrationConnection} from '#db/connections.js';
+import {listIntegrationSecretCleanups} from '#db/secret-cleanups.js';
 import {createTestApp, sourceProvider, useIntegrationRouteTest} from '#test/route-utils.js';
 
 describe('PATCH /integration-connections/:connectionId', () => {
@@ -111,6 +114,7 @@ describe('DELETE /integration-connections/:connectionId', () => {
   });
 
   it('runs provider cleanup hooks while retaining ownership of the core row', async () => {
+    const deleteConnectionRemoteResources = vi.fn(() => Promise.resolve(undefined));
     const deleteConnectionRecords = vi.fn(() => Promise.resolve());
     const deleteConnectionSecrets = vi.fn(() => Promise.resolve());
     const app = await createTestApp([
@@ -118,6 +122,7 @@ describe('DELETE /integration-connections/:connectionId', () => {
         provider: 'slack',
         displayName: 'Slack',
         adapters: {},
+        deleteConnectionRemoteResources,
         deleteConnectionRecords,
         deleteConnectionSecrets,
       }),
@@ -137,6 +142,7 @@ describe('DELETE /integration-connections/:connectionId', () => {
     });
 
     expect(res.statusCode).toBe(204);
+    expect(deleteConnectionRemoteResources).toHaveBeenCalledWith(connection);
     expect(deleteConnectionRecords).toHaveBeenCalledWith(connection, {
       tx: expect.anything(),
     });
@@ -144,6 +150,103 @@ describe('DELETE /integration-connections/:connectionId', () => {
     await expect(getIntegrationConnectionById(connection.id)).resolves.toBeUndefined();
   });
 
+  it('runs prepared remote cleanup after the local deletion commits', async () => {
+    const events: string[] = [];
+    const deleteConnectionRemoteResources = vi.fn(() => {
+      events.push('prepare');
+      return Promise.resolve(async () => {
+        events.push('remote');
+        await expect(getIntegrationConnectionById(connection.id)).resolves.toBeUndefined();
+      });
+    });
+    const deleteConnectionRecords = vi.fn(() => {
+      events.push('records');
+      return Promise.resolve();
+    });
+    const deleteConnectionSecrets = vi.fn(() => {
+      events.push('secrets');
+      return Promise.resolve();
+    });
+    const app = await createTestApp([
+      sourceProvider({
+        provider: 'slack',
+        displayName: 'Slack',
+        adapters: {},
+        deleteConnectionRemoteResources,
+        deleteConnectionRecords,
+        deleteConnectionSecrets,
+      }),
+    ]);
+    const connection = await upsertIntegrationConnection({
+      workspaceId: context.workspaceId,
+      provider: 'slack',
+      externalAccountId: 'T123',
+      slug: 'slack_acme',
+      displayName: 'Slack Acme',
+    });
+
+    const res = await app.inject({
+      method: 'DELETE',
+      url: `/integration-connections/${connection.id}`,
+      headers: {authorization: 'Bearer user'},
+    });
+
+    expect(res.statusCode).toBe(204);
+    expect(events).toEqual(['prepare', 'records', 'remote', 'secrets']);
+    await expect(getIntegrationConnectionById(connection.id)).resolves.toBeUndefined();
+  });
+
+  it('holds the provider deletion lock across local and remote cleanup', async () => {
+    const events: string[] = [];
+    const withConnectionDeletionLock = vi.fn(
+      async (_connection: unknown, fn: () => Promise<void>) => {
+        events.push('lock-enter');
+        await fn();
+        events.push('lock-exit');
+      },
+    );
+    const app = await createTestApp([
+      sourceProvider({
+        provider: 'slack',
+        displayName: 'Slack',
+        adapters: {},
+        withConnectionDeletionLock,
+        deleteConnectionRemoteResources: vi.fn(() => {
+          events.push('prepare');
+          return Promise.resolve(() => {
+            events.push('remote');
+            return Promise.resolve();
+          });
+        }),
+        deleteConnectionRecords: vi.fn(() => {
+          events.push('records');
+          return Promise.resolve();
+        }),
+        deleteConnectionSecrets: vi.fn(() => {
+          events.push('secrets');
+          return Promise.resolve();
+        }),
+      }),
+    ]);
+    const connection = await upsertIntegrationConnection({
+      workspaceId: context.workspaceId,
+      provider: 'slack',
+      externalAccountId: 'T123',
+      slug: 'slack_acme',
+      displayName: 'Slack Acme',
+    });
+
+    const res = await app.inject({
+      method: 'DELETE',
+      url: `/integration-connections/${connection.id}`,
+      headers: {authorization: 'Bearer user'},
+    });
+
+    expect(res.statusCode).toBe(204);
+    expect(events).toEqual(['lock-enter', 'prepare', 'records', 'remote', 'secrets', 'lock-exit']);
+    expect(withConnectionDeletionLock).toHaveBeenCalledWith(connection, expect.any(Function));
+  });
+
   it('keeps the connection when provider record cleanup fails', async () => {
     const deleteConnectionSecrets = vi.fn(() => Promise.resolve());
     const app = await createTestApp([
@@ -176,6 +279,68 @@ describe('DELETE /integration-connections/:connectionId', () => {
     expect(deleteConnectionSecrets).not.toHaveBeenCalled();
   });
 
+  it('continues connection deletion when remote cleanup preparation fails', async () => {
+    const deleteConnectionRemoteResources = vi.fn(() =>
+      Promise.reject(new Error('remote cleanup failed')),
+    );
+    const app = await createTestApp([
+      sourceProvider({
+        provider: 'slack',
+        displayName: 'Slack',
+        adapters: {},
+        deleteConnectionRemoteResources,
+      }),
+    ]);
+    const connection = await upsertIntegrationConnection({
+      workspaceId: context.workspaceId,
+      provider: 'slack',
+      externalAccountId: 'T123',
+      slug: 'slack_acme',
+      displayName: 'Slack Acme',
+    });
+
+    const res = await app.inject({
+      method: 'DELETE',
+      url: `/integration-connections/${connection.id}`,
+      headers: {authorization: 'Bearer user'},
+    });
+
+    expect(res.statusCode).toBe(204);
+    expect(deleteConnectionRemoteResources).toHaveBeenCalledWith(connection);
+    await expect(getIntegrationConnectionById(connection.id)).resolves.toBeUndefined();
+  });
+
+  it('continues connection deletion when post-commit remote cleanup fails', async () => {
+    const deleteConnectionRemoteResources = vi.fn(() =>
+      Promise.resolve(() => Promise.reject(new Error('remote cleanup failed'))),
+    );
+    const app = await createTestApp([
+      sourceProvider({
+        provider: 'slack',
+        displayName: 'Slack',
+        adapters: {},
+        deleteConnectionRemoteResources,
+      }),
+    ]);
+    const connection = await upsertIntegrationConnection({
+      workspaceId: context.workspaceId,
+      provider: 'slack',
+      externalAccountId: 'T123',
+      slug: 'slack_acme',
+      displayName: 'Slack Acme',
+    });
+
+    const res = await app.inject({
+      method: 'DELETE',
+      url: `/integration-connections/${connection.id}`,
+      headers: {authorization: 'Bearer user'},
+    });
+
+    expect(res.statusCode).toBe(204);
+    expect(deleteConnectionRemoteResources).toHaveBeenCalledWith(connection);
+    await expect(getIntegrationConnectionById(connection.id)).resolves.toBeUndefined();
+  });
+
   it('deletes the connection when provider secret cleanup fails after commit', async () => {
     const app = await createTestApp([
       sourceProvider({
@@ -201,6 +366,73 @@ describe('DELETE /integration-connections/:connectionId', () => {
 
     expect(res.statusCode).toBe(204);
     await expect(getIntegrationConnectionById(connection.id)).resolves.toBeUndefined();
+    await expect(
+      listIntegrationSecretCleanups({connectionId: connection.id}),
+    ).resolves.toMatchObject([
+      {
+        provider: 'slack',
+        connectionId: connection.id,
+        attemptCount: 1,
+        leaseToken: null,
+      },
+    ]);
+  });
+
+  it('retries provider secret cleanup from a durable post-commit record', async () => {
+    const deleteConnectionSecrets = vi
+      .fn<() => Promise<void>>()
+      .mockRejectedValueOnce(new Error('transient cleanup failure'))
+      .mockResolvedValue(undefined);
+    const provider = sourceProvider({
+      provider: 'slack',
+      displayName: 'Slack',
+      adapters: {},
+      deleteConnectionSecrets,
+    });
+    const app = await createTestApp([provider]);
+    const connection = await upsertIntegrationConnection({
+      workspaceId: context.workspaceId,
+      provider: 'slack',
+      externalAccountId: 'T123',
+      slug: 'slack_acme',
+      displayName: 'Slack Acme',
+    });
+
+    const res = await app.inject({
+      method: 'DELETE',
+      url: `/integration-connections/${connection.id}`,
+      headers: {authorization: 'Bearer user'},
+    });
+
+    const pending = await listIntegrationSecretCleanups({connectionId: connection.id});
+    expect(res.statusCode).toBe(204);
+    expect(pending).toHaveLength(1);
+    expect(pending[0]).toMatchObject({
+      workspaceId: context.workspaceId,
+      provider: 'slack',
+      connectionId: connection.id,
+      attemptCount: 1,
+      leaseToken: null,
+    });
+
+    const result = await processIntegrationSecretCleanups({
+      registry: createIntegrationProviderRegistry([provider]),
+      connectionId: connection.id,
+      now: new Date(Date.now() + 5 * 60 * 1_000),
+      limit: 1,
+    });
+
+    expect(result).toEqual({
+      claimed: 1,
+      completed: 1,
+      failed: 0,
+      unavailable: 0,
+      unacknowledged: 0,
+    });
+    await expect(listIntegrationSecretCleanups({connectionId: connection.id})).resolves.toEqual([]);
+    expect(deleteConnectionSecrets).toHaveBeenCalledTimes(2);
+    expect(deleteConnectionSecrets).toHaveBeenNthCalledWith(1, connection);
+    expect(deleteConnectionSecrets).toHaveBeenNthCalledWith(2, connection);
   });
 
   it('deletes an unregistered provider connection without provider cleanup', async () => {

package/src/presentation/routes/manage-connections.ts

@@ -6,12 +6,14 @@ import {
 import {ClientError, defineRoute} from '@shipfox/node-fastify';
 import {z} from 'zod';
 import type {IntegrationProviderRegistry} from '#core/providers/registry.js';
+import {processIntegrationSecretCleanups} from '#core/secret-cleanup.js';
 import {
   deleteIntegrationConnection,
   getIntegrationConnectionById,
   updateIntegrationConnectionLifecycleStatus,
 } from '#db/connections.js';
 import {db} from '#db/db.js';
+import {enqueueIntegrationSecretCleanup} from '#db/secret-cleanups.js';
 import {toIntegrationConnectionDto} from '#presentation/dto/integrations.js';
 
 const connectionParamsSchema = z.object({
@@ -75,6 +77,7 @@ export function createDeleteIntegrationConnectionRoute(registry: IntegrationProv
         .list()
         .find((candidate) => candidate.provider === connection.provider);
       const hasCleanupHooks =
+        provider?.deleteConnectionRemoteResources !== undefined ||
         provider?.deleteConnectionRecords !== undefined ||
         provider?.deleteConnectionSecrets !== undefined;
       if (!hasCleanupHooks) {
@@ -83,17 +86,48 @@ export function createDeleteIntegrationConnectionRoute(registry: IntegrationProv
           'Deleting integration connection without provider cleanup',
         );
       }
-      await db().transaction(async (tx) => {
-        await provider?.deleteConnectionRecords?.(connection, {tx});
-        await deleteIntegrationConnection({id: connection.id}, {tx});
-      });
-      try {
-        await provider?.deleteConnectionSecrets?.(connection);
-      } catch (error) {
-        request.log.error(
-          {connectionId: connection.id, provider: connection.provider, err: error},
-          'Integration connection secret cleanup failed after connection deletion',
-        );
+      const deleteConnection = async (): Promise<void> => {
+        let deleteRemoteResources: (() => Promise<void>) | undefined;
+        try {
+          deleteRemoteResources = await provider?.deleteConnectionRemoteResources?.(connection);
+        } catch (error) {
+          request.log.error(
+            {connectionId: connection.id, provider: connection.provider, err: error},
+            'Integration connection remote cleanup preparation failed',
+          );
+        }
+        await db().transaction(async (tx) => {
+          await provider?.deleteConnectionRecords?.(connection, {tx});
+          await enqueueIntegrationSecretCleanup({connection}, {tx});
+          await deleteIntegrationConnection({id: connection.id}, {tx});
+        });
+        try {
+          await deleteRemoteResources?.();
+        } catch (error) {
+          request.log.error(
+            {connectionId: connection.id, provider: connection.provider, err: error},
+            'Integration connection remote cleanup failed after deletion',
+          );
+        }
+        try {
+          await processIntegrationSecretCleanups({
+            registry,
+            connectionId: connection.id,
+            connection,
+            limit: 1,
+          });
+        } catch (error) {
+          // The durable cleanup row survives, so a later sweep retries this connection.
+          request.log.error(
+            {connectionId: connection.id, provider: connection.provider, err: error},
+            'Integration connection secret cleanup failed after connection deletion',
+          );
+        }
+      };
+      if (provider?.withConnectionDeletionLock) {
+        await provider.withConnectionDeletionLock(connection, deleteConnection);
+      } else {
+        await deleteConnection();
       }
       reply.status(204);
     },