@shipfox/api-integration-core 12.1.1 → 12.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +11 -8
- package/.turbo/turbo-type.log +0 -1
- package/CHANGELOG.md +44 -0
- package/dist/core/secret-cleanup.d.ts +20 -0
- package/dist/core/secret-cleanup.d.ts.map +1 -0
- package/dist/core/secret-cleanup.js +178 -0
- package/dist/core/secret-cleanup.js.map +1 -0
- package/dist/db/connections.d.ts.map +1 -1
- package/dist/db/connections.js +66 -13
- package/dist/db/connections.js.map +1 -1
- package/dist/db/db.d.ts +562 -0
- package/dist/db/db.d.ts.map +1 -1
- package/dist/db/db.js +2 -0
- package/dist/db/db.js.map +1 -1
- package/dist/db/schema/secret-cleanups.d.ts +284 -0
- package/dist/db/schema/secret-cleanups.d.ts.map +1 -0
- package/dist/db/schema/secret-cleanups.js +39 -0
- package/dist/db/schema/secret-cleanups.js.map +1 -0
- package/dist/db/secret-cleanups.d.ts +52 -0
- package/dist/db/secret-cleanups.d.ts.map +1 -0
- package/dist/db/secret-cleanups.js +105 -0
- package/dist/db/secret-cleanups.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -1
- package/dist/index.js.map +1 -1
- package/dist/presentation/routes/manage-connections.d.ts.map +1 -1
- package/dist/presentation/routes/manage-connections.js +57 -18
- package/dist/presentation/routes/manage-connections.js.map +1 -1
- package/dist/providers/jira.d.ts.map +1 -1
- package/dist/providers/jira.js +67 -14
- package/dist/providers/jira.js.map +1 -1
- package/dist/temporal/activities/index.d.ts +5 -1
- package/dist/temporal/activities/index.d.ts.map +1 -1
- package/dist/temporal/activities/index.js +8 -2
- package/dist/temporal/activities/index.js.map +1 -1
- package/dist/temporal/constants.d.ts +1 -0
- package/dist/temporal/constants.d.ts.map +1 -1
- package/dist/temporal/constants.js +1 -0
- package/dist/temporal/constants.js.map +1 -1
- package/dist/temporal/workflows/cleanup-integration-secrets-cron.d.ts +2 -0
- package/dist/temporal/workflows/cleanup-integration-secrets-cron.d.ts.map +1 -0
- package/dist/temporal/workflows/cleanup-integration-secrets-cron.js +21 -0
- package/dist/temporal/workflows/cleanup-integration-secrets-cron.js.map +1 -0
- package/dist/temporal/workflows/index.bundle.js +64 -3
- package/dist/temporal/workflows/index.d.ts +1 -0
- package/dist/temporal/workflows/index.d.ts.map +1 -1
- package/dist/temporal/workflows/index.js +1 -0
- package/dist/temporal/workflows/index.js.map +1 -1
- package/dist/tsconfig.test.tsbuildinfo +1 -1
- package/drizzle/0001_durable_secret_cleanup.sql +22 -0
- package/drizzle/meta/0001_snapshot.json +180 -2
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +18 -17
- package/src/core/secret-cleanup.test.ts +129 -0
- package/src/core/secret-cleanup.ts +229 -0
- package/src/db/connections.test.ts +90 -0
- package/src/db/connections.ts +92 -13
- package/src/db/db.ts +2 -0
- package/src/db/schema/secret-cleanups.ts +42 -0
- package/src/db/secret-cleanups.test.ts +113 -0
- package/src/db/secret-cleanups.ts +209 -0
- package/src/index.ts +6 -1
- package/src/presentation/routes/manage-connections.test.ts +232 -0
- package/src/presentation/routes/manage-connections.ts +45 -11
- package/src/providers/jira.test.ts +133 -0
- package/src/providers/jira.ts +78 -13
- package/src/temporal/activities/index.ts +11 -1
- package/src/temporal/constants.ts +2 -0
- package/src/temporal/workflows/cleanup-integration-secrets-cron.ts +20 -0
- package/src/temporal/workflows/index.ts +1 -0
- package/test/env.ts +3 -0
- package/test/globalSetup.ts +1 -0
- package/test/route-utils.ts +3 -0
- package/test/setup.ts +12 -0
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import {INTEGRATION_CONNECTION_AVAILABLE} from '@shipfox/api-integration-core-dto';
|
|
1
2
|
import {upsertGithubInstallation} from '@shipfox/api-integration-github';
|
|
2
3
|
import {ConnectionSlugConflictError} from '@shipfox/api-integration-spi';
|
|
4
|
+
import {sql} from 'drizzle-orm';
|
|
3
5
|
import {IntegrationConnectionAlreadyExistsError} from '#core/errors.js';
|
|
4
6
|
import {
|
|
5
7
|
createIntegrationConnection,
|
|
@@ -13,6 +15,16 @@ import {
|
|
|
13
15
|
upsertIntegrationConnection,
|
|
14
16
|
} from './connections.js';
|
|
15
17
|
import {db} from './db.js';
|
|
18
|
+
import {integrationsOutbox} from './schema/outbox.js';
|
|
19
|
+
|
|
20
|
+
function connectionEvents(connectionId: string) {
|
|
21
|
+
return db()
|
|
22
|
+
.select()
|
|
23
|
+
.from(integrationsOutbox)
|
|
24
|
+
.where(
|
|
25
|
+
sql`${integrationsOutbox.eventType} = ${INTEGRATION_CONNECTION_AVAILABLE} AND ${integrationsOutbox.payload}->>'connectionId' = ${connectionId}`,
|
|
26
|
+
);
|
|
27
|
+
}
|
|
16
28
|
|
|
17
29
|
describe('integration connection queries', () => {
|
|
18
30
|
let workspaceId: string;
|
|
@@ -41,6 +53,31 @@ describe('integration connection queries', () => {
|
|
|
41
53
|
expect(second.id).toBe(first.id);
|
|
42
54
|
expect(second.displayName).toBe('Renamed Debug');
|
|
43
55
|
expect(second.slug).toBe('gitea_owner');
|
|
56
|
+
expect(await connectionEvents(first.id)).toHaveLength(1);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('publishes availability when an upsert activates an existing connection', async () => {
|
|
60
|
+
const connection = await upsertIntegrationConnection({
|
|
61
|
+
workspaceId,
|
|
62
|
+
provider: 'linear',
|
|
63
|
+
externalAccountId: 'linear-acme',
|
|
64
|
+
slug: 'linear_acme',
|
|
65
|
+
displayName: 'Linear Acme',
|
|
66
|
+
lifecycleStatus: 'disabled',
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
expect(await connectionEvents(connection.id)).toHaveLength(0);
|
|
70
|
+
|
|
71
|
+
await upsertIntegrationConnection({
|
|
72
|
+
workspaceId,
|
|
73
|
+
provider: 'linear',
|
|
74
|
+
externalAccountId: 'linear-acme',
|
|
75
|
+
slug: 'linear_acme',
|
|
76
|
+
displayName: 'Linear Acme',
|
|
77
|
+
lifecycleStatus: 'active',
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
expect(await connectionEvents(connection.id)).toHaveLength(1);
|
|
44
81
|
});
|
|
45
82
|
|
|
46
83
|
it('allows multiple same-provider connections when external account differs', async () => {
|
|
@@ -148,6 +185,7 @@ describe('integration connection queries', () => {
|
|
|
148
185
|
expect(connections).toHaveLength(1);
|
|
149
186
|
expect(connections[0]?.id).toBe(first.id);
|
|
150
187
|
expect(connections[0]?.displayName).toBe('Stripe');
|
|
188
|
+
expect(await connectionEvents(first.id)).toHaveLength(1);
|
|
151
189
|
});
|
|
152
190
|
|
|
153
191
|
it('reports slug collisions separately from duplicate external accounts', async () => {
|
|
@@ -252,6 +290,51 @@ describe('integration connection queries', () => {
|
|
|
252
290
|
expect(updated?.lifecycleStatus).toBe('disabled');
|
|
253
291
|
const reloaded = await getIntegrationConnectionById(connection.id);
|
|
254
292
|
expect(reloaded?.lifecycleStatus).toBe('disabled');
|
|
293
|
+
expect(await connectionEvents(connection.id)).toHaveLength(1);
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
it('publishes availability when a disabled connection becomes active', async () => {
|
|
297
|
+
const connection = await upsertIntegrationConnection({
|
|
298
|
+
workspaceId,
|
|
299
|
+
provider: 'linear',
|
|
300
|
+
externalAccountId: 'linear-acme',
|
|
301
|
+
slug: 'linear_acme',
|
|
302
|
+
displayName: 'Linear Acme',
|
|
303
|
+
lifecycleStatus: 'disabled',
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
expect(await connectionEvents(connection.id)).toHaveLength(0);
|
|
307
|
+
|
|
308
|
+
await updateIntegrationConnectionLifecycleStatus({
|
|
309
|
+
id: connection.id,
|
|
310
|
+
lifecycleStatus: 'active',
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
const events = await connectionEvents(connection.id);
|
|
314
|
+
expect(events).toHaveLength(1);
|
|
315
|
+
expect(events[0]?.payload).toEqual({
|
|
316
|
+
provider: 'linear',
|
|
317
|
+
workspaceId,
|
|
318
|
+
connectionId: connection.id,
|
|
319
|
+
slug: 'linear_acme',
|
|
320
|
+
});
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
it('does not republish availability when an active connection stays active', async () => {
|
|
324
|
+
const connection = await upsertIntegrationConnection({
|
|
325
|
+
workspaceId,
|
|
326
|
+
provider: 'linear',
|
|
327
|
+
externalAccountId: 'linear-acme',
|
|
328
|
+
slug: 'linear_acme',
|
|
329
|
+
displayName: 'Linear Acme',
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
await updateIntegrationConnectionLifecycleStatus({
|
|
333
|
+
id: connection.id,
|
|
334
|
+
lifecycleStatus: 'active',
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
expect(await connectionEvents(connection.id)).toHaveLength(1);
|
|
255
338
|
});
|
|
256
339
|
|
|
257
340
|
it('returns undefined when updating the lifecycle status of an unknown connection', async () => {
|
|
@@ -310,5 +393,12 @@ describe('integration connection queries', () => {
|
|
|
310
393
|
|
|
311
394
|
const connections = await listIntegrationConnections({workspaceId});
|
|
312
395
|
expect(connections).toHaveLength(0);
|
|
396
|
+
const events = await db()
|
|
397
|
+
.select()
|
|
398
|
+
.from(integrationsOutbox)
|
|
399
|
+
.where(
|
|
400
|
+
sql`${integrationsOutbox.eventType} = ${INTEGRATION_CONNECTION_AVAILABLE} AND ${integrationsOutbox.payload}->>'workspaceId' = ${workspaceId}`,
|
|
401
|
+
);
|
|
402
|
+
expect(events).toHaveLength(0);
|
|
313
403
|
});
|
|
314
404
|
});
|
package/src/db/connections.ts
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
CONNECTION_SLUG_MAX_LENGTH,
|
|
3
|
+
INTEGRATION_CONNECTION_AVAILABLE,
|
|
4
|
+
type IntegrationsEventMap,
|
|
5
|
+
} from '@shipfox/api-integration-core-dto';
|
|
2
6
|
import {ConnectionSlugConflictError} from '@shipfox/api-integration-spi';
|
|
7
|
+
import {writeOutboxEvent} from '@shipfox/node-outbox';
|
|
3
8
|
import {and, eq} from 'drizzle-orm';
|
|
4
9
|
import type {
|
|
5
10
|
IntegrationConnection,
|
|
@@ -9,6 +14,7 @@ import type {IntegrationProviderKind} from '#core/entities/provider.js';
|
|
|
9
14
|
import {IntegrationConnectionAlreadyExistsError} from '#core/errors.js';
|
|
10
15
|
import {db} from './db.js';
|
|
11
16
|
import {integrationConnections, toIntegrationConnection} from './schema/connections.js';
|
|
17
|
+
import {integrationsOutbox} from './schema/outbox.js';
|
|
12
18
|
|
|
13
19
|
type IntegrationDb = ReturnType<typeof db>;
|
|
14
20
|
type IntegrationTx = Parameters<Parameters<IntegrationDb['transaction']>[0]>[0];
|
|
@@ -26,9 +32,13 @@ export async function upsertIntegrationConnection(
|
|
|
26
32
|
params: UpsertIntegrationConnectionParams,
|
|
27
33
|
options: {tx?: IntegrationDb | IntegrationTx | undefined} = {},
|
|
28
34
|
): Promise<IntegrationConnection> {
|
|
29
|
-
|
|
35
|
+
if (options.tx === undefined) {
|
|
36
|
+
return await db().transaction((tx) => upsertIntegrationConnection(params, {tx}));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const executor = options.tx;
|
|
30
40
|
const now = new Date();
|
|
31
|
-
|
|
41
|
+
let [row] = await executor
|
|
32
42
|
.insert(integrationConnections)
|
|
33
43
|
.values({
|
|
34
44
|
workspaceId: params.workspaceId,
|
|
@@ -38,22 +48,52 @@ export async function upsertIntegrationConnection(
|
|
|
38
48
|
displayName: params.displayName,
|
|
39
49
|
lifecycleStatus: params.lifecycleStatus ?? 'active',
|
|
40
50
|
})
|
|
41
|
-
.
|
|
51
|
+
.onConflictDoNothing({
|
|
42
52
|
target: [
|
|
43
53
|
integrationConnections.workspaceId,
|
|
44
54
|
integrationConnections.provider,
|
|
45
55
|
integrationConnections.externalAccountId,
|
|
46
56
|
],
|
|
47
|
-
|
|
57
|
+
})
|
|
58
|
+
.returning();
|
|
59
|
+
|
|
60
|
+
let becameAvailable = row?.lifecycleStatus === 'active';
|
|
61
|
+
if (!row) {
|
|
62
|
+
const [existing] = await executor
|
|
63
|
+
.select({
|
|
64
|
+
id: integrationConnections.id,
|
|
65
|
+
lifecycleStatus: integrationConnections.lifecycleStatus,
|
|
66
|
+
})
|
|
67
|
+
.from(integrationConnections)
|
|
68
|
+
.where(
|
|
69
|
+
and(
|
|
70
|
+
eq(integrationConnections.workspaceId, params.workspaceId),
|
|
71
|
+
eq(integrationConnections.provider, params.provider),
|
|
72
|
+
eq(integrationConnections.externalAccountId, params.externalAccountId),
|
|
73
|
+
),
|
|
74
|
+
)
|
|
75
|
+
.limit(1)
|
|
76
|
+
.for('update');
|
|
77
|
+
if (!existing) throw new Error('Integration connection upsert conflict row was not found');
|
|
78
|
+
|
|
79
|
+
[row] = await executor
|
|
80
|
+
.update(integrationConnections)
|
|
81
|
+
.set({
|
|
48
82
|
displayName: params.displayName,
|
|
49
83
|
lifecycleStatus: params.lifecycleStatus ?? 'active',
|
|
50
84
|
updatedAt: now,
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
|
|
85
|
+
})
|
|
86
|
+
.where(eq(integrationConnections.id, existing.id))
|
|
87
|
+
.returning();
|
|
88
|
+
becameAvailable = existing.lifecycleStatus !== 'active' && row?.lifecycleStatus === 'active';
|
|
89
|
+
}
|
|
54
90
|
|
|
55
91
|
if (!row) throw new Error('Integration connection upsert returned no rows');
|
|
56
|
-
|
|
92
|
+
const connection = toIntegrationConnection(row);
|
|
93
|
+
if (becameAvailable) {
|
|
94
|
+
await writeConnectionAvailableEvent(executor, connection);
|
|
95
|
+
}
|
|
96
|
+
return connection;
|
|
57
97
|
}
|
|
58
98
|
|
|
59
99
|
export interface CreateIntegrationConnectionParams {
|
|
@@ -100,7 +140,11 @@ export async function createIntegrationConnection(
|
|
|
100
140
|
params: CreateIntegrationConnectionParams,
|
|
101
141
|
options: {tx?: IntegrationDb | IntegrationTx | undefined} = {},
|
|
102
142
|
): Promise<IntegrationConnection> {
|
|
103
|
-
|
|
143
|
+
if (options.tx === undefined) {
|
|
144
|
+
return await db().transaction((tx) => createIntegrationConnection(params, {tx}));
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const executor = options.tx;
|
|
104
148
|
let rows: (typeof integrationConnections.$inferSelect)[];
|
|
105
149
|
try {
|
|
106
150
|
rows = await executor
|
|
@@ -130,7 +174,11 @@ export async function createIntegrationConnection(
|
|
|
130
174
|
|
|
131
175
|
const row = rows[0];
|
|
132
176
|
if (!row) throw new Error('Integration connection insert returned no rows');
|
|
133
|
-
|
|
177
|
+
const connection = toIntegrationConnection(row);
|
|
178
|
+
if (connection.lifecycleStatus === 'active') {
|
|
179
|
+
await writeConnectionAvailableEvent(executor, connection);
|
|
180
|
+
}
|
|
181
|
+
return connection;
|
|
134
182
|
}
|
|
135
183
|
|
|
136
184
|
export interface ResolveUniqueConnectionSlugParams {
|
|
@@ -225,19 +273,50 @@ export async function updateIntegrationConnectionLifecycleStatus(
|
|
|
225
273
|
params: UpdateIntegrationConnectionLifecycleStatusParams,
|
|
226
274
|
options: {tx?: IntegrationDb | IntegrationTx | undefined} = {},
|
|
227
275
|
): Promise<IntegrationConnection | undefined> {
|
|
228
|
-
|
|
276
|
+
if (options.tx === undefined) {
|
|
277
|
+
return await db().transaction((tx) => updateIntegrationConnectionLifecycleStatus(params, {tx}));
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
const executor = options.tx;
|
|
281
|
+
const [existing] = await executor
|
|
282
|
+
.select({lifecycleStatus: integrationConnections.lifecycleStatus})
|
|
283
|
+
.from(integrationConnections)
|
|
284
|
+
.where(eq(integrationConnections.id, params.id))
|
|
285
|
+
.limit(1)
|
|
286
|
+
.for('update');
|
|
287
|
+
if (!existing) return undefined;
|
|
288
|
+
|
|
229
289
|
const [row] = await executor
|
|
230
290
|
.update(integrationConnections)
|
|
231
291
|
.set({lifecycleStatus: params.lifecycleStatus, updatedAt: new Date()})
|
|
232
292
|
.where(eq(integrationConnections.id, params.id))
|
|
233
293
|
.returning();
|
|
234
294
|
if (!row) return undefined;
|
|
235
|
-
|
|
295
|
+
const connection = toIntegrationConnection(row);
|
|
296
|
+
if (existing.lifecycleStatus !== 'active' && connection.lifecycleStatus === 'active') {
|
|
297
|
+
await writeConnectionAvailableEvent(executor, connection);
|
|
298
|
+
}
|
|
299
|
+
return connection;
|
|
236
300
|
}
|
|
237
301
|
|
|
238
302
|
export type UpdateIntegrationConnectionLifecycleStatusFn =
|
|
239
303
|
typeof updateIntegrationConnectionLifecycleStatus;
|
|
240
304
|
|
|
305
|
+
async function writeConnectionAvailableEvent(
|
|
306
|
+
executor: IntegrationDb | IntegrationTx,
|
|
307
|
+
connection: IntegrationConnection,
|
|
308
|
+
): Promise<void> {
|
|
309
|
+
await writeOutboxEvent<IntegrationsEventMap>(executor, integrationsOutbox, {
|
|
310
|
+
type: INTEGRATION_CONNECTION_AVAILABLE,
|
|
311
|
+
payload: {
|
|
312
|
+
provider: connection.provider,
|
|
313
|
+
workspaceId: connection.workspaceId,
|
|
314
|
+
connectionId: connection.id,
|
|
315
|
+
slug: connection.slug,
|
|
316
|
+
},
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
|
|
241
320
|
export async function deleteIntegrationConnection(
|
|
242
321
|
params: {id: string},
|
|
243
322
|
options: {tx?: IntegrationDb | IntegrationTx | undefined} = {},
|
package/src/db/db.ts
CHANGED
|
@@ -2,10 +2,12 @@ import {drizzle, type NodePgDatabase} from '@shipfox/node-drizzle';
|
|
|
2
2
|
import {pgClient} from '@shipfox/node-postgres';
|
|
3
3
|
import {integrationConnections} from './schema/connections.js';
|
|
4
4
|
import {integrationsOutbox} from './schema/outbox.js';
|
|
5
|
+
import {integrationSecretCleanups} from './schema/secret-cleanups.js';
|
|
5
6
|
import {integrationsWebhookDeliveries} from './schema/webhook-deliveries.js';
|
|
6
7
|
|
|
7
8
|
export const schema = {
|
|
8
9
|
integrationConnections,
|
|
10
|
+
integrationSecretCleanups,
|
|
9
11
|
integrationsOutbox,
|
|
10
12
|
integrationsWebhookDeliveries,
|
|
11
13
|
};
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import {uuidv7PrimaryKey} from '@shipfox/node-drizzle';
|
|
2
|
+
import {index, integer, text, timestamp, uniqueIndex, uuid} from 'drizzle-orm/pg-core';
|
|
3
|
+
import type {IntegrationConnectionLifecycleStatus} from '#core/entities/connection.js';
|
|
4
|
+
import {pgTable} from './common.js';
|
|
5
|
+
|
|
6
|
+
export const integrationSecretCleanups = pgTable(
|
|
7
|
+
'secret_cleanups',
|
|
8
|
+
{
|
|
9
|
+
id: uuidv7PrimaryKey(),
|
|
10
|
+
workspaceId: uuid('workspace_id').notNull(),
|
|
11
|
+
provider: text('provider').notNull(),
|
|
12
|
+
connectionId: uuid('connection_id').notNull(),
|
|
13
|
+
externalAccountId: text('external_account_id').notNull(),
|
|
14
|
+
slug: text('slug').notNull(),
|
|
15
|
+
displayName: text('display_name').notNull(),
|
|
16
|
+
lifecycleStatus: text('lifecycle_status')
|
|
17
|
+
.$type<IntegrationConnectionLifecycleStatus>()
|
|
18
|
+
.notNull(),
|
|
19
|
+
connectionCreatedAt: timestamp('connection_created_at', {withTimezone: true}).notNull(),
|
|
20
|
+
connectionUpdatedAt: timestamp('connection_updated_at', {withTimezone: true}).notNull(),
|
|
21
|
+
attemptCount: integer('attempt_count').notNull().default(0),
|
|
22
|
+
nextAttemptAt: timestamp('next_attempt_at', {withTimezone: true}).notNull().defaultNow(),
|
|
23
|
+
leaseToken: uuid('lease_token'),
|
|
24
|
+
leaseExpiresAt: timestamp('lease_expires_at', {withTimezone: true}),
|
|
25
|
+
createdAt: timestamp('created_at', {withTimezone: true}).notNull().defaultNow(),
|
|
26
|
+
updatedAt: timestamp('updated_at', {withTimezone: true}).notNull().defaultNow(),
|
|
27
|
+
},
|
|
28
|
+
(table) => [
|
|
29
|
+
uniqueIndex('integrations_secret_cleanups_provider_connection_unique').on(
|
|
30
|
+
table.provider,
|
|
31
|
+
table.connectionId,
|
|
32
|
+
),
|
|
33
|
+
index('integrations_secret_cleanups_connection_id_idx').on(table.connectionId),
|
|
34
|
+
index('integrations_secret_cleanups_pending_idx').on(
|
|
35
|
+
table.nextAttemptAt,
|
|
36
|
+
table.createdAt,
|
|
37
|
+
table.id,
|
|
38
|
+
),
|
|
39
|
+
],
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
export type IntegrationSecretCleanupDb = typeof integrationSecretCleanups.$inferSelect;
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import {afterEach} from '@shipfox/vitest/vi';
|
|
2
|
+
import {sql} from 'drizzle-orm';
|
|
3
|
+
import {CLEANUP_SECRETS_ACTIVITY_TIMEOUT_MS} from '#temporal/constants.js';
|
|
4
|
+
import {upsertIntegrationConnection} from './connections.js';
|
|
5
|
+
import {db} from './db.js';
|
|
6
|
+
import {
|
|
7
|
+
claimIntegrationSecretCleanups,
|
|
8
|
+
completeIntegrationSecretCleanup,
|
|
9
|
+
enqueueIntegrationSecretCleanup,
|
|
10
|
+
retryIntegrationSecretCleanup,
|
|
11
|
+
} from './secret-cleanups.js';
|
|
12
|
+
|
|
13
|
+
const now = new Date(Date.now() + 60 * 1_000);
|
|
14
|
+
|
|
15
|
+
afterEach(async () => {
|
|
16
|
+
await db().execute(sql`TRUNCATE integrations_secret_cleanups CASCADE`);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
describe('integration secret cleanup persistence', () => {
|
|
20
|
+
it('keeps the default lease longer than the cleanup activity timeout', async () => {
|
|
21
|
+
await createCleanupConnection();
|
|
22
|
+
|
|
23
|
+
const [claimed] = await claimIntegrationSecretCleanups({limit: 1, now});
|
|
24
|
+
|
|
25
|
+
if (!claimed?.leaseExpiresAt) throw new Error('Expected the cleanup to have a lease');
|
|
26
|
+
expect(claimed.leaseExpiresAt.getTime() - now.getTime()).toBeGreaterThan(
|
|
27
|
+
CLEANUP_SECRETS_ACTIVITY_TIMEOUT_MS,
|
|
28
|
+
);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
it('claims a due row once and reclaims it after its lease expires', async () => {
|
|
32
|
+
const connection = await createCleanupConnection();
|
|
33
|
+
const [queued] = await claimIntegrationSecretCleanups({limit: 1, now, leaseDurationMs: 1_000});
|
|
34
|
+
|
|
35
|
+
expect(queued).toMatchObject({
|
|
36
|
+
connectionId: connection.id,
|
|
37
|
+
attemptCount: 1,
|
|
38
|
+
leaseToken: expect.any(String),
|
|
39
|
+
});
|
|
40
|
+
await expect(claimIntegrationSecretCleanups({limit: 1, now})).resolves.toEqual([]);
|
|
41
|
+
|
|
42
|
+
const wrongLeaseToken = crypto.randomUUID();
|
|
43
|
+
await expect(
|
|
44
|
+
completeIntegrationSecretCleanup({
|
|
45
|
+
id: queued?.id ?? '',
|
|
46
|
+
leaseToken: wrongLeaseToken,
|
|
47
|
+
now,
|
|
48
|
+
}),
|
|
49
|
+
).resolves.toBe(false);
|
|
50
|
+
await expect(
|
|
51
|
+
retryIntegrationSecretCleanup({
|
|
52
|
+
id: queued?.id ?? '',
|
|
53
|
+
leaseToken: wrongLeaseToken,
|
|
54
|
+
delayMs: 1_000,
|
|
55
|
+
now,
|
|
56
|
+
}),
|
|
57
|
+
).resolves.toBe(false);
|
|
58
|
+
|
|
59
|
+
const [reclaimed] = await claimIntegrationSecretCleanups({
|
|
60
|
+
limit: 1,
|
|
61
|
+
now: new Date(now.getTime() + 1_001),
|
|
62
|
+
leaseDurationMs: 1_000,
|
|
63
|
+
});
|
|
64
|
+
expect(reclaimed).toMatchObject({
|
|
65
|
+
id: queued?.id,
|
|
66
|
+
attemptCount: 2,
|
|
67
|
+
leaseToken: expect.any(String),
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
await expect(
|
|
71
|
+
completeIntegrationSecretCleanup({
|
|
72
|
+
id: reclaimed?.id ?? '',
|
|
73
|
+
leaseToken: reclaimed?.leaseToken ?? '',
|
|
74
|
+
now: new Date(now.getTime() + 1_001),
|
|
75
|
+
}),
|
|
76
|
+
).resolves.toBe(true);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('honors the batch limit and retry backoff', async () => {
|
|
80
|
+
await createCleanupConnection({externalAccountId: 'first'});
|
|
81
|
+
await createCleanupConnection({externalAccountId: 'second'});
|
|
82
|
+
|
|
83
|
+
const firstClaim = await claimIntegrationSecretCleanups({limit: 1, now});
|
|
84
|
+
expect(firstClaim).toHaveLength(1);
|
|
85
|
+
const first = firstClaim[0];
|
|
86
|
+
if (!first?.leaseToken) throw new Error('Expected the first cleanup to have a lease');
|
|
87
|
+
|
|
88
|
+
await expect(
|
|
89
|
+
retryIntegrationSecretCleanup({
|
|
90
|
+
id: first.id,
|
|
91
|
+
leaseToken: first.leaseToken,
|
|
92
|
+
delayMs: 5_000,
|
|
93
|
+
now,
|
|
94
|
+
}),
|
|
95
|
+
).resolves.toBe(true);
|
|
96
|
+
|
|
97
|
+
const secondClaim = await claimIntegrationSecretCleanups({limit: 2, now});
|
|
98
|
+
expect(secondClaim).toHaveLength(1);
|
|
99
|
+
expect(secondClaim[0]?.id).not.toBe(first.id);
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
async function createCleanupConnection(overrides: {externalAccountId?: string} = {}) {
|
|
104
|
+
const connection = await upsertIntegrationConnection({
|
|
105
|
+
workspaceId: crypto.randomUUID(),
|
|
106
|
+
provider: 'slack',
|
|
107
|
+
externalAccountId: overrides.externalAccountId ?? crypto.randomUUID(),
|
|
108
|
+
slug: `slack_${crypto.randomUUID()}`,
|
|
109
|
+
displayName: 'Slack',
|
|
110
|
+
});
|
|
111
|
+
await enqueueIntegrationSecretCleanup({connection});
|
|
112
|
+
return connection;
|
|
113
|
+
}
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import {and, asc, eq, gt, inArray, isNull, lte, or, sql} from 'drizzle-orm';
|
|
2
|
+
import type {IntegrationConnection} from '#core/entities/connection.js';
|
|
3
|
+
import {db} from './db.js';
|
|
4
|
+
import {
|
|
5
|
+
type IntegrationSecretCleanupDb,
|
|
6
|
+
integrationSecretCleanups,
|
|
7
|
+
} from './schema/secret-cleanups.js';
|
|
8
|
+
|
|
9
|
+
// Keep the lease longer than cleanupIntegrationSecretsCron's 5-minute activity timeout so
|
|
10
|
+
// a timed-out sweep cannot have its claimed rows reclaimed while it is still running.
|
|
11
|
+
const DEFAULT_LEASE_DURATION_MS = 10 * 60 * 1_000;
|
|
12
|
+
|
|
13
|
+
export interface EnqueueIntegrationSecretCleanupParams {
|
|
14
|
+
connection: IntegrationConnection;
|
|
15
|
+
now?: Date | undefined;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface IntegrationSecretCleanup {
|
|
19
|
+
id: string;
|
|
20
|
+
workspaceId: string;
|
|
21
|
+
provider: string;
|
|
22
|
+
connectionId: string;
|
|
23
|
+
externalAccountId: string;
|
|
24
|
+
slug: string;
|
|
25
|
+
displayName: string;
|
|
26
|
+
lifecycleStatus: IntegrationConnection['lifecycleStatus'];
|
|
27
|
+
connectionCreatedAt: Date;
|
|
28
|
+
connectionUpdatedAt: Date;
|
|
29
|
+
attemptCount: number;
|
|
30
|
+
nextAttemptAt: Date;
|
|
31
|
+
leaseToken: string | null;
|
|
32
|
+
leaseExpiresAt: Date | null;
|
|
33
|
+
createdAt: Date;
|
|
34
|
+
updatedAt: Date;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
type IntegrationDb = ReturnType<typeof db>;
|
|
38
|
+
type IntegrationTx = Parameters<Parameters<IntegrationDb['transaction']>[0]>[0];
|
|
39
|
+
|
|
40
|
+
export async function enqueueIntegrationSecretCleanup(
|
|
41
|
+
params: EnqueueIntegrationSecretCleanupParams,
|
|
42
|
+
options: {tx?: IntegrationDb | IntegrationTx | undefined} = {},
|
|
43
|
+
): Promise<void> {
|
|
44
|
+
const executor = options.tx ?? db();
|
|
45
|
+
await executor
|
|
46
|
+
.insert(integrationSecretCleanups)
|
|
47
|
+
.values({
|
|
48
|
+
workspaceId: params.connection.workspaceId,
|
|
49
|
+
provider: params.connection.provider,
|
|
50
|
+
connectionId: params.connection.id,
|
|
51
|
+
externalAccountId: params.connection.externalAccountId,
|
|
52
|
+
slug: params.connection.slug,
|
|
53
|
+
displayName: params.connection.displayName,
|
|
54
|
+
lifecycleStatus: params.connection.lifecycleStatus,
|
|
55
|
+
connectionCreatedAt: params.connection.createdAt,
|
|
56
|
+
connectionUpdatedAt: params.connection.updatedAt,
|
|
57
|
+
// Claims compare against the application clock, so the first due time must come
|
|
58
|
+
// from that clock too. A database clock even milliseconds ahead would otherwise
|
|
59
|
+
// hide the row from the sweep that runs right after this insert commits.
|
|
60
|
+
nextAttemptAt: params.now ?? new Date(),
|
|
61
|
+
})
|
|
62
|
+
.onConflictDoNothing({
|
|
63
|
+
target: [integrationSecretCleanups.provider, integrationSecretCleanups.connectionId],
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export interface ClaimIntegrationSecretCleanupsParams {
|
|
68
|
+
limit: number;
|
|
69
|
+
connectionId?: string | undefined;
|
|
70
|
+
now?: Date | undefined;
|
|
71
|
+
leaseDurationMs?: number | undefined;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export async function claimIntegrationSecretCleanups(
|
|
75
|
+
params: ClaimIntegrationSecretCleanupsParams,
|
|
76
|
+
): Promise<IntegrationSecretCleanup[]> {
|
|
77
|
+
if (!Number.isInteger(params.limit) || params.limit <= 0) {
|
|
78
|
+
throw new Error('Secret cleanup claim limit must be a positive integer');
|
|
79
|
+
}
|
|
80
|
+
const now = params.now ?? new Date();
|
|
81
|
+
const leaseDurationMs = params.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
|
82
|
+
if (!Number.isInteger(leaseDurationMs) || leaseDurationMs <= 0) {
|
|
83
|
+
throw new Error('Secret cleanup lease duration must be a positive integer');
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return await db().transaction(async (tx) => {
|
|
87
|
+
const conditions = [
|
|
88
|
+
lte(integrationSecretCleanups.nextAttemptAt, now),
|
|
89
|
+
or(
|
|
90
|
+
isNull(integrationSecretCleanups.leaseExpiresAt),
|
|
91
|
+
lte(integrationSecretCleanups.leaseExpiresAt, now),
|
|
92
|
+
),
|
|
93
|
+
...(params.connectionId
|
|
94
|
+
? [eq(integrationSecretCleanups.connectionId, params.connectionId)]
|
|
95
|
+
: []),
|
|
96
|
+
];
|
|
97
|
+
const candidates = await tx
|
|
98
|
+
.select({id: integrationSecretCleanups.id})
|
|
99
|
+
.from(integrationSecretCleanups)
|
|
100
|
+
.where(and(...conditions))
|
|
101
|
+
.orderBy(
|
|
102
|
+
asc(integrationSecretCleanups.nextAttemptAt),
|
|
103
|
+
asc(integrationSecretCleanups.createdAt),
|
|
104
|
+
asc(integrationSecretCleanups.id),
|
|
105
|
+
)
|
|
106
|
+
.limit(params.limit)
|
|
107
|
+
.for('update', {skipLocked: true});
|
|
108
|
+
if (candidates.length === 0) return [];
|
|
109
|
+
|
|
110
|
+
const rows = await tx
|
|
111
|
+
.update(integrationSecretCleanups)
|
|
112
|
+
.set({
|
|
113
|
+
attemptCount: sql`${integrationSecretCleanups.attemptCount} + 1`,
|
|
114
|
+
leaseToken: sql`gen_random_uuid()`,
|
|
115
|
+
leaseExpiresAt: new Date(now.getTime() + leaseDurationMs),
|
|
116
|
+
updatedAt: now,
|
|
117
|
+
})
|
|
118
|
+
.where(
|
|
119
|
+
inArray(
|
|
120
|
+
integrationSecretCleanups.id,
|
|
121
|
+
candidates.map((candidate) => candidate.id),
|
|
122
|
+
),
|
|
123
|
+
)
|
|
124
|
+
.returning();
|
|
125
|
+
return rows.map(toIntegrationSecretCleanup);
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export async function completeIntegrationSecretCleanup(params: {
|
|
130
|
+
id: string;
|
|
131
|
+
leaseToken: string;
|
|
132
|
+
now?: Date | undefined;
|
|
133
|
+
}): Promise<boolean> {
|
|
134
|
+
const now = params.now ?? new Date();
|
|
135
|
+
const result = await db()
|
|
136
|
+
.delete(integrationSecretCleanups)
|
|
137
|
+
.where(
|
|
138
|
+
and(
|
|
139
|
+
eq(integrationSecretCleanups.id, params.id),
|
|
140
|
+
eq(integrationSecretCleanups.leaseToken, params.leaseToken),
|
|
141
|
+
gt(integrationSecretCleanups.leaseExpiresAt, now),
|
|
142
|
+
),
|
|
143
|
+
);
|
|
144
|
+
return (result.rowCount ?? 0) > 0;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export async function retryIntegrationSecretCleanup(params: {
|
|
148
|
+
id: string;
|
|
149
|
+
leaseToken: string;
|
|
150
|
+
delayMs: number;
|
|
151
|
+
now?: Date | undefined;
|
|
152
|
+
}): Promise<boolean> {
|
|
153
|
+
if (!Number.isInteger(params.delayMs) || params.delayMs < 0) {
|
|
154
|
+
throw new Error('Secret cleanup retry delay must be a non-negative integer');
|
|
155
|
+
}
|
|
156
|
+
const now = params.now ?? new Date();
|
|
157
|
+
const result = await db()
|
|
158
|
+
.update(integrationSecretCleanups)
|
|
159
|
+
.set({
|
|
160
|
+
nextAttemptAt: new Date(now.getTime() + params.delayMs),
|
|
161
|
+
leaseToken: null,
|
|
162
|
+
leaseExpiresAt: null,
|
|
163
|
+
updatedAt: now,
|
|
164
|
+
})
|
|
165
|
+
.where(
|
|
166
|
+
and(
|
|
167
|
+
eq(integrationSecretCleanups.id, params.id),
|
|
168
|
+
eq(integrationSecretCleanups.leaseToken, params.leaseToken),
|
|
169
|
+
gt(integrationSecretCleanups.leaseExpiresAt, now),
|
|
170
|
+
),
|
|
171
|
+
);
|
|
172
|
+
return (result.rowCount ?? 0) > 0;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
export async function listIntegrationSecretCleanups(
|
|
176
|
+
params: {connectionId?: string | undefined} = {},
|
|
177
|
+
): Promise<IntegrationSecretCleanup[]> {
|
|
178
|
+
const rows = await db()
|
|
179
|
+
.select()
|
|
180
|
+
.from(integrationSecretCleanups)
|
|
181
|
+
.where(
|
|
182
|
+
params.connectionId
|
|
183
|
+
? eq(integrationSecretCleanups.connectionId, params.connectionId)
|
|
184
|
+
: undefined,
|
|
185
|
+
)
|
|
186
|
+
.orderBy(asc(integrationSecretCleanups.createdAt), asc(integrationSecretCleanups.id));
|
|
187
|
+
return rows.map(toIntegrationSecretCleanup);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function toIntegrationSecretCleanup(row: IntegrationSecretCleanupDb): IntegrationSecretCleanup {
|
|
191
|
+
return {
|
|
192
|
+
id: row.id,
|
|
193
|
+
workspaceId: row.workspaceId,
|
|
194
|
+
provider: row.provider,
|
|
195
|
+
connectionId: row.connectionId,
|
|
196
|
+
externalAccountId: row.externalAccountId,
|
|
197
|
+
slug: row.slug,
|
|
198
|
+
displayName: row.displayName,
|
|
199
|
+
lifecycleStatus: row.lifecycleStatus,
|
|
200
|
+
connectionCreatedAt: row.connectionCreatedAt,
|
|
201
|
+
connectionUpdatedAt: row.connectionUpdatedAt,
|
|
202
|
+
attemptCount: row.attemptCount,
|
|
203
|
+
nextAttemptAt: row.nextAttemptAt,
|
|
204
|
+
leaseToken: row.leaseToken,
|
|
205
|
+
leaseExpiresAt: row.leaseExpiresAt,
|
|
206
|
+
createdAt: row.createdAt,
|
|
207
|
+
updatedAt: row.updatedAt,
|
|
208
|
+
};
|
|
209
|
+
}
|