@remit/account-worker 0.0.28 → 0.0.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -7
- package/src/config.ts +23 -25
- package/src/events.ts +8 -71
- package/src/handlers/account-fanout.test.ts +13 -84
- package/src/handlers/account-fanout.ts +18 -73
- package/src/handlers/account-purge.test.ts +22 -39
- package/src/handlers/account-purge.ts +8 -28
- package/src/handlers/organize-job.test.ts +2 -0
- package/src/config.test.ts +0 -54
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remit/account-worker",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.30",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -41,12 +41,12 @@
|
|
|
41
41
|
"@remit/sqs-client": "*",
|
|
42
42
|
"@remit/storage-service": "*",
|
|
43
43
|
"@remit/electrodb-entities": "*",
|
|
44
|
-
"@aws-sdk/client-sqs": "
|
|
45
|
-
"archiver": "
|
|
46
|
-
"electrodb": "
|
|
47
|
-
"expect-env": "
|
|
48
|
-
"@types/archiver": "
|
|
49
|
-
"@types/aws-lambda": "
|
|
44
|
+
"@aws-sdk/client-sqs": "^3.1090.0",
|
|
45
|
+
"archiver": "^8.0.0",
|
|
46
|
+
"electrodb": "^3.9.1",
|
|
47
|
+
"expect-env": "^1.0.1",
|
|
48
|
+
"@types/archiver": "^8.0.0",
|
|
49
|
+
"@types/aws-lambda": "^8.10.162"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|
|
52
52
|
"@aws-sdk/core": "*",
|
package/src/config.ts
CHANGED
|
@@ -1,44 +1,42 @@
|
|
|
1
|
-
import type { SQSClient } from "@aws-sdk/client-sqs";
|
|
2
1
|
import { getClient } from "@remit/backend/client";
|
|
3
|
-
import {
|
|
2
|
+
import type { GroupedQueueProducer, QueueProducer } from "@remit/data-ports";
|
|
3
|
+
import {
|
|
4
|
+
createGroupedQueueProducer,
|
|
5
|
+
createStandardQueueProducer,
|
|
6
|
+
} from "@remit/sqs-client/producer";
|
|
4
7
|
import type { StorageService } from "@remit/storage-service";
|
|
5
8
|
import { createStorageService } from "@remit/storage-service/s3";
|
|
6
9
|
import { env } from "expect-env";
|
|
7
10
|
import type { CascadeServices } from "./cascade.js";
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
// real SQS keeps the default. Building at import is not an option: the queue
|
|
13
|
-
// URLs are read lazily for the reason given below.
|
|
14
|
-
const sqsClients = new Map<string, SQSClient>();
|
|
15
|
-
|
|
16
|
-
export const getSqsClient = (queueUrl: string): SQSClient => {
|
|
17
|
-
const existing = sqsClients.get(queueUrl);
|
|
18
|
-
if (existing) return existing;
|
|
19
|
-
const client = createQueueProducer({ queueUrl });
|
|
20
|
-
sqsClients.set(queueUrl, client);
|
|
21
|
-
return client;
|
|
22
|
-
};
|
|
11
|
+
import type {
|
|
12
|
+
AccountDataPurgeFinalizeEvent,
|
|
13
|
+
AccountFinalizeEvent,
|
|
14
|
+
} from "./events.js";
|
|
23
15
|
|
|
24
16
|
// SQS env vars are lazy-evaluated. The fanout worker needs them at handler time;
|
|
25
17
|
// the finalize worker imports `getCascadeServices` from this module but talks to
|
|
26
18
|
// no queue, so its Lambda doesn't carry the env vars. Eager evaluation here would
|
|
27
19
|
// crash finalize at module load — getters defer the read to the fanout-only call
|
|
28
20
|
// sites.
|
|
29
|
-
// Optional, and read through `process.env` rather than expect-env so an unset
|
|
30
|
-
// var yields undefined instead of throwing. IMAP_WORKER_STOP is a no-op
|
|
31
|
-
// acknowledgement in the cascade contract — the account tombstone fence is what
|
|
32
|
-
// actually halts the worker — so a deployment that never provisions a dedicated
|
|
33
|
-
// imap-worker stop queue (the self-host compose stacks) skips the signal rather
|
|
34
|
-
// than failing the whole fanout. AWS sets the var and keeps sending it.
|
|
35
|
-
export const getImapWorkerQueueUrl = (): string | undefined =>
|
|
36
|
-
process.env.SQS_QUEUE_URL_IMAP_WORKER;
|
|
37
21
|
export const getAccountFinalizeQueueUrl = (): string =>
|
|
38
22
|
env.SQS_QUEUE_URL_ACCOUNT_FINALIZE;
|
|
39
23
|
export const getAccountPurgeDeleteQueueUrl = (): string =>
|
|
40
24
|
env.SQS_QUEUE_URL_ACCOUNT_PURGE_DELETE;
|
|
41
25
|
|
|
26
|
+
export interface FanoutQueues {
|
|
27
|
+
accountFinalize: QueueProducer<AccountFinalizeEvent>;
|
|
28
|
+
accountPurgeDelete: GroupedQueueProducer<AccountDataPurgeFinalizeEvent>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export const sqsFanoutQueues = (): FanoutQueues => ({
|
|
32
|
+
accountFinalize: createStandardQueueProducer({
|
|
33
|
+
queueUrl: getAccountFinalizeQueueUrl(),
|
|
34
|
+
}),
|
|
35
|
+
accountPurgeDelete: createGroupedQueueProducer({
|
|
36
|
+
queueUrl: getAccountPurgeDeleteQueueUrl(),
|
|
37
|
+
}),
|
|
38
|
+
});
|
|
39
|
+
|
|
42
40
|
// The RemitClient is resolved lazily and cached, not at module load: the
|
|
43
41
|
// DynamoDB backend is injected by the composition root before the first cascade
|
|
44
42
|
// runs, so building at import time would race the registration.
|
package/src/events.ts
CHANGED
|
@@ -1,74 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
};
|
|
9
|
-
|
|
10
|
-
/**
|
|
11
|
-
* Per-account hard-delete. Purges ONE mail account's data (mailboxes,
|
|
12
|
-
* messages, body parts, thread messages, S3 objects) while
|
|
13
|
-
* keeping the AccountConfig/tenant and any sibling accounts intact. Emitted
|
|
14
|
-
* by the API on `deleteAccount`; consumed by the fanout worker.
|
|
15
|
-
*/
|
|
16
|
-
export type AccountDataPurgeEvent = {
|
|
17
|
-
type: "AccountDataPurge";
|
|
18
|
-
accountId: string;
|
|
19
|
-
accountConfigId: string;
|
|
20
|
-
};
|
|
21
|
-
|
|
22
|
-
export type AccountExportEvent = {
|
|
23
|
-
type: "AccountExport";
|
|
24
|
-
accountConfigId: string;
|
|
25
|
-
accountExportRequestId: string;
|
|
26
|
-
};
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* A "all like these" back-apply job (RFC 034, #1278). Rides the same
|
|
30
|
-
* account-fanout queue as export: the API writes a Pending OrganizeJobRequest
|
|
31
|
-
* row and enqueues this; the fanout worker matches the corpus and applies the
|
|
32
|
-
* action, driving the row to Complete/Failed. Never creates a Filter row.
|
|
33
|
-
*/
|
|
34
|
-
export type OrganizeJobEvent = {
|
|
35
|
-
type: "OrganizeJob";
|
|
36
|
-
accountConfigId: string;
|
|
37
|
-
organizeJobId: string;
|
|
38
|
-
};
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* Retroactive half of a sender-category override (#415, RFC 039 Decision 3).
|
|
42
|
-
* Rides the same account-fanout queue as the organize back-apply: the API
|
|
43
|
-
* enqueues one when `Address.flags.category` is SET, and the fanout worker
|
|
44
|
-
* re-labels a bounded recent batch of that sender's already-classified mail.
|
|
45
|
-
* A CLEAR never enqueues one: reverting to auto-classification says nothing
|
|
46
|
-
* about what the classifier would have answered, and re-deriving it would need
|
|
47
|
-
* every message's body back.
|
|
48
|
-
*
|
|
49
|
-
* `category` and `categorySetAt` together name WHICH set this job was fired
|
|
50
|
-
* for, and the worker refuses a job whose set no longer stands on the Address.
|
|
51
|
-
* That check is what the standard queue's semantics make necessary rather than
|
|
52
|
-
* merely tidy: delivery is at-least-once and unordered, so without it a job
|
|
53
|
-
* that fails and redelivers after the user has corrected the category re-labels
|
|
54
|
-
* those 500 messages back to the superseded one, leaving the flag and the mail
|
|
55
|
-
* permanently disagreeing with nothing to reconcile them.
|
|
56
|
-
*/
|
|
57
|
-
export type SenderCategoryBackApplyEvent = {
|
|
58
|
-
type: "SenderCategoryBackApply";
|
|
59
|
-
accountConfigId: string;
|
|
60
|
-
addressId: string;
|
|
61
|
-
normalizedEmail: string;
|
|
62
|
-
category: MessageCategoryValue;
|
|
63
|
-
categorySetAt: number;
|
|
64
|
-
};
|
|
65
|
-
|
|
66
|
-
export type AccountFanoutEvent =
|
|
67
|
-
| AccountDeleteEvent
|
|
68
|
-
| AccountDataPurgeEvent
|
|
69
|
-
| AccountExportEvent
|
|
70
|
-
| OrganizeJobEvent
|
|
71
|
-
| SenderCategoryBackApplyEvent;
|
|
1
|
+
export type {
|
|
2
|
+
AccountDataPurgeEvent,
|
|
3
|
+
AccountDeleteEvent,
|
|
4
|
+
AccountExportEvent,
|
|
5
|
+
AccountFanoutEvent,
|
|
6
|
+
OrganizeJobEvent,
|
|
7
|
+
SenderCategoryBackApplyEvent,
|
|
8
|
+
} from "@remit/data-ports";
|
|
72
9
|
|
|
73
10
|
export type AccountDeleteFinalizeEvent = {
|
|
74
11
|
type: "FinalizeAccountDelete";
|
|
@@ -1,108 +1,37 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import {
|
|
3
|
-
import
|
|
2
|
+
import { describe, it } from "node:test";
|
|
3
|
+
import { recordingQueue } from "@remit/data-ports/recording-queue";
|
|
4
4
|
import { noopLogger } from "@remit/logger";
|
|
5
5
|
import type { CascadeServices } from "../cascade.js";
|
|
6
6
|
import type { AccountDeleteEvent } from "../events.js";
|
|
7
7
|
import { processAccountFanout } from "./account-fanout.js";
|
|
8
8
|
|
|
9
|
-
// A CascadeServices fake with one account and no data: enough for
|
|
10
|
-
// enumerateCascadeEntities to yield a single Account row (so the stop-signal
|
|
11
|
-
// loop has one accountId to iterate) without standing up a database.
|
|
12
9
|
const buildServices = (): CascadeServices =>
|
|
13
10
|
({
|
|
14
|
-
accountConfigService: {
|
|
15
|
-
get: async () => ({ userId: "user-1" }),
|
|
16
|
-
describe: async () => ({
|
|
17
|
-
account: [{ accountId: "acc-1" }],
|
|
18
|
-
address: [],
|
|
19
|
-
}),
|
|
20
|
-
},
|
|
21
|
-
accountService: {
|
|
22
|
-
describe: async () => ({ mailbox: [] }),
|
|
23
|
-
},
|
|
24
|
-
messageService: {
|
|
25
|
-
listAllByMailbox: async () => [],
|
|
26
|
-
describe: async () => ({}),
|
|
27
|
-
},
|
|
28
|
-
outboxMessageService: { listByAccount: async () => ({ items: [] }) },
|
|
29
|
-
mailboxLockService: { listByAccount: async () => [] },
|
|
30
|
-
messagePlacementMoveService: { listByAccountId: async () => [] },
|
|
31
|
-
messageFlagPushService: { listByAccountId: async () => [] },
|
|
32
|
-
threadMessageService: { listAllByAccount: async () => [] },
|
|
33
|
-
accountSettingService: { listByAccountConfig: async () => [] },
|
|
34
|
-
filterService: { listByAccountConfig: async () => [] },
|
|
35
|
-
labelService: { listByAccountConfig: async () => [] },
|
|
36
|
-
messageLabelService: { listByLabelId: async () => [] },
|
|
11
|
+
accountConfigService: { get: async () => ({ userId: "user-1" }) },
|
|
37
12
|
}) as unknown as CascadeServices;
|
|
38
13
|
|
|
39
|
-
interface SentMessage {
|
|
40
|
-
queueUrl: string | undefined;
|
|
41
|
-
body: { type?: string; accountId?: string };
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
const recordingSqs = (sent: SentMessage[]): SQSClient =>
|
|
45
|
-
({
|
|
46
|
-
send: async (command: {
|
|
47
|
-
input: { QueueUrl?: string; MessageBody?: string };
|
|
48
|
-
}) => {
|
|
49
|
-
sent.push({
|
|
50
|
-
queueUrl: command.input.QueueUrl,
|
|
51
|
-
body: JSON.parse(command.input.MessageBody ?? "{}"),
|
|
52
|
-
});
|
|
53
|
-
return {};
|
|
54
|
-
},
|
|
55
|
-
}) as unknown as SQSClient;
|
|
56
|
-
|
|
57
14
|
const deleteEvent: AccountDeleteEvent = {
|
|
58
15
|
type: "AccountDelete",
|
|
59
16
|
accountConfigId: "cfg-1",
|
|
60
17
|
};
|
|
61
18
|
|
|
62
|
-
describe("processAccountFanout
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
if (prev === undefined) delete process.env.SQS_QUEUE_URL_IMAP_WORKER;
|
|
66
|
-
else process.env.SQS_QUEUE_URL_IMAP_WORKER = prev;
|
|
67
|
-
});
|
|
68
|
-
|
|
69
|
-
it("skips the stop signal when no imap-worker queue is configured, still enqueuing finalize", async () => {
|
|
70
|
-
delete process.env.SQS_QUEUE_URL_IMAP_WORKER;
|
|
71
|
-
const sent: SentMessage[] = [];
|
|
72
|
-
|
|
73
|
-
await processAccountFanout(deleteEvent, noopLogger, {
|
|
74
|
-
services: buildServices(),
|
|
75
|
-
sqs: recordingSqs(sent),
|
|
76
|
-
signOut: async () => {},
|
|
77
|
-
accountFinalizeQueueUrl: "http://queue/finalize",
|
|
78
|
-
});
|
|
79
|
-
|
|
80
|
-
const stops = sent.filter((m) => m.body.type === "IMAP_WORKER_STOP");
|
|
81
|
-
const finalize = sent.filter(
|
|
82
|
-
(m) => m.body.type === "FinalizeAccountDelete",
|
|
83
|
-
);
|
|
84
|
-
assert.equal(stops.length, 0, "no stop signal without a configured queue");
|
|
85
|
-
assert.equal(finalize.length, 1, "finalize is still enqueued");
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
it("enqueues one stop signal per account when a queue is configured", async () => {
|
|
89
|
-
const sent: SentMessage[] = [];
|
|
19
|
+
describe("processAccountFanout on account delete", () => {
|
|
20
|
+
it("enqueues the finalize step", async () => {
|
|
21
|
+
const accountFinalize = recordingQueue();
|
|
90
22
|
|
|
91
23
|
await processAccountFanout(deleteEvent, noopLogger, {
|
|
24
|
+
queues: {
|
|
25
|
+
accountFinalize,
|
|
26
|
+
accountPurgeDelete: recordingQueue().grouped,
|
|
27
|
+
},
|
|
92
28
|
services: buildServices(),
|
|
93
|
-
sqs: recordingSqs(sent),
|
|
94
29
|
signOut: async () => {},
|
|
95
|
-
imapWorkerQueueUrl: "http://queue/imap",
|
|
96
|
-
accountFinalizeQueueUrl: "http://queue/finalize",
|
|
97
30
|
});
|
|
98
31
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
assert.equal(stops[0]?.body.accountId, "acc-1");
|
|
103
|
-
assert.equal(
|
|
104
|
-
sent.filter((m) => m.body.type === "FinalizeAccountDelete").length,
|
|
105
|
-
1,
|
|
32
|
+
assert.deepEqual(
|
|
33
|
+
accountFinalize.sent.map((m) => m.body.type),
|
|
34
|
+
["FinalizeAccountDelete"],
|
|
106
35
|
);
|
|
107
36
|
});
|
|
108
37
|
});
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
|
|
2
1
|
import { createLogger, type Logger } from "@remit/logger";
|
|
3
2
|
import {
|
|
4
3
|
queueNameFromEventSource,
|
|
@@ -7,13 +6,10 @@ import {
|
|
|
7
6
|
import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
|
|
8
7
|
import type { SQSBatchResponse, SQSEvent, SQSHandler } from "aws-lambda";
|
|
9
8
|
import type { CascadeServices } from "../cascade.js";
|
|
10
|
-
import { enumerateCascadeEntities } from "../cascade.js";
|
|
11
9
|
import {
|
|
12
|
-
|
|
13
|
-
getAccountPurgeDeleteQueueUrl,
|
|
10
|
+
type FanoutQueues,
|
|
14
11
|
getCascadeServices,
|
|
15
|
-
|
|
16
|
-
getSqsClient,
|
|
12
|
+
sqsFanoutQueues,
|
|
17
13
|
} from "../config.js";
|
|
18
14
|
import {
|
|
19
15
|
type DeletionCapabilities,
|
|
@@ -32,36 +28,22 @@ import { processSenderCategoryBackApply } from "./sender-category-backapply.js";
|
|
|
32
28
|
installNodeLogger();
|
|
33
29
|
|
|
34
30
|
export interface ProcessAccountFanoutDeps {
|
|
31
|
+
queues: FanoutQueues;
|
|
35
32
|
services?: CascadeServices;
|
|
36
|
-
sqs?: SQSClient;
|
|
37
33
|
signOut?: DeletionCapabilities["signOut"];
|
|
38
|
-
imapWorkerQueueUrl?: string;
|
|
39
|
-
accountFinalizeQueueUrl?: string;
|
|
40
|
-
accountPurgeDeleteQueueUrl?: string;
|
|
41
34
|
}
|
|
42
35
|
|
|
43
|
-
/**
|
|
44
|
-
* Fanout step of the account-deletion cascade. Read-only: enumerates the
|
|
45
|
-
* AccountConfig's data and dispatches downstream work — per-account
|
|
46
|
-
* imap-worker stops, a sign-out, and the finalize
|
|
47
|
-
* enqueue. Throws on any non-swallowable error so SQS partial batch-failure
|
|
48
|
-
* retries the whole record.
|
|
49
|
-
*/
|
|
50
36
|
export const processAccountFanout = async (
|
|
51
37
|
event: AccountFanoutEvent,
|
|
52
38
|
log: Logger,
|
|
53
|
-
deps: ProcessAccountFanoutDeps
|
|
39
|
+
deps: ProcessAccountFanoutDeps,
|
|
54
40
|
): Promise<void> => {
|
|
55
41
|
const services = deps.services ?? (await getCascadeServices());
|
|
56
|
-
const sqsFor = (queueUrl: string): SQSClient =>
|
|
57
|
-
deps.sqs ?? getSqsClient(queueUrl);
|
|
58
42
|
|
|
59
43
|
if (event.type === "AccountDataPurge") {
|
|
60
44
|
await processAccountDataPurge(event, log, {
|
|
61
45
|
services,
|
|
62
|
-
|
|
63
|
-
accountPurgeDeleteQueueUrl:
|
|
64
|
-
deps.accountPurgeDeleteQueueUrl ?? getAccountPurgeDeleteQueueUrl(),
|
|
46
|
+
queue: deps.queues.accountPurgeDelete,
|
|
65
47
|
});
|
|
66
48
|
return;
|
|
67
49
|
}
|
|
@@ -81,81 +63,42 @@ export const processAccountFanout = async (
|
|
|
81
63
|
return;
|
|
82
64
|
}
|
|
83
65
|
|
|
84
|
-
await processAccountDelete(event, log, services,
|
|
66
|
+
await processAccountDelete(event, log, services, deps);
|
|
85
67
|
};
|
|
86
68
|
|
|
87
69
|
const processAccountDelete = async (
|
|
88
70
|
event: AccountDeleteEvent,
|
|
89
71
|
log: Logger,
|
|
90
72
|
services: CascadeServices,
|
|
91
|
-
sqsFor: (queueUrl: string) => SQSClient,
|
|
92
73
|
deps: ProcessAccountFanoutDeps,
|
|
93
74
|
): Promise<void> => {
|
|
94
75
|
const { accountConfigId } = event;
|
|
95
76
|
const signOut = deps.signOut ?? (await getDeletionCapabilities()).signOut;
|
|
96
|
-
const
|
|
97
|
-
const accountFinalizeQueueUrl =
|
|
98
|
-
deps.accountFinalizeQueueUrl ?? getAccountFinalizeQueueUrl();
|
|
77
|
+
const { accountFinalize } = deps.queues;
|
|
99
78
|
|
|
100
79
|
const accountConfig =
|
|
101
80
|
await services.accountConfigService.get(accountConfigId);
|
|
102
81
|
const userId = accountConfig.userId;
|
|
103
82
|
|
|
104
|
-
const { entities, messageIds: _messageIds } = await enumerateCascadeEntities(
|
|
105
|
-
accountConfigId,
|
|
106
|
-
services,
|
|
107
|
-
log,
|
|
108
|
-
);
|
|
109
|
-
|
|
110
|
-
const accountIds = entities
|
|
111
|
-
.filter((e) => e.entityType === "Account")
|
|
112
|
-
.map((e) => e.key.accountId);
|
|
113
|
-
|
|
114
|
-
// IMAP_WORKER_STOP is a no-op acknowledgement — the account tombstone fence
|
|
115
|
-
// (isActive=false + deletedAt) is what actually halts the worker. Deployments
|
|
116
|
-
// that provision a dedicated imap-worker stop queue get the signal; the
|
|
117
|
-
// self-host stacks, which do not, skip it and let the fence do the work.
|
|
118
|
-
if (imapWorkerQueueUrl) {
|
|
119
|
-
for (const accountId of accountIds) {
|
|
120
|
-
await sqsFor(imapWorkerQueueUrl).send(
|
|
121
|
-
new SendMessageCommand({
|
|
122
|
-
QueueUrl: imapWorkerQueueUrl,
|
|
123
|
-
MessageBody: JSON.stringify({
|
|
124
|
-
type: "IMAP_WORKER_STOP",
|
|
125
|
-
accountConfigId,
|
|
126
|
-
accountId,
|
|
127
|
-
}),
|
|
128
|
-
}),
|
|
129
|
-
);
|
|
130
|
-
}
|
|
131
|
-
log.info(
|
|
132
|
-
{ accountConfigId, count: accountIds.length },
|
|
133
|
-
"Enqueued imap-worker stop signals",
|
|
134
|
-
);
|
|
135
|
-
} else {
|
|
136
|
-
log.info(
|
|
137
|
-
{ accountConfigId, count: accountIds.length },
|
|
138
|
-
"No imap-worker stop queue configured; skipping stop signals (tombstone fence halts the worker)",
|
|
139
|
-
);
|
|
140
|
-
}
|
|
141
|
-
|
|
142
83
|
await signOut(userId, log);
|
|
143
84
|
|
|
144
85
|
const finalizeEvent: AccountFinalizeEvent = {
|
|
145
86
|
type: "FinalizeAccountDelete",
|
|
146
87
|
accountConfigId,
|
|
147
88
|
};
|
|
148
|
-
await
|
|
149
|
-
new SendMessageCommand({
|
|
150
|
-
QueueUrl: accountFinalizeQueueUrl,
|
|
151
|
-
MessageBody: JSON.stringify(finalizeEvent),
|
|
152
|
-
}),
|
|
153
|
-
);
|
|
89
|
+
await accountFinalize.send({ body: finalizeEvent });
|
|
154
90
|
log.info({ accountConfigId }, "Enqueued finalize event");
|
|
155
91
|
};
|
|
156
92
|
|
|
157
93
|
const log = createLogger();
|
|
158
94
|
|
|
95
|
+
let fanoutQueues: FanoutQueues | null = null;
|
|
96
|
+
|
|
97
|
+
const composedQueues = (): FanoutQueues => {
|
|
98
|
+
fanoutQueues ??= sqsFanoutQueues();
|
|
99
|
+
return fanoutQueues;
|
|
100
|
+
};
|
|
101
|
+
|
|
159
102
|
export const handler: SQSHandler = withTelemetry(
|
|
160
103
|
async (event: SQSEvent): Promise<SQSBatchResponse> => {
|
|
161
104
|
const batchItemFailures: { itemIdentifier: string }[] = [];
|
|
@@ -171,7 +114,9 @@ export const handler: SQSHandler = withTelemetry(
|
|
|
171
114
|
);
|
|
172
115
|
|
|
173
116
|
const start = Date.now();
|
|
174
|
-
const failed = await processAccountFanout(fanoutEvent, log
|
|
117
|
+
const failed = await processAccountFanout(fanoutEvent, log, {
|
|
118
|
+
queues: composedQueues(),
|
|
119
|
+
})
|
|
175
120
|
.then(() => false)
|
|
176
121
|
.catch((error) => {
|
|
177
122
|
log.error(
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { describe, it } from "node:test";
|
|
3
|
-
import
|
|
3
|
+
import {
|
|
4
|
+
type RecordingQueue,
|
|
5
|
+
recordingQueue,
|
|
6
|
+
} from "@remit/data-ports/recording-queue";
|
|
4
7
|
import { noopLogger } from "@remit/logger";
|
|
5
8
|
import type { CascadeServices } from "../cascade.js";
|
|
6
9
|
import type { AccountDataPurgeEvent } from "../events.js";
|
|
@@ -9,26 +12,6 @@ import {
|
|
|
9
12
|
processAccountDataPurge,
|
|
10
13
|
} from "./account-purge.js";
|
|
11
14
|
|
|
12
|
-
interface Sent {
|
|
13
|
-
queueUrl: string | undefined;
|
|
14
|
-
body: Record<string, unknown>;
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
const recordingSqs = (sent: Sent[]): SQSClient =>
|
|
18
|
-
({
|
|
19
|
-
send: async (command: {
|
|
20
|
-
input: { QueueUrl?: string; MessageBody?: string };
|
|
21
|
-
}) => {
|
|
22
|
-
sent.push({
|
|
23
|
-
queueUrl: command.input.QueueUrl,
|
|
24
|
-
body: command.input.MessageBody
|
|
25
|
-
? JSON.parse(command.input.MessageBody)
|
|
26
|
-
: {},
|
|
27
|
-
});
|
|
28
|
-
return {};
|
|
29
|
-
},
|
|
30
|
-
}) as unknown as SQSClient;
|
|
31
|
-
|
|
32
15
|
const buildServices = (
|
|
33
16
|
manifest: Array<{
|
|
34
17
|
threadMessageId: string;
|
|
@@ -54,25 +37,21 @@ const event: AccountDataPurgeEvent = {
|
|
|
54
37
|
accountConfigId: "cfg-1",
|
|
55
38
|
};
|
|
56
39
|
|
|
57
|
-
const baseDeps = (
|
|
58
|
-
sent: Sent[],
|
|
59
|
-
overrides: Partial<ProcessPurgeFanoutDeps> = {},
|
|
60
|
-
): ProcessPurgeFanoutDeps => ({
|
|
40
|
+
const baseDeps = (queue: RecordingQueue): ProcessPurgeFanoutDeps => ({
|
|
61
41
|
services: buildServices([
|
|
62
42
|
{ threadMessageId: "tm-1", messageId: "msg-1", mailboxId: "mbx-1" },
|
|
63
43
|
{ threadMessageId: "tm-2", messageId: "msg-2", mailboxId: "mbx-1" },
|
|
64
44
|
]),
|
|
65
|
-
|
|
66
|
-
accountPurgeDeleteQueueUrl: "http://queue/purge",
|
|
67
|
-
...overrides,
|
|
45
|
+
queue: queue.grouped,
|
|
68
46
|
});
|
|
69
47
|
|
|
70
48
|
describe("processAccountDataPurge", () => {
|
|
71
49
|
it("enqueues a subtrees batch and one container leftover", async () => {
|
|
72
|
-
const
|
|
73
|
-
await processAccountDataPurge(event, noopLogger, baseDeps(
|
|
50
|
+
const queue = recordingQueue();
|
|
51
|
+
await processAccountDataPurge(event, noopLogger, baseDeps(queue));
|
|
74
52
|
|
|
75
|
-
const finalize = sent
|
|
53
|
+
const finalize = queue.sent;
|
|
54
|
+
assert.ok(finalize.every((m) => m.groupId === "cfg-1"));
|
|
76
55
|
const kinds = finalize.map((m) => m.body.kind);
|
|
77
56
|
assert.deepEqual(kinds, ["subtrees", "container"]);
|
|
78
57
|
assert.equal(
|
|
@@ -83,23 +62,23 @@ describe("processAccountDataPurge", () => {
|
|
|
83
62
|
});
|
|
84
63
|
|
|
85
64
|
it("drops manifest rows outside the account's mailbox set", async () => {
|
|
86
|
-
const
|
|
65
|
+
const queue = recordingQueue();
|
|
87
66
|
await processAccountDataPurge(event, noopLogger, {
|
|
88
|
-
...baseDeps(
|
|
67
|
+
...baseDeps(queue),
|
|
89
68
|
services: buildServices([
|
|
90
69
|
{ threadMessageId: "tm-1", messageId: "msg-1", mailboxId: "mbx-1" },
|
|
91
70
|
{ threadMessageId: "tm-9", messageId: "msg-9", mailboxId: "other" },
|
|
92
71
|
]),
|
|
93
72
|
});
|
|
94
73
|
|
|
95
|
-
const subtrees = sent.find((m) => m.body.kind === "subtrees");
|
|
74
|
+
const subtrees = queue.sent.find((m) => m.body.kind === "subtrees");
|
|
96
75
|
assert.equal((subtrees?.body.items as unknown[])?.length, 1);
|
|
97
76
|
});
|
|
98
77
|
|
|
99
78
|
it("no-ops when the account is already gone", async () => {
|
|
100
|
-
const
|
|
79
|
+
const queue = recordingQueue();
|
|
101
80
|
await processAccountDataPurge(event, noopLogger, {
|
|
102
|
-
...baseDeps(
|
|
81
|
+
...baseDeps(queue),
|
|
103
82
|
services: {
|
|
104
83
|
accountService: {
|
|
105
84
|
describe: async () => {
|
|
@@ -112,14 +91,18 @@ describe("processAccountDataPurge", () => {
|
|
|
112
91
|
} as unknown as CascadeServices,
|
|
113
92
|
});
|
|
114
93
|
|
|
115
|
-
assert.equal(
|
|
94
|
+
assert.equal(
|
|
95
|
+
queue.sent.length,
|
|
96
|
+
0,
|
|
97
|
+
"nothing enqueued for a missing account",
|
|
98
|
+
);
|
|
116
99
|
});
|
|
117
100
|
|
|
118
101
|
it("rethrows a non-NotFound describe error", async () => {
|
|
119
|
-
const
|
|
102
|
+
const queue = recordingQueue();
|
|
120
103
|
await assert.rejects(
|
|
121
104
|
processAccountDataPurge(event, noopLogger, {
|
|
122
|
-
...baseDeps(
|
|
105
|
+
...baseDeps(queue),
|
|
123
106
|
services: {
|
|
124
107
|
accountService: {
|
|
125
108
|
describe: async () => {
|
|
@@ -1,11 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { GroupedQueueProducer } from "@remit/data-ports";
|
|
2
2
|
import type { Logger } from "@remit/logger";
|
|
3
3
|
import type { CascadeServices } from "../cascade.js";
|
|
4
|
-
import {
|
|
5
|
-
getAccountPurgeDeleteQueueUrl,
|
|
6
|
-
getCascadeServices,
|
|
7
|
-
getSqsClient,
|
|
8
|
-
} from "../config.js";
|
|
4
|
+
import { getCascadeServices } from "../config.js";
|
|
9
5
|
import type {
|
|
10
6
|
AccountDataPurgeEvent,
|
|
11
7
|
AccountDataPurgeFinalizeEvent,
|
|
@@ -13,9 +9,8 @@ import type {
|
|
|
13
9
|
} from "../events.js";
|
|
14
10
|
|
|
15
11
|
export interface ProcessPurgeFanoutDeps {
|
|
12
|
+
queue: GroupedQueueProducer<AccountDataPurgeFinalizeEvent>;
|
|
16
13
|
services?: CascadeServices;
|
|
17
|
-
sqs?: SQSClient;
|
|
18
|
-
accountPurgeDeleteQueueUrl?: string;
|
|
19
14
|
}
|
|
20
15
|
|
|
21
16
|
/**
|
|
@@ -36,20 +31,13 @@ const SUBTREE_BATCH_SIZE = 100;
|
|
|
36
31
|
* window; beyond it, the worker's deletes are no-ops on already-gone rows.
|
|
37
32
|
*/
|
|
38
33
|
const enqueuePurgeFinalize = async (
|
|
39
|
-
|
|
40
|
-
queueUrl: string,
|
|
34
|
+
queue: GroupedQueueProducer<AccountDataPurgeFinalizeEvent>,
|
|
41
35
|
accountId: string,
|
|
42
36
|
accountConfigId: string,
|
|
43
37
|
items: AccountDataPurgeSubtreeItem[],
|
|
44
38
|
): Promise<void> => {
|
|
45
|
-
const send = (event: AccountDataPurgeFinalizeEvent): Promise<
|
|
46
|
-
|
|
47
|
-
new SendMessageCommand({
|
|
48
|
-
QueueUrl: queueUrl,
|
|
49
|
-
MessageBody: JSON.stringify(event),
|
|
50
|
-
MessageGroupId: accountConfigId,
|
|
51
|
-
}),
|
|
52
|
-
);
|
|
39
|
+
const send = (event: AccountDataPurgeFinalizeEvent): Promise<void> =>
|
|
40
|
+
queue.send({ body: event, groupId: accountConfigId });
|
|
53
41
|
|
|
54
42
|
for (let i = 0; i < items.length; i += SUBTREE_BATCH_SIZE) {
|
|
55
43
|
await send({
|
|
@@ -90,12 +78,10 @@ const enqueuePurgeFinalize = async (
|
|
|
90
78
|
export const processAccountDataPurge = async (
|
|
91
79
|
event: AccountDataPurgeEvent,
|
|
92
80
|
log: Logger,
|
|
93
|
-
deps: ProcessPurgeFanoutDeps
|
|
81
|
+
deps: ProcessPurgeFanoutDeps,
|
|
94
82
|
): Promise<void> => {
|
|
95
83
|
const { accountId, accountConfigId } = event;
|
|
96
84
|
const services = deps.services ?? (await getCascadeServices());
|
|
97
|
-
const accountPurgeDeleteQueueUrl =
|
|
98
|
-
deps.accountPurgeDeleteQueueUrl ?? getAccountPurgeDeleteQueueUrl();
|
|
99
85
|
|
|
100
86
|
let mailboxIds: Set<string>;
|
|
101
87
|
try {
|
|
@@ -121,13 +107,7 @@ export const processAccountDataPurge = async (
|
|
|
121
107
|
messageId: tm.messageId,
|
|
122
108
|
}));
|
|
123
109
|
|
|
124
|
-
await enqueuePurgeFinalize(
|
|
125
|
-
deps.sqs ?? getSqsClient(accountPurgeDeleteQueueUrl),
|
|
126
|
-
accountPurgeDeleteQueueUrl,
|
|
127
|
-
accountId,
|
|
128
|
-
accountConfigId,
|
|
129
|
-
items,
|
|
130
|
-
);
|
|
110
|
+
await enqueuePurgeFinalize(deps.queue, accountId, accountConfigId, items);
|
|
131
111
|
log.info(
|
|
132
112
|
{
|
|
133
113
|
accountConfigId,
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
} from "@remit/backend/organize";
|
|
9
9
|
import type { FilterItem } from "@remit/data-ports";
|
|
10
10
|
import { NotFoundError } from "@remit/data-ports/errors";
|
|
11
|
+
import { recordingQueue } from "@remit/data-ports/recording-queue";
|
|
11
12
|
import {
|
|
12
13
|
createSqliteRepositories,
|
|
13
14
|
type DrizzleThreadMessageRepository,
|
|
@@ -39,6 +40,7 @@ const jobClient = (
|
|
|
39
40
|
literalClauses: Array<{ field: string; value: string }>,
|
|
40
41
|
): RemitClient =>
|
|
41
42
|
({
|
|
43
|
+
queues: { messageManagement: recordingQueue() },
|
|
42
44
|
organizeJobRequest: {
|
|
43
45
|
get: async () => ({
|
|
44
46
|
organizeJobId: "job-1",
|
package/src/config.test.ts
DELETED
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import { afterEach, describe, it } from "node:test";
|
|
3
|
-
import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
|
|
4
|
-
import { getImapWorkerQueueUrl, getSqsClient } from "./config.js";
|
|
5
|
-
|
|
6
|
-
// The queue URL a self-hosted compose stack hands the account worker
|
|
7
|
-
// (deploy/vps/remit.env.template). It is a plain http:// URL that is not
|
|
8
|
-
// localhost, which is exactly the case the endpoint heuristic used to miss.
|
|
9
|
-
const composeQueueUrl =
|
|
10
|
-
"http://queue:9324/000000000000/remit-account-purge-delete.fifo";
|
|
11
|
-
const awsQueueUrl =
|
|
12
|
-
"https://sqs.eu-west-1.amazonaws.com/123456789012/remit-account-purge-delete.fifo";
|
|
13
|
-
|
|
14
|
-
describe("getSqsClient", () => {
|
|
15
|
-
// Regression: the account-deletion cascade sends through this client. It
|
|
16
|
-
// used to be built once at import with no endpoint and no protocol, so on a
|
|
17
|
-
// self-hosted stack every cascade send went to real SQS instead of the local
|
|
18
|
-
// queue server and account deletion never completed.
|
|
19
|
-
it("selects the query protocol and the queue origin for a compose-stack queue URL", async () => {
|
|
20
|
-
const client = getSqsClient(composeQueueUrl);
|
|
21
|
-
assert.equal(client.config.protocol instanceof AwsQueryProtocol, true);
|
|
22
|
-
const endpoint = await client.config.endpoint?.();
|
|
23
|
-
assert.equal(endpoint?.hostname, "queue");
|
|
24
|
-
assert.equal(endpoint?.port, 9324);
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
it("leaves the default protocol and endpoint for a real SQS queue URL", async () => {
|
|
28
|
-
const client = getSqsClient(awsQueueUrl);
|
|
29
|
-
assert.equal(client.config.protocol instanceof AwsQueryProtocol, false);
|
|
30
|
-
assert.equal(client.config.endpoint, undefined);
|
|
31
|
-
});
|
|
32
|
-
|
|
33
|
-
it("reuses one client per queue URL and keeps distinct queues apart", () => {
|
|
34
|
-
assert.equal(getSqsClient(composeQueueUrl), getSqsClient(composeQueueUrl));
|
|
35
|
-
assert.notEqual(getSqsClient(composeQueueUrl), getSqsClient(awsQueueUrl));
|
|
36
|
-
});
|
|
37
|
-
});
|
|
38
|
-
|
|
39
|
-
describe("getImapWorkerQueueUrl", () => {
|
|
40
|
-
// The imap-worker stop queue is optional: the self-host compose stacks do not
|
|
41
|
-
// provision it, and reading it through expect-env would throw and abort the
|
|
42
|
-
// deletion fanout before it ever reached the finalize step. It must yield
|
|
43
|
-
// undefined when unset so the fanout can skip the (no-op) stop signal.
|
|
44
|
-
const prev = process.env.SQS_QUEUE_URL_IMAP_WORKER;
|
|
45
|
-
afterEach(() => {
|
|
46
|
-
if (prev === undefined) delete process.env.SQS_QUEUE_URL_IMAP_WORKER;
|
|
47
|
-
else process.env.SQS_QUEUE_URL_IMAP_WORKER = prev;
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
it("returns undefined when the var is unset instead of throwing", () => {
|
|
51
|
-
delete process.env.SQS_QUEUE_URL_IMAP_WORKER;
|
|
52
|
-
assert.equal(getImapWorkerQueueUrl(), undefined);
|
|
53
|
-
});
|
|
54
|
-
});
|