@remit/account-worker 0.0.28 → 0.0.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@remit/account-worker",
3
- "version": "0.0.28",
3
+ "version": "0.0.29",
4
4
  "type": "module",
5
5
  "main": "dist/index.js",
6
6
  "exports": {
@@ -41,12 +41,12 @@
41
41
  "@remit/sqs-client": "*",
42
42
  "@remit/storage-service": "*",
43
43
  "@remit/electrodb-entities": "*",
44
- "@aws-sdk/client-sqs": "*",
45
- "archiver": "*",
46
- "electrodb": "*",
47
- "expect-env": "*",
48
- "@types/archiver": "*",
49
- "@types/aws-lambda": "*"
44
+ "@aws-sdk/client-sqs": "^3.1090.0",
45
+ "archiver": "^8.0.0",
46
+ "electrodb": "^3.9.1",
47
+ "expect-env": "^1.0.1",
48
+ "@types/archiver": "^8.0.0",
49
+ "@types/aws-lambda": "^8.10.162"
50
50
  },
51
51
  "devDependencies": {
52
52
  "@aws-sdk/core": "*",
package/src/config.ts CHANGED
@@ -1,44 +1,39 @@
1
- import type { SQSClient } from "@aws-sdk/client-sqs";
2
1
  import { getClient } from "@remit/backend/client";
2
+ import type { GroupedQueueProducer, QueueProducer } from "@remit/data-ports";
3
3
  import { createQueueProducer } from "@remit/sqs-client/producer";
4
4
  import type { StorageService } from "@remit/storage-service";
5
5
  import { createStorageService } from "@remit/storage-service/s3";
6
6
  import { env } from "expect-env";
7
7
  import type { CascadeServices } from "./cascade.js";
8
-
9
- // A client per queue URL, resolved at send time rather than at import. The
10
- // endpoint and wire protocol are derived from the URL, so a self-hosted stack
11
- // (http://queue:9324/...) gets the query protocol its queue server speaks while
12
- // real SQS keeps the default. Building at import is not an option: the queue
13
- // URLs are read lazily for the reason given below.
14
- const sqsClients = new Map<string, SQSClient>();
15
-
16
- export const getSqsClient = (queueUrl: string): SQSClient => {
17
- const existing = sqsClients.get(queueUrl);
18
- if (existing) return existing;
19
- const client = createQueueProducer({ queueUrl });
20
- sqsClients.set(queueUrl, client);
21
- return client;
22
- };
8
+ import type {
9
+ AccountDataPurgeFinalizeEvent,
10
+ AccountFinalizeEvent,
11
+ } from "./events.js";
23
12
 
24
13
  // SQS env vars are lazy-evaluated. The fanout worker needs them at handler time;
25
14
  // the finalize worker imports `getCascadeServices` from this module but talks to
26
15
  // no queue, so its Lambda doesn't carry the env vars. Eager evaluation here would
27
16
  // crash finalize at module load — getters defer the read to the fanout-only call
28
17
  // sites.
29
- // Optional, and read through `process.env` rather than expect-env so an unset
30
- // var yields undefined instead of throwing. IMAP_WORKER_STOP is a no-op
31
- // acknowledgement in the cascade contract — the account tombstone fence is what
32
- // actually halts the worker — so a deployment that never provisions a dedicated
33
- // imap-worker stop queue (the self-host compose stacks) skips the signal rather
34
- // than failing the whole fanout. AWS sets the var and keeps sending it.
35
- export const getImapWorkerQueueUrl = (): string | undefined =>
36
- process.env.SQS_QUEUE_URL_IMAP_WORKER;
37
18
  export const getAccountFinalizeQueueUrl = (): string =>
38
19
  env.SQS_QUEUE_URL_ACCOUNT_FINALIZE;
39
20
  export const getAccountPurgeDeleteQueueUrl = (): string =>
40
21
  env.SQS_QUEUE_URL_ACCOUNT_PURGE_DELETE;
41
22
 
23
+ export interface FanoutQueues {
24
+ accountFinalize: QueueProducer<AccountFinalizeEvent>;
25
+ accountPurgeDelete: GroupedQueueProducer<AccountDataPurgeFinalizeEvent>;
26
+ }
27
+
28
+ export const sqsFanoutQueues = (): FanoutQueues => ({
29
+ accountFinalize: createQueueProducer({
30
+ queueUrl: getAccountFinalizeQueueUrl(),
31
+ }),
32
+ accountPurgeDelete: createQueueProducer({
33
+ queueUrl: getAccountPurgeDeleteQueueUrl(),
34
+ }),
35
+ });
36
+
42
37
  // The RemitClient is resolved lazily and cached, not at module load: the
43
38
  // DynamoDB backend is injected by the composition root before the first cascade
44
39
  // runs, so building at import time would race the registration.
package/src/events.ts CHANGED
@@ -1,74 +1,11 @@
1
- import type { ThreadMessageItem } from "@remit/data-ports";
2
-
3
- type MessageCategoryValue = ThreadMessageItem["category"];
4
-
5
- export type AccountDeleteEvent = {
6
- type: "AccountDelete";
7
- accountConfigId: string;
8
- };
9
-
10
- /**
11
- * Per-account hard-delete. Purges ONE mail account's data (mailboxes,
12
- * messages, body parts, thread messages, S3 objects) while
13
- * keeping the AccountConfig/tenant and any sibling accounts intact. Emitted
14
- * by the API on `deleteAccount`; consumed by the fanout worker.
15
- */
16
- export type AccountDataPurgeEvent = {
17
- type: "AccountDataPurge";
18
- accountId: string;
19
- accountConfigId: string;
20
- };
21
-
22
- export type AccountExportEvent = {
23
- type: "AccountExport";
24
- accountConfigId: string;
25
- accountExportRequestId: string;
26
- };
27
-
28
- /**
29
- * A "all like these" back-apply job (RFC 034, #1278). Rides the same
30
- * account-fanout queue as export: the API writes a Pending OrganizeJobRequest
31
- * row and enqueues this; the fanout worker matches the corpus and applies the
32
- * action, driving the row to Complete/Failed. Never creates a Filter row.
33
- */
34
- export type OrganizeJobEvent = {
35
- type: "OrganizeJob";
36
- accountConfigId: string;
37
- organizeJobId: string;
38
- };
39
-
40
- /**
41
- * Retroactive half of a sender-category override (#415, RFC 039 Decision 3).
42
- * Rides the same account-fanout queue as the organize back-apply: the API
43
- * enqueues one when `Address.flags.category` is SET, and the fanout worker
44
- * re-labels a bounded recent batch of that sender's already-classified mail.
45
- * A CLEAR never enqueues one: reverting to auto-classification says nothing
46
- * about what the classifier would have answered, and re-deriving it would need
47
- * every message's body back.
48
- *
49
- * `category` and `categorySetAt` together name WHICH set this job was fired
50
- * for, and the worker refuses a job whose set no longer stands on the Address.
51
- * That check is what the standard queue's semantics make necessary rather than
52
- * merely tidy: delivery is at-least-once and unordered, so without it a job
53
- * that fails and redelivers after the user has corrected the category re-labels
54
- * those 500 messages back to the superseded one, leaving the flag and the mail
55
- * permanently disagreeing with nothing to reconcile them.
56
- */
57
- export type SenderCategoryBackApplyEvent = {
58
- type: "SenderCategoryBackApply";
59
- accountConfigId: string;
60
- addressId: string;
61
- normalizedEmail: string;
62
- category: MessageCategoryValue;
63
- categorySetAt: number;
64
- };
65
-
66
- export type AccountFanoutEvent =
67
- | AccountDeleteEvent
68
- | AccountDataPurgeEvent
69
- | AccountExportEvent
70
- | OrganizeJobEvent
71
- | SenderCategoryBackApplyEvent;
1
+ export type {
2
+ AccountDataPurgeEvent,
3
+ AccountDeleteEvent,
4
+ AccountExportEvent,
5
+ AccountFanoutEvent,
6
+ OrganizeJobEvent,
7
+ SenderCategoryBackApplyEvent,
8
+ } from "@remit/data-ports";
72
9
 
73
10
  export type AccountDeleteFinalizeEvent = {
74
11
  type: "FinalizeAccountDelete";
@@ -1,6 +1,6 @@
1
1
  import assert from "node:assert/strict";
2
- import { afterEach, describe, it } from "node:test";
3
- import type { SQSClient } from "@aws-sdk/client-sqs";
2
+ import { describe, it } from "node:test";
3
+ import { recordingQueue } from "@remit/data-ports/recording-queue";
4
4
  import { noopLogger } from "@remit/logger";
5
5
  import type { CascadeServices } from "../cascade.js";
6
6
  import type { AccountDeleteEvent } from "../events.js";
@@ -36,73 +36,27 @@ const buildServices = (): CascadeServices =>
36
36
  messageLabelService: { listByLabelId: async () => [] },
37
37
  }) as unknown as CascadeServices;
38
38
 
39
- interface SentMessage {
40
- queueUrl: string | undefined;
41
- body: { type?: string; accountId?: string };
42
- }
43
-
44
- const recordingSqs = (sent: SentMessage[]): SQSClient =>
45
- ({
46
- send: async (command: {
47
- input: { QueueUrl?: string; MessageBody?: string };
48
- }) => {
49
- sent.push({
50
- queueUrl: command.input.QueueUrl,
51
- body: JSON.parse(command.input.MessageBody ?? "{}"),
52
- });
53
- return {};
54
- },
55
- }) as unknown as SQSClient;
56
-
57
39
  const deleteEvent: AccountDeleteEvent = {
58
40
  type: "AccountDelete",
59
41
  accountConfigId: "cfg-1",
60
42
  };
61
43
 
62
- describe("processAccountFanout — imap-worker stop signal on account delete", () => {
63
- const prev = process.env.SQS_QUEUE_URL_IMAP_WORKER;
64
- afterEach(() => {
65
- if (prev === undefined) delete process.env.SQS_QUEUE_URL_IMAP_WORKER;
66
- else process.env.SQS_QUEUE_URL_IMAP_WORKER = prev;
67
- });
68
-
69
- it("skips the stop signal when no imap-worker queue is configured, still enqueuing finalize", async () => {
70
- delete process.env.SQS_QUEUE_URL_IMAP_WORKER;
71
- const sent: SentMessage[] = [];
72
-
73
- await processAccountFanout(deleteEvent, noopLogger, {
74
- services: buildServices(),
75
- sqs: recordingSqs(sent),
76
- signOut: async () => {},
77
- accountFinalizeQueueUrl: "http://queue/finalize",
78
- });
79
-
80
- const stops = sent.filter((m) => m.body.type === "IMAP_WORKER_STOP");
81
- const finalize = sent.filter(
82
- (m) => m.body.type === "FinalizeAccountDelete",
83
- );
84
- assert.equal(stops.length, 0, "no stop signal without a configured queue");
85
- assert.equal(finalize.length, 1, "finalize is still enqueued");
86
- });
87
-
88
- it("enqueues one stop signal per account when a queue is configured", async () => {
89
- const sent: SentMessage[] = [];
44
+ describe("processAccountFanout on account delete", () => {
45
+ it("enqueues the finalize step", async () => {
46
+ const accountFinalize = recordingQueue();
90
47
 
91
48
  await processAccountFanout(deleteEvent, noopLogger, {
49
+ queues: {
50
+ accountFinalize,
51
+ accountPurgeDelete: recordingQueue(),
52
+ },
92
53
  services: buildServices(),
93
- sqs: recordingSqs(sent),
94
54
  signOut: async () => {},
95
- imapWorkerQueueUrl: "http://queue/imap",
96
- accountFinalizeQueueUrl: "http://queue/finalize",
97
55
  });
98
56
 
99
- const stops = sent.filter((m) => m.body.type === "IMAP_WORKER_STOP");
100
- assert.equal(stops.length, 1, "one stop signal for the single account");
101
- assert.equal(stops[0]?.queueUrl, "http://queue/imap");
102
- assert.equal(stops[0]?.body.accountId, "acc-1");
103
- assert.equal(
104
- sent.filter((m) => m.body.type === "FinalizeAccountDelete").length,
105
- 1,
57
+ assert.deepEqual(
58
+ accountFinalize.sent.map((m) => m.body.type),
59
+ ["FinalizeAccountDelete"],
106
60
  );
107
61
  });
108
62
  });
@@ -1,4 +1,3 @@
1
- import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
2
1
  import { createLogger, type Logger } from "@remit/logger";
3
2
  import {
4
3
  queueNameFromEventSource,
@@ -7,13 +6,10 @@ import {
7
6
  import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
8
7
  import type { SQSBatchResponse, SQSEvent, SQSHandler } from "aws-lambda";
9
8
  import type { CascadeServices } from "../cascade.js";
10
- import { enumerateCascadeEntities } from "../cascade.js";
11
9
  import {
12
- getAccountFinalizeQueueUrl,
13
- getAccountPurgeDeleteQueueUrl,
10
+ type FanoutQueues,
14
11
  getCascadeServices,
15
- getImapWorkerQueueUrl,
16
- getSqsClient,
12
+ sqsFanoutQueues,
17
13
  } from "../config.js";
18
14
  import {
19
15
  type DeletionCapabilities,
@@ -32,36 +28,27 @@ import { processSenderCategoryBackApply } from "./sender-category-backapply.js";
32
28
  installNodeLogger();
33
29
 
34
30
  export interface ProcessAccountFanoutDeps {
31
+ queues: FanoutQueues;
35
32
  services?: CascadeServices;
36
- sqs?: SQSClient;
37
33
  signOut?: DeletionCapabilities["signOut"];
38
- imapWorkerQueueUrl?: string;
39
- accountFinalizeQueueUrl?: string;
40
- accountPurgeDeleteQueueUrl?: string;
41
34
  }
42
35
 
43
36
  /**
44
- * Fanout step of the account-deletion cascade. Read-only: enumerates the
45
- * AccountConfig's data and dispatches downstream work — per-account
46
- * imap-worker stops, a sign-out, and the finalize
47
- * enqueue. Throws on any non-swallowable error so SQS partial batch-failure
37
+ * Fanout step of the account-deletion cascade. Signs the user out and
38
+ * enqueues the finalize step. Throws on any non-swallowable error so SQS partial batch-failure
48
39
  * retries the whole record.
49
40
  */
50
41
  export const processAccountFanout = async (
51
42
  event: AccountFanoutEvent,
52
43
  log: Logger,
53
- deps: ProcessAccountFanoutDeps = {},
44
+ deps: ProcessAccountFanoutDeps,
54
45
  ): Promise<void> => {
55
46
  const services = deps.services ?? (await getCascadeServices());
56
- const sqsFor = (queueUrl: string): SQSClient =>
57
- deps.sqs ?? getSqsClient(queueUrl);
58
47
 
59
48
  if (event.type === "AccountDataPurge") {
60
49
  await processAccountDataPurge(event, log, {
61
50
  services,
62
- sqs: deps.sqs,
63
- accountPurgeDeleteQueueUrl:
64
- deps.accountPurgeDeleteQueueUrl ?? getAccountPurgeDeleteQueueUrl(),
51
+ queue: deps.queues.accountPurgeDelete,
65
52
  });
66
53
  return;
67
54
  }
@@ -81,81 +68,42 @@ export const processAccountFanout = async (
81
68
  return;
82
69
  }
83
70
 
84
- await processAccountDelete(event, log, services, sqsFor, deps);
71
+ await processAccountDelete(event, log, services, deps);
85
72
  };
86
73
 
87
74
  const processAccountDelete = async (
88
75
  event: AccountDeleteEvent,
89
76
  log: Logger,
90
77
  services: CascadeServices,
91
- sqsFor: (queueUrl: string) => SQSClient,
92
78
  deps: ProcessAccountFanoutDeps,
93
79
  ): Promise<void> => {
94
80
  const { accountConfigId } = event;
95
81
  const signOut = deps.signOut ?? (await getDeletionCapabilities()).signOut;
96
- const imapWorkerQueueUrl = deps.imapWorkerQueueUrl ?? getImapWorkerQueueUrl();
97
- const accountFinalizeQueueUrl =
98
- deps.accountFinalizeQueueUrl ?? getAccountFinalizeQueueUrl();
82
+ const { accountFinalize } = deps.queues;
99
83
 
100
84
  const accountConfig =
101
85
  await services.accountConfigService.get(accountConfigId);
102
86
  const userId = accountConfig.userId;
103
87
 
104
- const { entities, messageIds: _messageIds } = await enumerateCascadeEntities(
105
- accountConfigId,
106
- services,
107
- log,
108
- );
109
-
110
- const accountIds = entities
111
- .filter((e) => e.entityType === "Account")
112
- .map((e) => e.key.accountId);
113
-
114
- // IMAP_WORKER_STOP is a no-op acknowledgement — the account tombstone fence
115
- // (isActive=false + deletedAt) is what actually halts the worker. Deployments
116
- // that provision a dedicated imap-worker stop queue get the signal; the
117
- // self-host stacks, which do not, skip it and let the fence do the work.
118
- if (imapWorkerQueueUrl) {
119
- for (const accountId of accountIds) {
120
- await sqsFor(imapWorkerQueueUrl).send(
121
- new SendMessageCommand({
122
- QueueUrl: imapWorkerQueueUrl,
123
- MessageBody: JSON.stringify({
124
- type: "IMAP_WORKER_STOP",
125
- accountConfigId,
126
- accountId,
127
- }),
128
- }),
129
- );
130
- }
131
- log.info(
132
- { accountConfigId, count: accountIds.length },
133
- "Enqueued imap-worker stop signals",
134
- );
135
- } else {
136
- log.info(
137
- { accountConfigId, count: accountIds.length },
138
- "No imap-worker stop queue configured; skipping stop signals (tombstone fence halts the worker)",
139
- );
140
- }
141
-
142
88
  await signOut(userId, log);
143
89
 
144
90
  const finalizeEvent: AccountFinalizeEvent = {
145
91
  type: "FinalizeAccountDelete",
146
92
  accountConfigId,
147
93
  };
148
- await sqsFor(accountFinalizeQueueUrl).send(
149
- new SendMessageCommand({
150
- QueueUrl: accountFinalizeQueueUrl,
151
- MessageBody: JSON.stringify(finalizeEvent),
152
- }),
153
- );
94
+ await accountFinalize.send({ body: finalizeEvent });
154
95
  log.info({ accountConfigId }, "Enqueued finalize event");
155
96
  };
156
97
 
157
98
  const log = createLogger();
158
99
 
100
+ let fanoutQueues: FanoutQueues | null = null;
101
+
102
+ const composedQueues = (): FanoutQueues => {
103
+ fanoutQueues ??= sqsFanoutQueues();
104
+ return fanoutQueues;
105
+ };
106
+
159
107
  export const handler: SQSHandler = withTelemetry(
160
108
  async (event: SQSEvent): Promise<SQSBatchResponse> => {
161
109
  const batchItemFailures: { itemIdentifier: string }[] = [];
@@ -171,7 +119,9 @@ export const handler: SQSHandler = withTelemetry(
171
119
  );
172
120
 
173
121
  const start = Date.now();
174
- const failed = await processAccountFanout(fanoutEvent, log)
122
+ const failed = await processAccountFanout(fanoutEvent, log, {
123
+ queues: composedQueues(),
124
+ })
175
125
  .then(() => false)
176
126
  .catch((error) => {
177
127
  log.error(
@@ -1,6 +1,9 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { describe, it } from "node:test";
3
- import type { SQSClient } from "@aws-sdk/client-sqs";
3
+ import {
4
+ type RecordingQueue,
5
+ recordingQueue,
6
+ } from "@remit/data-ports/recording-queue";
4
7
  import { noopLogger } from "@remit/logger";
5
8
  import type { CascadeServices } from "../cascade.js";
6
9
  import type { AccountDataPurgeEvent } from "../events.js";
@@ -9,26 +12,6 @@ import {
9
12
  processAccountDataPurge,
10
13
  } from "./account-purge.js";
11
14
 
12
- interface Sent {
13
- queueUrl: string | undefined;
14
- body: Record<string, unknown>;
15
- }
16
-
17
- const recordingSqs = (sent: Sent[]): SQSClient =>
18
- ({
19
- send: async (command: {
20
- input: { QueueUrl?: string; MessageBody?: string };
21
- }) => {
22
- sent.push({
23
- queueUrl: command.input.QueueUrl,
24
- body: command.input.MessageBody
25
- ? JSON.parse(command.input.MessageBody)
26
- : {},
27
- });
28
- return {};
29
- },
30
- }) as unknown as SQSClient;
31
-
32
15
  const buildServices = (
33
16
  manifest: Array<{
34
17
  threadMessageId: string;
@@ -54,25 +37,21 @@ const event: AccountDataPurgeEvent = {
54
37
  accountConfigId: "cfg-1",
55
38
  };
56
39
 
57
- const baseDeps = (
58
- sent: Sent[],
59
- overrides: Partial<ProcessPurgeFanoutDeps> = {},
60
- ): ProcessPurgeFanoutDeps => ({
40
+ const baseDeps = (queue: RecordingQueue): ProcessPurgeFanoutDeps => ({
61
41
  services: buildServices([
62
42
  { threadMessageId: "tm-1", messageId: "msg-1", mailboxId: "mbx-1" },
63
43
  { threadMessageId: "tm-2", messageId: "msg-2", mailboxId: "mbx-1" },
64
44
  ]),
65
- sqs: recordingSqs(sent),
66
- accountPurgeDeleteQueueUrl: "http://queue/purge",
67
- ...overrides,
45
+ queue,
68
46
  });
69
47
 
70
48
  describe("processAccountDataPurge", () => {
71
49
  it("enqueues a subtrees batch and one container leftover", async () => {
72
- const sent: Sent[] = [];
73
- await processAccountDataPurge(event, noopLogger, baseDeps(sent));
50
+ const queue = recordingQueue();
51
+ await processAccountDataPurge(event, noopLogger, baseDeps(queue));
74
52
 
75
- const finalize = sent.filter((m) => m.queueUrl === "http://queue/purge");
53
+ const finalize = queue.sent;
54
+ assert.ok(finalize.every((m) => m.groupId === "cfg-1"));
76
55
  const kinds = finalize.map((m) => m.body.kind);
77
56
  assert.deepEqual(kinds, ["subtrees", "container"]);
78
57
  assert.equal(
@@ -83,23 +62,23 @@ describe("processAccountDataPurge", () => {
83
62
  });
84
63
 
85
64
  it("drops manifest rows outside the account's mailbox set", async () => {
86
- const sent: Sent[] = [];
65
+ const queue = recordingQueue();
87
66
  await processAccountDataPurge(event, noopLogger, {
88
- ...baseDeps(sent),
67
+ ...baseDeps(queue),
89
68
  services: buildServices([
90
69
  { threadMessageId: "tm-1", messageId: "msg-1", mailboxId: "mbx-1" },
91
70
  { threadMessageId: "tm-9", messageId: "msg-9", mailboxId: "other" },
92
71
  ]),
93
72
  });
94
73
 
95
- const subtrees = sent.find((m) => m.body.kind === "subtrees");
74
+ const subtrees = queue.sent.find((m) => m.body.kind === "subtrees");
96
75
  assert.equal((subtrees?.body.items as unknown[])?.length, 1);
97
76
  });
98
77
 
99
78
  it("no-ops when the account is already gone", async () => {
100
- const sent: Sent[] = [];
79
+ const queue = recordingQueue();
101
80
  await processAccountDataPurge(event, noopLogger, {
102
- ...baseDeps(sent),
81
+ ...baseDeps(queue),
103
82
  services: {
104
83
  accountService: {
105
84
  describe: async () => {
@@ -112,14 +91,18 @@ describe("processAccountDataPurge", () => {
112
91
  } as unknown as CascadeServices,
113
92
  });
114
93
 
115
- assert.equal(sent.length, 0, "nothing enqueued for a missing account");
94
+ assert.equal(
95
+ queue.sent.length,
96
+ 0,
97
+ "nothing enqueued for a missing account",
98
+ );
116
99
  });
117
100
 
118
101
  it("rethrows a non-NotFound describe error", async () => {
119
- const sent: Sent[] = [];
102
+ const queue = recordingQueue();
120
103
  await assert.rejects(
121
104
  processAccountDataPurge(event, noopLogger, {
122
- ...baseDeps(sent),
105
+ ...baseDeps(queue),
123
106
  services: {
124
107
  accountService: {
125
108
  describe: async () => {
@@ -1,11 +1,7 @@
1
- import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
1
+ import type { GroupedQueueProducer } from "@remit/data-ports";
2
2
  import type { Logger } from "@remit/logger";
3
3
  import type { CascadeServices } from "../cascade.js";
4
- import {
5
- getAccountPurgeDeleteQueueUrl,
6
- getCascadeServices,
7
- getSqsClient,
8
- } from "../config.js";
4
+ import { getCascadeServices } from "../config.js";
9
5
  import type {
10
6
  AccountDataPurgeEvent,
11
7
  AccountDataPurgeFinalizeEvent,
@@ -13,9 +9,8 @@ import type {
13
9
  } from "../events.js";
14
10
 
15
11
  export interface ProcessPurgeFanoutDeps {
12
+ queue: GroupedQueueProducer<AccountDataPurgeFinalizeEvent>;
16
13
  services?: CascadeServices;
17
- sqs?: SQSClient;
18
- accountPurgeDeleteQueueUrl?: string;
19
14
  }
20
15
 
21
16
  /**
@@ -36,20 +31,13 @@ const SUBTREE_BATCH_SIZE = 100;
36
31
  * window; beyond it, the worker's deletes are no-ops on already-gone rows.
37
32
  */
38
33
  const enqueuePurgeFinalize = async (
39
- sqs: SQSClient,
40
- queueUrl: string,
34
+ queue: GroupedQueueProducer<AccountDataPurgeFinalizeEvent>,
41
35
  accountId: string,
42
36
  accountConfigId: string,
43
37
  items: AccountDataPurgeSubtreeItem[],
44
38
  ): Promise<void> => {
45
- const send = (event: AccountDataPurgeFinalizeEvent): Promise<unknown> =>
46
- sqs.send(
47
- new SendMessageCommand({
48
- QueueUrl: queueUrl,
49
- MessageBody: JSON.stringify(event),
50
- MessageGroupId: accountConfigId,
51
- }),
52
- );
39
+ const send = (event: AccountDataPurgeFinalizeEvent): Promise<void> =>
40
+ queue.send({ body: event, groupId: accountConfigId });
53
41
 
54
42
  for (let i = 0; i < items.length; i += SUBTREE_BATCH_SIZE) {
55
43
  await send({
@@ -90,12 +78,10 @@ const enqueuePurgeFinalize = async (
90
78
  export const processAccountDataPurge = async (
91
79
  event: AccountDataPurgeEvent,
92
80
  log: Logger,
93
- deps: ProcessPurgeFanoutDeps = {},
81
+ deps: ProcessPurgeFanoutDeps,
94
82
  ): Promise<void> => {
95
83
  const { accountId, accountConfigId } = event;
96
84
  const services = deps.services ?? (await getCascadeServices());
97
- const accountPurgeDeleteQueueUrl =
98
- deps.accountPurgeDeleteQueueUrl ?? getAccountPurgeDeleteQueueUrl();
99
85
 
100
86
  let mailboxIds: Set<string>;
101
87
  try {
@@ -121,13 +107,7 @@ export const processAccountDataPurge = async (
121
107
  messageId: tm.messageId,
122
108
  }));
123
109
 
124
- await enqueuePurgeFinalize(
125
- deps.sqs ?? getSqsClient(accountPurgeDeleteQueueUrl),
126
- accountPurgeDeleteQueueUrl,
127
- accountId,
128
- accountConfigId,
129
- items,
130
- );
110
+ await enqueuePurgeFinalize(deps.queue, accountId, accountConfigId, items);
131
111
  log.info(
132
112
  {
133
113
  accountConfigId,
@@ -8,6 +8,7 @@ import {
8
8
  } from "@remit/backend/organize";
9
9
  import type { FilterItem } from "@remit/data-ports";
10
10
  import { NotFoundError } from "@remit/data-ports/errors";
11
+ import { recordingQueue } from "@remit/data-ports/recording-queue";
11
12
  import {
12
13
  createSqliteRepositories,
13
14
  type DrizzleThreadMessageRepository,
@@ -39,6 +40,7 @@ const jobClient = (
39
40
  literalClauses: Array<{ field: string; value: string }>,
40
41
  ): RemitClient =>
41
42
  ({
43
+ queues: { messageManagement: recordingQueue() },
42
44
  organizeJobRequest: {
43
45
  get: async () => ({
44
46
  organizeJobId: "job-1",
@@ -1,54 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { afterEach, describe, it } from "node:test";
3
- import { AwsQueryProtocol } from "@aws-sdk/core/protocols";
4
- import { getImapWorkerQueueUrl, getSqsClient } from "./config.js";
5
-
6
- // The queue URL a self-hosted compose stack hands the account worker
7
- // (deploy/vps/remit.env.template). It is a plain http:// URL that is not
8
- // localhost, which is exactly the case the endpoint heuristic used to miss.
9
- const composeQueueUrl =
10
- "http://queue:9324/000000000000/remit-account-purge-delete.fifo";
11
- const awsQueueUrl =
12
- "https://sqs.eu-west-1.amazonaws.com/123456789012/remit-account-purge-delete.fifo";
13
-
14
- describe("getSqsClient", () => {
15
- // Regression: the account-deletion cascade sends through this client. It
16
- // used to be built once at import with no endpoint and no protocol, so on a
17
- // self-hosted stack every cascade send went to real SQS instead of the local
18
- // queue server and account deletion never completed.
19
- it("selects the query protocol and the queue origin for a compose-stack queue URL", async () => {
20
- const client = getSqsClient(composeQueueUrl);
21
- assert.equal(client.config.protocol instanceof AwsQueryProtocol, true);
22
- const endpoint = await client.config.endpoint?.();
23
- assert.equal(endpoint?.hostname, "queue");
24
- assert.equal(endpoint?.port, 9324);
25
- });
26
-
27
- it("leaves the default protocol and endpoint for a real SQS queue URL", async () => {
28
- const client = getSqsClient(awsQueueUrl);
29
- assert.equal(client.config.protocol instanceof AwsQueryProtocol, false);
30
- assert.equal(client.config.endpoint, undefined);
31
- });
32
-
33
- it("reuses one client per queue URL and keeps distinct queues apart", () => {
34
- assert.equal(getSqsClient(composeQueueUrl), getSqsClient(composeQueueUrl));
35
- assert.notEqual(getSqsClient(composeQueueUrl), getSqsClient(awsQueueUrl));
36
- });
37
- });
38
-
39
- describe("getImapWorkerQueueUrl", () => {
40
- // The imap-worker stop queue is optional: the self-host compose stacks do not
41
- // provision it, and reading it through expect-env would throw and abort the
42
- // deletion fanout before it ever reached the finalize step. It must yield
43
- // undefined when unset so the fanout can skip the (no-op) stop signal.
44
- const prev = process.env.SQS_QUEUE_URL_IMAP_WORKER;
45
- afterEach(() => {
46
- if (prev === undefined) delete process.env.SQS_QUEUE_URL_IMAP_WORKER;
47
- else process.env.SQS_QUEUE_URL_IMAP_WORKER = prev;
48
- });
49
-
50
- it("returns undefined when the var is unset instead of throwing", () => {
51
- delete process.env.SQS_QUEUE_URL_IMAP_WORKER;
52
- assert.equal(getImapWorkerQueueUrl(), undefined);
53
- });
54
- });