@remit/account-worker 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +50 -0
- package/src/cascade.test.ts +80 -0
- package/src/cascade.ts +383 -0
- package/src/compose-relational.test.ts +104 -0
- package/src/compose-relational.ts +87 -0
- package/src/config.ts +70 -0
- package/src/deletion-capabilities.ts +94 -0
- package/src/events.ts +85 -0
- package/src/handlers/account-export.ts +132 -0
- package/src/handlers/account-fanout.ts +165 -0
- package/src/handlers/account-finalize.ts +311 -0
- package/src/handlers/account-purge.ts +226 -0
- package/src/handlers/organize-job.ts +78 -0
- package/src/index.ts +11 -0
- package/src/poller.ts +39 -0
- package/tsconfig.json +8 -0
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
import { inspect } from "node:util";
|
|
2
|
+
import { createLogger, type Logger, withTelemetry } from "@remit/logger-lambda";
|
|
3
|
+
import type { SQSEvent, SQSHandler } from "aws-lambda";
|
|
4
|
+
import {
|
|
5
|
+
type CascadeEntity,
|
|
6
|
+
type CascadeServices,
|
|
7
|
+
collectMessageChildEntities,
|
|
8
|
+
enumerateCascadeEntities,
|
|
9
|
+
} from "../cascade.js";
|
|
10
|
+
import { cascadeServices as defaultCascadeServices } from "../config.js";
|
|
11
|
+
import {
|
|
12
|
+
type DeletionCapabilities,
|
|
13
|
+
getDeletionCapabilities,
|
|
14
|
+
} from "../deletion-capabilities.js";
|
|
15
|
+
import type {
|
|
16
|
+
AccountDataPurgeFinalizeEvent,
|
|
17
|
+
AccountDeleteFinalizeEvent,
|
|
18
|
+
AccountFinalizeEvent,
|
|
19
|
+
} from "../events.js";
|
|
20
|
+
|
|
21
|
+
export interface ProcessFinalizeDeps {
|
|
22
|
+
capabilities?: DeletionCapabilities;
|
|
23
|
+
cascadeServices?: CascadeServices;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const resolveDeps = async (
|
|
27
|
+
deps: ProcessFinalizeDeps,
|
|
28
|
+
): Promise<{
|
|
29
|
+
capabilities: DeletionCapabilities;
|
|
30
|
+
services: CascadeServices;
|
|
31
|
+
}> => ({
|
|
32
|
+
capabilities: deps.capabilities ?? (await getDeletionCapabilities()),
|
|
33
|
+
services: deps.cascadeServices ?? defaultCascadeServices,
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* GDPR hard-delete: every row tied to the deleted AccountConfig is removed and
|
|
38
|
+
* every stored object under `accounts/{accountConfigId}/` is purged. The
|
|
39
|
+
* AccountConfig row itself is the LAST delete so that a mid-cascade replay still
|
|
40
|
+
* sees the cascade-in-progress flag (`deletedAt` set, `isActive=false`, written
|
|
41
|
+
* API-side) and re-runs cleanly. After a successful run nothing tied to the
|
|
42
|
+
* AccountConfig persists.
|
|
43
|
+
*
|
|
44
|
+
* Step order (non-negotiable per #320):
|
|
45
|
+
* 1. Content invalidation — runs FIRST so cached body parts cannot leak after
|
|
46
|
+
* the underlying objects are gone (a no-op where no CDN fronts content).
|
|
47
|
+
* 2. Relational/DDB cascade delete in dependency order (children → parents).
|
|
48
|
+
* 3. Storage prefix cleanup `accounts/{accountConfigId}/`.
|
|
49
|
+
* 4. AccountConfig delete.
|
|
50
|
+
*
|
|
51
|
+
* Idempotency: every step is replay-safe. The cascade on a missing key is a
|
|
52
|
+
* no-op, storage prefix delete on missing keys is a no-op, and invalidation is
|
|
53
|
+
* always idempotent. No explicit "already deleted" pre-checks — they make
|
|
54
|
+
* replays racier, not safer. Errors propagate so SQS retries the message;
|
|
55
|
+
* eventually the DLQ alarm fires and an operator inspects the failure.
|
|
56
|
+
*/
|
|
57
|
+
export const processAccountFinalize = async (
|
|
58
|
+
event: AccountDeleteFinalizeEvent,
|
|
59
|
+
log: Logger,
|
|
60
|
+
deps: ProcessFinalizeDeps = {},
|
|
61
|
+
): Promise<void> => {
|
|
62
|
+
const { accountConfigId } = event;
|
|
63
|
+
const { capabilities, services } = await resolveDeps(deps);
|
|
64
|
+
|
|
65
|
+
// Validate every deploy-time precondition BEFORE the first destructive step,
|
|
66
|
+
// so a missing var fails loud instead of deleting rows in the cascade and
|
|
67
|
+
// only then erroring at the storage step.
|
|
68
|
+
await capabilities.assertReady(log);
|
|
69
|
+
|
|
70
|
+
// Step 1: content invalidation — first, so cached body parts cannot leak
|
|
71
|
+
// after the underlying objects are gone.
|
|
72
|
+
log.info(
|
|
73
|
+
{ accountConfigId },
|
|
74
|
+
"Invalidating cached content for erased account",
|
|
75
|
+
);
|
|
76
|
+
await capabilities.invalidateContent(accountConfigId, log);
|
|
77
|
+
|
|
78
|
+
// Step 2: cascade delete (children → parents). The AccountConfig row is
|
|
79
|
+
// excluded from the cascade plan and removed last, after storage.
|
|
80
|
+
//
|
|
81
|
+
// SQS is at-least-once: a successful cascade can be redelivered. After a
|
|
82
|
+
// clean run the AccountConfig row is gone, so `describe()` throws
|
|
83
|
+
// `NotFoundError` — that's the success signal, not a failure. Treat it as
|
|
84
|
+
// "cascade already complete" and return cleanly. Any other error propagates
|
|
85
|
+
// so SQS retries → DLQ.
|
|
86
|
+
let entities: CascadeEntity[];
|
|
87
|
+
try {
|
|
88
|
+
const enumeration = await enumerateCascadeEntities(
|
|
89
|
+
accountConfigId,
|
|
90
|
+
services,
|
|
91
|
+
log,
|
|
92
|
+
);
|
|
93
|
+
entities = enumeration.entities;
|
|
94
|
+
} catch (error) {
|
|
95
|
+
if ((error as { name?: string })?.name === "NotFoundError") {
|
|
96
|
+
log.info(
|
|
97
|
+
{ accountConfigId },
|
|
98
|
+
"Cascade already complete (AccountConfig not found on replay) — no-op",
|
|
99
|
+
);
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
throw error;
|
|
103
|
+
}
|
|
104
|
+
const cascadeEntities = entities.filter(
|
|
105
|
+
(e) => e.entityType !== "AccountConfig",
|
|
106
|
+
);
|
|
107
|
+
await capabilities.cascadeDelete(cascadeEntities, log);
|
|
108
|
+
|
|
109
|
+
// Step 3: storage prefix cleanup. Runs AFTER the cascade so a mid-cascade
|
|
110
|
+
// replay always re-runs the (idempotent) storage step.
|
|
111
|
+
await capabilities.deleteStoragePrefix(`accounts/${accountConfigId}/`, log);
|
|
112
|
+
|
|
113
|
+
// Step 4: AccountConfig delete — the last write. The presence of an
|
|
114
|
+
// AccountConfig row with `deletedAt` set is the cascade-in-progress flag;
|
|
115
|
+
// removing the row is the only signal the cascade fully finished.
|
|
116
|
+
await services.accountConfigService.delete(accountConfigId);
|
|
117
|
+
log.info({ accountConfigId }, "AccountConfig deleted; cascade complete");
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Destructive phase of the per-account purge. Consumes the FIFO stream the
|
|
122
|
+
* fanout producer emits (#1069), one message group per account so the messages
|
|
123
|
+
* arrive strictly in order: every `subtrees` batch, then exactly one
|
|
124
|
+
* `container` leftover. There is NO self-re-enqueue — the producer enqueues all
|
|
125
|
+
* the work; this worker only ever consumes.
|
|
126
|
+
*
|
|
127
|
+
* Idempotency: the cascade on a missing key is a no-op, so a redelivered batch
|
|
128
|
+
* (or container) no-ops on already-gone rows. Search-index cleanup is not driven
|
|
129
|
+
* from here: on DynamoDB the fanout step enqueues a REMOVE per message id up
|
|
130
|
+
* front; on the relational backends the cascade emits a `message.removed` outbox
|
|
131
|
+
* row the search-index worker relays.
|
|
132
|
+
*/
|
|
133
|
+
export const processAccountDataPurgeFinalize = async (
|
|
134
|
+
event: AccountDataPurgeFinalizeEvent,
|
|
135
|
+
log: Logger,
|
|
136
|
+
deps: ProcessFinalizeDeps = {},
|
|
137
|
+
): Promise<void> => {
|
|
138
|
+
if (event.kind === "subtrees") {
|
|
139
|
+
await processPurgeSubtrees(event, log, deps);
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
await processPurgeContainer(event, log, deps);
|
|
143
|
+
};
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Delete a batch of message subtrees. For each `{ threadMessageId, messageId }`
|
|
147
|
+
* the manifest row is deleted UNCONDITIONALLY — there are ghost ThreadMessages
|
|
148
|
+
* whose Message is already gone — and the Message + its 9 child entities are
|
|
149
|
+
* deleted only when `describe()` resolves. A `describe()` 404 means the subtree
|
|
150
|
+
* is already gone: skip the children, still drop the manifest row.
|
|
151
|
+
*/
|
|
152
|
+
const processPurgeSubtrees = async (
|
|
153
|
+
event: Extract<AccountDataPurgeFinalizeEvent, { kind: "subtrees" }>,
|
|
154
|
+
log: Logger,
|
|
155
|
+
deps: ProcessFinalizeDeps,
|
|
156
|
+
): Promise<void> => {
|
|
157
|
+
const { accountId, accountConfigId, items } = event;
|
|
158
|
+
const { capabilities, services } = await resolveDeps(deps);
|
|
159
|
+
|
|
160
|
+
const entities: CascadeEntity[] = [];
|
|
161
|
+
for (const { threadMessageId, messageId } of items) {
|
|
162
|
+
entities.push({
|
|
163
|
+
entityType: "ThreadMessage",
|
|
164
|
+
key: { accountConfigId, threadMessageId },
|
|
165
|
+
});
|
|
166
|
+
try {
|
|
167
|
+
const messageData = await services.messageService.describe(messageId);
|
|
168
|
+
entities.push({ entityType: "Message", key: { messageId } });
|
|
169
|
+
collectMessageChildEntities(entities, messageData);
|
|
170
|
+
} catch (error) {
|
|
171
|
+
if ((error as { name?: string })?.name === "NotFoundError") {
|
|
172
|
+
log.info(
|
|
173
|
+
{ accountConfigId, accountId, messageId },
|
|
174
|
+
"Message subtree already gone — deleting manifest row only",
|
|
175
|
+
);
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
throw error;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
await capabilities.cascadeDelete(entities, log);
|
|
183
|
+
log.info(
|
|
184
|
+
{ accountConfigId, accountId, count: items.length },
|
|
185
|
+
"Per-account purge subtree batch deleted",
|
|
186
|
+
);
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* The container leftover — processed last in the FIFO group, after every subtree
|
|
191
|
+
* delete. Deletes the account-keyed container rows (mailboxes, outbox, locks),
|
|
192
|
+
* the storage prefix `accounts/{cfg}/{acct}/`, and invalidates the account's
|
|
193
|
+
* cached content. Ordering is invalidate-before-storage, storage-after-cascade.
|
|
194
|
+
* The tenant-shared Address rows, the AccountConfig, sibling accounts, and the
|
|
195
|
+
* soft-deleted account row (the purge-in-progress marker) are all kept.
|
|
196
|
+
*/
|
|
197
|
+
const processPurgeContainer = async (
|
|
198
|
+
event: Extract<AccountDataPurgeFinalizeEvent, { kind: "container" }>,
|
|
199
|
+
log: Logger,
|
|
200
|
+
deps: ProcessFinalizeDeps,
|
|
201
|
+
): Promise<void> => {
|
|
202
|
+
const { accountId, accountConfigId } = event;
|
|
203
|
+
const { capabilities, services } = await resolveDeps(deps);
|
|
204
|
+
|
|
205
|
+
// The account row is kept as the purge-in-progress marker. Its absence means
|
|
206
|
+
// the container leftover already ran — no-op on redelivery.
|
|
207
|
+
let accountDescription: Awaited<
|
|
208
|
+
ReturnType<CascadeServices["accountService"]["describe"]>
|
|
209
|
+
>;
|
|
210
|
+
try {
|
|
211
|
+
accountDescription = await services.accountService.describe(accountId);
|
|
212
|
+
} catch (error) {
|
|
213
|
+
if ((error as { name?: string })?.name === "NotFoundError") {
|
|
214
|
+
log.info(
|
|
215
|
+
{ accountConfigId, accountId },
|
|
216
|
+
"Per-account purge already complete (account row gone) — no-op",
|
|
217
|
+
);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
throw error;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Validate every deploy-time precondition BEFORE the first destructive step.
|
|
224
|
+
// Runs after the account-existence guard so a redelivery whose account row is
|
|
225
|
+
// already gone still returns cleanly without needing the config present.
|
|
226
|
+
await capabilities.assertReady(log);
|
|
227
|
+
|
|
228
|
+
// Step 1: content invalidation — first, so cached body parts cannot leak
|
|
229
|
+
// after the underlying objects are gone.
|
|
230
|
+
log.info(
|
|
231
|
+
{ accountConfigId, accountId },
|
|
232
|
+
"Invalidating cached content for purged account",
|
|
233
|
+
);
|
|
234
|
+
await capabilities.invalidateContent(`${accountConfigId}/${accountId}`, log);
|
|
235
|
+
|
|
236
|
+
// Step 2: container rows — mailboxes, outbox, locks. Address is
|
|
237
|
+
// tenant-shared and never deleted on a per-account purge.
|
|
238
|
+
const entities: CascadeEntity[] = [];
|
|
239
|
+
for (const mailbox of accountDescription.mailbox) {
|
|
240
|
+
entities.push({
|
|
241
|
+
entityType: "Mailbox",
|
|
242
|
+
key: { mailboxId: mailbox.mailboxId },
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
const outboxMessages =
|
|
246
|
+
await services.outboxMessageService.listByAccount(accountId);
|
|
247
|
+
for (const outbox of outboxMessages.items) {
|
|
248
|
+
entities.push({
|
|
249
|
+
entityType: "OutboxMessage",
|
|
250
|
+
key: { outboxMessageId: outbox.outboxMessageId },
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
const locks = await services.mailboxLockService.listByAccount(accountId);
|
|
254
|
+
for (const lock of locks) {
|
|
255
|
+
entities.push({
|
|
256
|
+
entityType: "MailboxLock",
|
|
257
|
+
key: { mailboxId: lock.mailboxId, eventName: lock.eventName },
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
await capabilities.cascadeDelete(entities, log);
|
|
261
|
+
|
|
262
|
+
// Step 3: storage prefix cleanup scoped to this account only.
|
|
263
|
+
await capabilities.deleteStoragePrefix(
|
|
264
|
+
`accounts/${accountConfigId}/${accountId}/`,
|
|
265
|
+
log,
|
|
266
|
+
);
|
|
267
|
+
|
|
268
|
+
log.info({ accountConfigId, accountId }, "Per-account data purge complete");
|
|
269
|
+
};
|
|
270
|
+
|
|
271
|
+
// ---------- SQS handler ----------
|
|
272
|
+
|
|
273
|
+
const log = createLogger();
|
|
274
|
+
|
|
275
|
+
export const handler: SQSHandler = withTelemetry(async (event: SQSEvent) => {
|
|
276
|
+
const batchItemFailures: { itemIdentifier: string }[] = [];
|
|
277
|
+
|
|
278
|
+
for (const record of event.Records) {
|
|
279
|
+
const finalizeEvent: AccountFinalizeEvent = JSON.parse(record.body);
|
|
280
|
+
log.info(
|
|
281
|
+
{
|
|
282
|
+
eventType: finalizeEvent.type,
|
|
283
|
+
accountConfigId: finalizeEvent.accountConfigId,
|
|
284
|
+
},
|
|
285
|
+
"Processing account finalize event",
|
|
286
|
+
);
|
|
287
|
+
|
|
288
|
+
const work =
|
|
289
|
+
finalizeEvent.type === "FinalizeAccountDataPurge"
|
|
290
|
+
? processAccountDataPurgeFinalize(finalizeEvent, log)
|
|
291
|
+
: processAccountFinalize(finalizeEvent, log);
|
|
292
|
+
|
|
293
|
+
const failed = await work
|
|
294
|
+
.then(() => false)
|
|
295
|
+
.catch((error) => {
|
|
296
|
+
log.error(
|
|
297
|
+
{ error: inspect(error), messageId: record.messageId },
|
|
298
|
+
"Account finalize event processing failed",
|
|
299
|
+
);
|
|
300
|
+
return true;
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
if (failed) {
|
|
304
|
+
batchItemFailures.push({ itemIdentifier: record.messageId });
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
return { batchItemFailures };
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
export const finalizeHandler: SQSHandler = handler;
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import {
|
|
3
|
+
SendMessageBatchCommand,
|
|
4
|
+
SendMessageCommand,
|
|
5
|
+
type SQSClient,
|
|
6
|
+
} from "@aws-sdk/client-sqs";
|
|
7
|
+
import { Message } from "@remit/electrodb-entities";
|
|
8
|
+
import type { Logger } from "@remit/logger-lambda";
|
|
9
|
+
import type { SearchIndexMessage } from "@remit/search-index-worker";
|
|
10
|
+
import { Entity } from "electrodb";
|
|
11
|
+
import type { CascadeServices } from "../cascade.js";
|
|
12
|
+
import {
|
|
13
|
+
cascadeServices as defaultCascadeServices,
|
|
14
|
+
dataBackend as defaultDataBackend,
|
|
15
|
+
sqsClient as defaultSqsClient,
|
|
16
|
+
getAccountPurgeDeleteQueueUrl,
|
|
17
|
+
getSearchIndexQueueUrl,
|
|
18
|
+
} from "../config.js";
|
|
19
|
+
import type {
|
|
20
|
+
AccountDataPurgeEvent,
|
|
21
|
+
AccountDataPurgeFinalizeEvent,
|
|
22
|
+
AccountDataPurgeSubtreeItem,
|
|
23
|
+
} from "../events.js";
|
|
24
|
+
|
|
25
|
+
export interface ProcessPurgeFanoutDeps {
|
|
26
|
+
services?: CascadeServices;
|
|
27
|
+
sqs?: SQSClient;
|
|
28
|
+
accountPurgeDeleteQueueUrl?: string;
|
|
29
|
+
searchIndexQueueUrl?: string;
|
|
30
|
+
dataBackend?: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const SQS_BATCH_SIZE = 10;
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Message subtrees carried per FIFO finalize message. Each item is two ids
|
|
37
|
+
* (~70 bytes of JSON), so 100 stays far under the 256 KB SQS limit, and the
|
|
38
|
+
* worker's per-item `describe()` + batched delete for 100 subtrees fits well
|
|
39
|
+
* inside its timeout.
|
|
40
|
+
*/
|
|
41
|
+
const SUBTREE_BATCH_SIZE = 100;
|
|
42
|
+
|
|
43
|
+
const messageEntity = new Entity(Message, { table: "purge-keys" });
|
|
44
|
+
|
|
45
|
+
const messageKeys = (messageId: string): { pk: string; sk: string } => {
|
|
46
|
+
const { Key } = messageEntity.delete({ messageId }).params<{
|
|
47
|
+
Key: { pk: string; sk: string };
|
|
48
|
+
}>();
|
|
49
|
+
return Key;
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const buildVectorDeleteEvent = (
|
|
53
|
+
accountId: string,
|
|
54
|
+
messageId: string,
|
|
55
|
+
): SearchIndexMessage => ({
|
|
56
|
+
eventName: "REMOVE",
|
|
57
|
+
entity: "Message",
|
|
58
|
+
eventID: randomUUID(),
|
|
59
|
+
eventTimestamp: Date.now(),
|
|
60
|
+
accountId,
|
|
61
|
+
keys: messageKeys(messageId),
|
|
62
|
+
messageId,
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
const enqueueVectorDeletes = async (
|
|
66
|
+
sqs: SQSClient,
|
|
67
|
+
queueUrl: string,
|
|
68
|
+
accountId: string,
|
|
69
|
+
messageIds: string[],
|
|
70
|
+
): Promise<void> => {
|
|
71
|
+
for (let i = 0; i < messageIds.length; i += SQS_BATCH_SIZE) {
|
|
72
|
+
const batch = messageIds.slice(i, i + SQS_BATCH_SIZE);
|
|
73
|
+
const result = await sqs.send(
|
|
74
|
+
new SendMessageBatchCommand({
|
|
75
|
+
QueueUrl: queueUrl,
|
|
76
|
+
Entries: batch.map((messageId, idx) => ({
|
|
77
|
+
Id: `${i + idx}`,
|
|
78
|
+
MessageBody: JSON.stringify(
|
|
79
|
+
buildVectorDeleteEvent(accountId, messageId),
|
|
80
|
+
),
|
|
81
|
+
})),
|
|
82
|
+
}),
|
|
83
|
+
);
|
|
84
|
+
// SendMessageBatch returns HTTP 200 with per-entry failures in `Failed`;
|
|
85
|
+
// throw so the failed deletes surface to the DLQ and the whole fanout
|
|
86
|
+
// replays idempotently rather than silently skipping vector deletes.
|
|
87
|
+
if (result.Failed?.length) {
|
|
88
|
+
const failedIds = result.Failed.map((f) => f.Id).join(", ");
|
|
89
|
+
throw new Error(
|
|
90
|
+
`Search-index vector-delete batch had ${result.Failed.length} failed entries: ${failedIds}`,
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Enqueue the destructive purge work onto the FIFO finalize queue, single
|
|
98
|
+
* message group per account so the worker processes everything strictly in
|
|
99
|
+
* order: every `subtrees` batch first, then exactly one `container` leftover
|
|
100
|
+
* last. The FIFO ordering is the barrier — the container delete (mailboxes,
|
|
101
|
+
* S3, CloudFront) cannot run until all subtree deletes have. The queue's
|
|
102
|
+
* content-based deduplication makes a fanout replay idempotent within the dedup
|
|
103
|
+
* window; beyond it, the worker's deletes are no-ops on already-gone rows.
|
|
104
|
+
*/
|
|
105
|
+
const enqueuePurgeFinalize = async (
|
|
106
|
+
sqs: SQSClient,
|
|
107
|
+
queueUrl: string,
|
|
108
|
+
accountId: string,
|
|
109
|
+
accountConfigId: string,
|
|
110
|
+
items: AccountDataPurgeSubtreeItem[],
|
|
111
|
+
): Promise<void> => {
|
|
112
|
+
const send = (event: AccountDataPurgeFinalizeEvent): Promise<unknown> =>
|
|
113
|
+
sqs.send(
|
|
114
|
+
new SendMessageCommand({
|
|
115
|
+
QueueUrl: queueUrl,
|
|
116
|
+
MessageBody: JSON.stringify(event),
|
|
117
|
+
MessageGroupId: accountConfigId,
|
|
118
|
+
}),
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
for (let i = 0; i < items.length; i += SUBTREE_BATCH_SIZE) {
|
|
122
|
+
await send({
|
|
123
|
+
type: "FinalizeAccountDataPurge",
|
|
124
|
+
kind: "subtrees",
|
|
125
|
+
accountId,
|
|
126
|
+
accountConfigId,
|
|
127
|
+
items: items.slice(i, i + SUBTREE_BATCH_SIZE),
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
await send({
|
|
132
|
+
type: "FinalizeAccountDataPurge",
|
|
133
|
+
kind: "container",
|
|
134
|
+
accountId,
|
|
135
|
+
accountConfigId,
|
|
136
|
+
});
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Fanout step of the per-account purge. Reads the account's ThreadMessage
|
|
141
|
+
* manifest (`pk = accountConfigId`, scoped to this account's mailbox set) —
|
|
142
|
+
* since the threading-visibility fix every persisted Message has exactly one
|
|
143
|
+
* manifest row, so the partition is a complete, describe-free index of the
|
|
144
|
+
* account's messages. Enqueues the per-message search-index (vector) deletes
|
|
145
|
+
* (#457), then hands the destructive DDB+S3+CloudFront work to the finalize
|
|
146
|
+
* worker as a stream of FIFO messages (subtree batches + one container
|
|
147
|
+
* leftover, single message group). No self-loop, no recursion: the producer
|
|
148
|
+
* enqueues all the work and the worker only ever consumes (#1069).
|
|
149
|
+
*
|
|
150
|
+
* Runs in the fanout worker. Scoped entirely to `accountId`; the AccountConfig,
|
|
151
|
+
* its sibling accounts, the tenant-shared Address rows, and the soft-deleted
|
|
152
|
+
* account row are untouched.
|
|
153
|
+
*
|
|
154
|
+
* Replay-safe: the manifest read is read-only and the search-index/finalize
|
|
155
|
+
* enqueues are idempotent. A redelivery after the account's rows are gone reads
|
|
156
|
+
* an empty manifest and enqueues only a container leftover that no-ops.
|
|
157
|
+
*/
|
|
158
|
+
export const processAccountDataPurge = async (
|
|
159
|
+
event: AccountDataPurgeEvent,
|
|
160
|
+
log: Logger,
|
|
161
|
+
deps: ProcessPurgeFanoutDeps = {},
|
|
162
|
+
): Promise<void> => {
|
|
163
|
+
const { accountId, accountConfigId } = event;
|
|
164
|
+
const services = deps.services ?? defaultCascadeServices;
|
|
165
|
+
const sqs = deps.sqs ?? defaultSqsClient;
|
|
166
|
+
const accountPurgeDeleteQueueUrl =
|
|
167
|
+
deps.accountPurgeDeleteQueueUrl ?? getAccountPurgeDeleteQueueUrl();
|
|
168
|
+
const searchIndexQueueUrl =
|
|
169
|
+
deps.searchIndexQueueUrl ?? getSearchIndexQueueUrl();
|
|
170
|
+
const dataBackend = deps.dataBackend ?? defaultDataBackend;
|
|
171
|
+
|
|
172
|
+
let mailboxIds: Set<string>;
|
|
173
|
+
try {
|
|
174
|
+
const account = await services.accountService.describe(accountId);
|
|
175
|
+
mailboxIds = new Set(account.mailbox.map((m) => m.mailboxId));
|
|
176
|
+
} catch (error) {
|
|
177
|
+
if ((error as { name?: string })?.name === "NotFoundError") {
|
|
178
|
+
log.info(
|
|
179
|
+
{ accountConfigId, accountId },
|
|
180
|
+
"Account purge fanout: account already gone — no-op",
|
|
181
|
+
);
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
throw error;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const manifest =
|
|
188
|
+
await services.threadMessageService.listAllByAccount(accountConfigId);
|
|
189
|
+
const items: AccountDataPurgeSubtreeItem[] = manifest
|
|
190
|
+
.filter((tm) => mailboxIds.has(tm.mailboxId))
|
|
191
|
+
.map((tm) => ({
|
|
192
|
+
threadMessageId: tm.threadMessageId,
|
|
193
|
+
messageId: tm.messageId,
|
|
194
|
+
}));
|
|
195
|
+
|
|
196
|
+
const messageIds = [...new Set(items.map((i) => i.messageId))];
|
|
197
|
+
|
|
198
|
+
// On Postgres the destructive delete emits a `message.removed` outbox row per
|
|
199
|
+
// message, which the pg-index worker relays as a search-index REMOVE; enqueuing
|
|
200
|
+
// here too would double the removal. On DynamoDB this fanout is the sole
|
|
201
|
+
// producer of the vector deletes (#457).
|
|
202
|
+
if (dataBackend !== "postgres") {
|
|
203
|
+
await enqueueVectorDeletes(sqs, searchIndexQueueUrl, accountId, messageIds);
|
|
204
|
+
log.info(
|
|
205
|
+
{ accountConfigId, accountId, count: messageIds.length },
|
|
206
|
+
"Enqueued search-index deletes for purged account",
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
await enqueuePurgeFinalize(
|
|
211
|
+
sqs,
|
|
212
|
+
accountPurgeDeleteQueueUrl,
|
|
213
|
+
accountId,
|
|
214
|
+
accountConfigId,
|
|
215
|
+
items,
|
|
216
|
+
);
|
|
217
|
+
log.info(
|
|
218
|
+
{
|
|
219
|
+
accountConfigId,
|
|
220
|
+
accountId,
|
|
221
|
+
subtreeCount: items.length,
|
|
222
|
+
batchCount: Math.ceil(items.length / SUBTREE_BATCH_SIZE),
|
|
223
|
+
},
|
|
224
|
+
"Enqueued per-account purge subtree batches + container leftover",
|
|
225
|
+
);
|
|
226
|
+
};
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { getClient } from "@remit/backend/client";
|
|
2
|
+
import {
|
|
3
|
+
applyOrganize,
|
|
4
|
+
buildOrganizeMatchDeps,
|
|
5
|
+
buildOrganizeMoveService,
|
|
6
|
+
matchOrganize,
|
|
7
|
+
ORGANIZE_MATCH_LIMIT,
|
|
8
|
+
predicateFromJob,
|
|
9
|
+
} from "@remit/backend/organize";
|
|
10
|
+
import type { Logger } from "@remit/logger-lambda";
|
|
11
|
+
import type { OrganizeJobEvent } from "../events.js";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Run a "all like these" back-apply job (RFC 034, #1278): match the corpus
|
|
15
|
+
* against the job's snapshotted predicate and apply the action to every match,
|
|
16
|
+
* in one pass, then record the counts. Mirrors the export job's lifecycle —
|
|
17
|
+
* Running, then Complete/Failed — on the same fanout seam.
|
|
18
|
+
*
|
|
19
|
+
* Reuses the shared matcher (so the applied set equals what preview returned)
|
|
20
|
+
* and the idempotent apply plumbing; `appliedByFilterId` is never written and no
|
|
21
|
+
* Filter/FilterAnchor row is ever created. Both back-apply actions run here: the
|
|
22
|
+
* additive label upsert and the exclusive folder move, the latter through the
|
|
23
|
+
* same local-first placement mover body sync uses (`buildOrganizeMoveService`),
|
|
24
|
+
* so a redelivered job re-applies both idempotently.
|
|
25
|
+
*/
|
|
26
|
+
export const processOrganizeJob = async (
|
|
27
|
+
event: OrganizeJobEvent,
|
|
28
|
+
log: Logger,
|
|
29
|
+
): Promise<void> => {
|
|
30
|
+
const { accountConfigId, organizeJobId } = event;
|
|
31
|
+
const client = await getClient();
|
|
32
|
+
|
|
33
|
+
const job = await client.organizeJobRequest.get(organizeJobId);
|
|
34
|
+
await client.organizeJobRequest.update(organizeJobId, { state: "Running" });
|
|
35
|
+
log.info(
|
|
36
|
+
{ accountConfigId, organizeJobId },
|
|
37
|
+
"Organize back-apply processing started",
|
|
38
|
+
);
|
|
39
|
+
|
|
40
|
+
try {
|
|
41
|
+
const predicate = predicateFromJob(job);
|
|
42
|
+
const messageIds = await matchOrganize(
|
|
43
|
+
buildOrganizeMatchDeps(client),
|
|
44
|
+
accountConfigId,
|
|
45
|
+
predicate,
|
|
46
|
+
ORGANIZE_MATCH_LIMIT,
|
|
47
|
+
);
|
|
48
|
+
const { applied, failed } = await applyOrganize(
|
|
49
|
+
{ client, moveService: buildOrganizeMoveService(client) },
|
|
50
|
+
accountConfigId,
|
|
51
|
+
messageIds,
|
|
52
|
+
predicate,
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
await client.organizeJobRequest.update(organizeJobId, {
|
|
56
|
+
state: "Complete",
|
|
57
|
+
matchedCount: messageIds.length,
|
|
58
|
+
appliedCount: applied,
|
|
59
|
+
failedCount: failed,
|
|
60
|
+
});
|
|
61
|
+
log.info(
|
|
62
|
+
{
|
|
63
|
+
accountConfigId,
|
|
64
|
+
organizeJobId,
|
|
65
|
+
matched: messageIds.length,
|
|
66
|
+
applied,
|
|
67
|
+
failed,
|
|
68
|
+
},
|
|
69
|
+
"Organize back-apply complete",
|
|
70
|
+
);
|
|
71
|
+
} catch (error) {
|
|
72
|
+
await client.organizeJobRequest.update(organizeJobId, {
|
|
73
|
+
state: "Failed",
|
|
74
|
+
errorMessage: error instanceof Error ? error.message : String(error),
|
|
75
|
+
});
|
|
76
|
+
throw error;
|
|
77
|
+
}
|
|
78
|
+
};
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export type {
|
|
2
|
+
AccountDataPurgeEvent,
|
|
3
|
+
AccountDataPurgeFinalizeEvent,
|
|
4
|
+
AccountDeleteEvent,
|
|
5
|
+
AccountDeleteFinalizeEvent,
|
|
6
|
+
AccountExportEvent,
|
|
7
|
+
AccountFanoutEvent,
|
|
8
|
+
AccountFinalizeEvent,
|
|
9
|
+
} from "./events.js";
|
|
10
|
+
export { handler as fanoutHandler } from "./handlers/account-fanout.js";
|
|
11
|
+
export { finalizeHandler } from "./handlers/account-finalize.js";
|
package/src/poller.ts
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { createLogger } from "@remit/logger-lambda";
|
|
2
|
+
import { runQueuePoller } from "@remit/sqs-client/poller";
|
|
3
|
+
import { env } from "expect-env";
|
|
4
|
+
import { fanoutHandler, finalizeHandler } from "./index.js";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Production queue poller. No e2e shim exists for account-worker today —
|
|
8
|
+
* the deletion cascade is not exercised on the Postgres/compose stack in
|
|
9
|
+
* CI (see AGENTS.md worker roster notes). This is the standalone
|
|
10
|
+
* production entrypoint for the dedicated image.
|
|
11
|
+
*
|
|
12
|
+
* The fanout worker's Cognito sign-out and the finalize worker's CloudFront
|
|
13
|
+
* invalidation are AWS-only calls with no portable counterpart yet — on a
|
|
14
|
+
* non-AWS deployment those specific steps of the deletion cascade will
|
|
15
|
+
* error if account deletion is exercised. That is a pre-existing gap in
|
|
16
|
+
* the application code, not something this packaging change fixes.
|
|
17
|
+
*/
|
|
18
|
+
const log = createLogger();
|
|
19
|
+
|
|
20
|
+
await runQueuePoller({
|
|
21
|
+
log,
|
|
22
|
+
targets: [
|
|
23
|
+
{
|
|
24
|
+
queueUrl: env.SQS_QUEUE_URL_ACCOUNT_FANOUT,
|
|
25
|
+
handler: fanoutHandler,
|
|
26
|
+
functionName: "account-fanout-worker",
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
queueUrl: env.SQS_QUEUE_URL_ACCOUNT_FINALIZE,
|
|
30
|
+
handler: finalizeHandler,
|
|
31
|
+
functionName: "account-finalize-worker",
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
queueUrl: env.SQS_QUEUE_URL_ACCOUNT_PURGE_DELETE,
|
|
35
|
+
handler: finalizeHandler,
|
|
36
|
+
functionName: "account-purge-worker",
|
|
37
|
+
},
|
|
38
|
+
],
|
|
39
|
+
});
|