@remit/account-worker 0.0.26 → 0.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/cascade-enumerate.test.ts +3 -53
- package/src/cascade.ts +2 -55
- package/src/compose-relational.test.ts +1 -14
- package/src/compose-relational.ts +5 -6
- package/src/config.test.ts +0 -5
- package/src/config.ts +1 -6
- package/src/deletion-capabilities.ts +2 -2
- package/src/events.ts +1 -1
- package/src/handlers/account-export.test.ts +1 -1
- package/src/handlers/account-export.ts +1 -1
- package/src/handlers/account-fanout.test.ts +1 -2
- package/src/handlers/account-fanout.ts +7 -6
- package/src/handlers/account-finalize.ts +6 -8
- package/src/handlers/account-purge.test.ts +4 -50
- package/src/handlers/account-purge.ts +7 -99
- package/src/handlers/organize-job.test.ts +74 -111
- package/src/handlers/organize-job.ts +9 -11
- package/src/handlers/sender-category-backapply.test.ts +1 -1
- package/src/handlers/sender-category-backapply.ts +1 -1
- package/src/poller.ts +5 -1
- package/src/cascade.test.ts +0 -77
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remit/account-worker",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.27",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
"dev": "node --import tsx src/poller.ts"
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
|
+
"@remit/logger": "*",
|
|
36
37
|
"@remit/data-ports": "*",
|
|
37
38
|
"@remit/logger-lambda": "*",
|
|
38
39
|
"@remit/drizzle-service": "*",
|
|
@@ -40,7 +41,6 @@
|
|
|
40
41
|
"@remit/sqs-client": "*",
|
|
41
42
|
"@remit/storage-service": "*",
|
|
42
43
|
"@remit/electrodb-entities": "*",
|
|
43
|
-
"@remit/search-index-worker": "*",
|
|
44
44
|
"@aws-sdk/client-sqs": "*",
|
|
45
45
|
"archiver": "*",
|
|
46
46
|
"electrodb": "*",
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { describe, it } from "node:test";
|
|
3
3
|
import type { MessageDescription } from "@remit/data-ports";
|
|
4
|
-
import { noopLogger } from "@remit/logger
|
|
4
|
+
import { noopLogger } from "@remit/logger";
|
|
5
5
|
import {
|
|
6
6
|
type CascadeEntity,
|
|
7
7
|
type CascadeServices,
|
|
@@ -18,11 +18,10 @@ const messageWithChildren = {
|
|
|
18
18
|
bodyPartParameter: [{ bodyPartParameterId: "bpp-1" }],
|
|
19
19
|
rawMessageStorage: [{ rawStorageId: "rms-1" }],
|
|
20
20
|
bodyPartStorage: [{ bodyPartStorageId: "bps-1" }],
|
|
21
|
-
bodyPartContent: [{ bodyPartContentId: "bpc-1" }],
|
|
22
21
|
} as unknown as MessageDescription;
|
|
23
22
|
|
|
24
23
|
describe("collectMessageChildEntities", () => {
|
|
25
|
-
it("expands the
|
|
24
|
+
it("expands the eight child entities of one message", () => {
|
|
26
25
|
const entities: CascadeEntity[] = [];
|
|
27
26
|
collectMessageChildEntities(entities, messageWithChildren);
|
|
28
27
|
|
|
@@ -36,13 +35,7 @@ describe("collectMessageChildEntities", () => {
|
|
|
36
35
|
"BodyPartParameter",
|
|
37
36
|
"RawMessageStorage",
|
|
38
37
|
"BodyPartStorage",
|
|
39
|
-
"BodyPartContent",
|
|
40
38
|
]);
|
|
41
|
-
assert.equal(
|
|
42
|
-
entities.find((e) => e.entityType === "BodyPartContent")?.key
|
|
43
|
-
.bodyPartContentId,
|
|
44
|
-
"bpc-1",
|
|
45
|
-
);
|
|
46
39
|
});
|
|
47
40
|
|
|
48
41
|
it("appends nothing when every child list is empty", () => {
|
|
@@ -56,7 +49,6 @@ describe("collectMessageChildEntities", () => {
|
|
|
56
49
|
bodyPartParameter: [],
|
|
57
50
|
rawMessageStorage: [],
|
|
58
51
|
bodyPartStorage: [],
|
|
59
|
-
bodyPartContent: [],
|
|
60
52
|
} as unknown as MessageDescription);
|
|
61
53
|
assert.equal(entities.length, 0);
|
|
62
54
|
});
|
|
@@ -96,13 +88,7 @@ const fullServices = (): CascadeServices =>
|
|
|
96
88
|
listByAccountConfig: async () => [{ accountSettingId: "set-1" }],
|
|
97
89
|
},
|
|
98
90
|
filterService: {
|
|
99
|
-
listByAccountConfig: async () => [
|
|
100
|
-
{ filterId: "flt-1", hasAnchor: true },
|
|
101
|
-
{ filterId: "flt-2", hasAnchor: false },
|
|
102
|
-
],
|
|
103
|
-
},
|
|
104
|
-
filterAnchorService: {
|
|
105
|
-
get: async () => ({ filterId: "flt-1" }),
|
|
91
|
+
listByAccountConfig: async () => [{ filterId: "flt-1" }],
|
|
106
92
|
},
|
|
107
93
|
labelService: {
|
|
108
94
|
listByAccountConfig: async () => [{ labelId: "lbl-1" }],
|
|
@@ -128,7 +114,6 @@ describe("enumerateCascadeEntities", () => {
|
|
|
128
114
|
"Mailbox",
|
|
129
115
|
"Message",
|
|
130
116
|
"MessageFlag",
|
|
131
|
-
"BodyPartContent",
|
|
132
117
|
"OutboxMessage",
|
|
133
118
|
"MailboxLock",
|
|
134
119
|
"MessagePlacementMove",
|
|
@@ -136,7 +121,6 @@ describe("enumerateCascadeEntities", () => {
|
|
|
136
121
|
"ThreadMessage",
|
|
137
122
|
"AccountSetting",
|
|
138
123
|
"Filter",
|
|
139
|
-
"FilterAnchor",
|
|
140
124
|
"Label",
|
|
141
125
|
"MessageLabel",
|
|
142
126
|
"Address",
|
|
@@ -148,40 +132,6 @@ describe("enumerateCascadeEntities", () => {
|
|
|
148
132
|
assert.equal(types[types.length - 1], "AccountConfig");
|
|
149
133
|
});
|
|
150
134
|
|
|
151
|
-
it("emits a Filter without a FilterAnchor when the filter has no anchor", async () => {
|
|
152
|
-
const services = fullServices();
|
|
153
|
-
services.filterService.listByAccountConfig = async () =>
|
|
154
|
-
[{ filterId: "flt-2", hasAnchor: false }] as never;
|
|
155
|
-
|
|
156
|
-
const { entities } = await enumerateCascadeEntities(
|
|
157
|
-
"cfg-1",
|
|
158
|
-
services,
|
|
159
|
-
noopLogger,
|
|
160
|
-
);
|
|
161
|
-
|
|
162
|
-
assert.equal(entities.filter((e) => e.entityType === "Filter").length, 1);
|
|
163
|
-
assert.equal(
|
|
164
|
-
entities.filter((e) => e.entityType === "FilterAnchor").length,
|
|
165
|
-
0,
|
|
166
|
-
);
|
|
167
|
-
});
|
|
168
|
-
|
|
169
|
-
it("skips the FilterAnchor when the anchor lookup returns null", async () => {
|
|
170
|
-
const services = fullServices();
|
|
171
|
-
services.filterAnchorService.get = async () => null as never;
|
|
172
|
-
|
|
173
|
-
const { entities } = await enumerateCascadeEntities(
|
|
174
|
-
"cfg-1",
|
|
175
|
-
services,
|
|
176
|
-
noopLogger,
|
|
177
|
-
);
|
|
178
|
-
|
|
179
|
-
assert.equal(
|
|
180
|
-
entities.filter((e) => e.entityType === "FilterAnchor").length,
|
|
181
|
-
0,
|
|
182
|
-
);
|
|
183
|
-
});
|
|
184
|
-
|
|
185
135
|
it("yields only the AccountConfig row for an empty tenant", async () => {
|
|
186
136
|
const services = fullServices();
|
|
187
137
|
services.accountConfigService.describe = async () =>
|
package/src/cascade.ts
CHANGED
|
@@ -5,7 +5,6 @@ import type {
|
|
|
5
5
|
IAccountSettingRepository,
|
|
6
6
|
IAddressRepository,
|
|
7
7
|
IEnvelopeRepository,
|
|
8
|
-
IFilterAnchorRepository,
|
|
9
8
|
IFilterRepository,
|
|
10
9
|
ILabelRepository,
|
|
11
10
|
IMailboxLockRepository,
|
|
@@ -19,7 +18,7 @@ import type {
|
|
|
19
18
|
IThreadMessageRepository,
|
|
20
19
|
MessageDescription,
|
|
21
20
|
} from "@remit/data-ports";
|
|
22
|
-
import type { Logger } from "@remit/logger
|
|
21
|
+
import type { Logger } from "@remit/logger";
|
|
23
22
|
|
|
24
23
|
export interface CascadeEntity {
|
|
25
24
|
entityType: string;
|
|
@@ -42,7 +41,6 @@ export interface CascadeServices {
|
|
|
42
41
|
accountExportRequestService: IAccountExportRequestRepository;
|
|
43
42
|
accountSettingService: IAccountSettingRepository;
|
|
44
43
|
filterService: IFilterRepository;
|
|
45
|
-
filterAnchorService: IFilterAnchorRepository;
|
|
46
44
|
labelService: ILabelRepository;
|
|
47
45
|
messageLabelService: IMessageLabelRepository;
|
|
48
46
|
}
|
|
@@ -111,12 +109,6 @@ export const collectMessageChildEntities = (
|
|
|
111
109
|
key: { bodyPartStorageId: bps.bodyPartStorageId },
|
|
112
110
|
});
|
|
113
111
|
}
|
|
114
|
-
for (const bpc of messageData.bodyPartContent) {
|
|
115
|
-
entities.push({
|
|
116
|
-
entityType: "BodyPartContent",
|
|
117
|
-
key: { bodyPartContentId: bpc.bodyPartContentId },
|
|
118
|
-
});
|
|
119
|
-
}
|
|
120
112
|
};
|
|
121
113
|
|
|
122
114
|
export const enumerateCascadeEntities = async (
|
|
@@ -138,7 +130,6 @@ export const enumerateCascadeEntities = async (
|
|
|
138
130
|
messageFlagPushService,
|
|
139
131
|
accountSettingService,
|
|
140
132
|
filterService,
|
|
141
|
-
filterAnchorService,
|
|
142
133
|
labelService,
|
|
143
134
|
messageLabelService,
|
|
144
135
|
} = services;
|
|
@@ -225,13 +216,6 @@ export const enumerateCascadeEntities = async (
|
|
|
225
216
|
key: { bodyPartStorageId: bps.bodyPartStorageId },
|
|
226
217
|
});
|
|
227
218
|
}
|
|
228
|
-
|
|
229
|
-
for (const bpc of messageData.bodyPartContent) {
|
|
230
|
-
entities.push({
|
|
231
|
-
entityType: "BodyPartContent",
|
|
232
|
-
key: { bodyPartContentId: bpc.bodyPartContentId },
|
|
233
|
-
});
|
|
234
|
-
}
|
|
235
219
|
}
|
|
236
220
|
}
|
|
237
221
|
|
|
@@ -292,7 +276,7 @@ export const enumerateCascadeEntities = async (
|
|
|
292
276
|
});
|
|
293
277
|
}
|
|
294
278
|
|
|
295
|
-
// Filter/
|
|
279
|
+
// Filter/Label/MessageLabel are accountConfig-scoped (RFC 034),
|
|
296
280
|
// not per sub-account, so — like AccountSetting and ThreadMessage above —
|
|
297
281
|
// they are enumerated once here rather than inside the per-account loop.
|
|
298
282
|
const filters = await filterService.listByAccountConfig(accountConfigId);
|
|
@@ -301,16 +285,6 @@ export const enumerateCascadeEntities = async (
|
|
|
301
285
|
entityType: "Filter",
|
|
302
286
|
key: { accountConfigId, filterId: filter.filterId },
|
|
303
287
|
});
|
|
304
|
-
if (!filter.hasAnchor) continue;
|
|
305
|
-
const anchor = await filterAnchorService.get(
|
|
306
|
-
accountConfigId,
|
|
307
|
-
filter.filterId,
|
|
308
|
-
);
|
|
309
|
-
if (!anchor) continue;
|
|
310
|
-
entities.push({
|
|
311
|
-
entityType: "FilterAnchor",
|
|
312
|
-
key: { accountConfigId, filterId: filter.filterId },
|
|
313
|
-
});
|
|
314
288
|
}
|
|
315
289
|
|
|
316
290
|
const labels = await labelService.listByAccountConfig(accountConfigId);
|
|
@@ -354,30 +328,3 @@ export const enumerateCascadeEntities = async (
|
|
|
354
328
|
|
|
355
329
|
return { entities, messageIds };
|
|
356
330
|
};
|
|
357
|
-
|
|
358
|
-
export const COVERED_ENTITY_TYPES = [
|
|
359
|
-
"Account",
|
|
360
|
-
"AccountConfig",
|
|
361
|
-
"AccountSetting",
|
|
362
|
-
"Address",
|
|
363
|
-
"BodyPart",
|
|
364
|
-
"BodyPartContent",
|
|
365
|
-
"BodyPartParameter",
|
|
366
|
-
"BodyPartStorage",
|
|
367
|
-
"Envelope",
|
|
368
|
-
"EnvelopeAddress",
|
|
369
|
-
"Filter",
|
|
370
|
-
"FilterAnchor",
|
|
371
|
-
"Label",
|
|
372
|
-
"Mailbox",
|
|
373
|
-
"MailboxLock",
|
|
374
|
-
"Message",
|
|
375
|
-
"MessageFlag",
|
|
376
|
-
"MessageFlagPush",
|
|
377
|
-
"MessageLabel",
|
|
378
|
-
"MessagePlacementMove",
|
|
379
|
-
"MessageReference",
|
|
380
|
-
"OutboxMessage",
|
|
381
|
-
"RawMessageStorage",
|
|
382
|
-
"ThreadMessage",
|
|
383
|
-
] as const;
|
|
@@ -3,7 +3,7 @@ import { mkdir, mkdtemp, stat, writeFile } from "node:fs/promises";
|
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { dirname, join } from "node:path";
|
|
5
5
|
import { afterEach, beforeEach, describe, it } from "node:test";
|
|
6
|
-
import { noopLogger } from "@remit/logger
|
|
6
|
+
import { noopLogger } from "@remit/logger";
|
|
7
7
|
import { buildRelationalDeletionCapabilities } from "./compose-relational.js";
|
|
8
8
|
|
|
9
9
|
const exists = (path: string): Promise<boolean> =>
|
|
@@ -79,16 +79,3 @@ describe("relational deletion capabilities — filesystem storage cleanup", () =
|
|
|
79
79
|
await caps.deleteStoragePrefix("accounts/never-existed/", noopLogger);
|
|
80
80
|
});
|
|
81
81
|
});
|
|
82
|
-
|
|
83
|
-
describe("relational deletion capabilities — CDN and sign-out are no-ops", () => {
|
|
84
|
-
it("invalidateContent resolves without a CDN", async () => {
|
|
85
|
-
const caps = buildRelationalDeletionCapabilities();
|
|
86
|
-
await caps.invalidateContent("cfg-1", noopLogger);
|
|
87
|
-
await caps.invalidateContent("cfg-1/acc-1", noopLogger);
|
|
88
|
-
});
|
|
89
|
-
|
|
90
|
-
it("signOut resolves without a federated session store", async () => {
|
|
91
|
-
const caps = buildRelationalDeletionCapabilities();
|
|
92
|
-
await caps.signOut("user-123", noopLogger);
|
|
93
|
-
});
|
|
94
|
-
});
|
|
@@ -3,9 +3,9 @@ import { join } from "node:path";
|
|
|
3
3
|
import {
|
|
4
4
|
type CascadeDeleter,
|
|
5
5
|
createSqliteCascadeDeleter,
|
|
6
|
+
sqliteDatabasePathsFromEnv,
|
|
6
7
|
} from "@remit/drizzle-service";
|
|
7
|
-
import type { Logger } from "@remit/logger
|
|
8
|
-
import { env } from "expect-env";
|
|
8
|
+
import type { Logger } from "@remit/logger";
|
|
9
9
|
import type { DeletionCapabilities } from "./deletion-capabilities.js";
|
|
10
10
|
|
|
11
11
|
// Mirrors `@remit/storage-service`'s filesystem-backend default: bodies
|
|
@@ -30,14 +30,14 @@ const deleteStoragePrefix = async (
|
|
|
30
30
|
};
|
|
31
31
|
|
|
32
32
|
// The Drizzle cascade deleter is opened once, on first use, and reused: it
|
|
33
|
-
// opens the
|
|
33
|
+
// opens the app, mail and search databases (the native binding loads there,
|
|
34
34
|
// hence async). Built lazily so the fanout worker — which signs out but never
|
|
35
35
|
// cascade-deletes — does not open a database handle it will not use.
|
|
36
36
|
let deleterPromise: Promise<CascadeDeleter> | undefined;
|
|
37
37
|
|
|
38
38
|
const getDeleter = (): Promise<CascadeDeleter> => {
|
|
39
39
|
if (!deleterPromise)
|
|
40
|
-
deleterPromise = createSqliteCascadeDeleter(
|
|
40
|
+
deleterPromise = createSqliteCascadeDeleter(sqliteDatabasePathsFromEnv());
|
|
41
41
|
return deleterPromise;
|
|
42
42
|
};
|
|
43
43
|
|
|
@@ -52,8 +52,7 @@ const getDeleter = (): Promise<CascadeDeleter> => {
|
|
|
52
52
|
* backend with no response cache (`deploy/vps/caddy/routes.caddy`), so there
|
|
53
53
|
* is nothing to invalidate.
|
|
54
54
|
* - `deleteStoragePrefix`: recursive filesystem removal.
|
|
55
|
-
* - `cascadeDelete`: the Drizzle cascade over SQLite.
|
|
56
|
-
* subtrees emit `message.removed` outbox rows the search-index worker relays.
|
|
55
|
+
* - `cascadeDelete`: the Drizzle cascade over SQLite.
|
|
57
56
|
*/
|
|
58
57
|
export const buildRelationalDeletionCapabilities =
|
|
59
58
|
(): DeletionCapabilities => ({
|
package/src/config.test.ts
CHANGED
|
@@ -51,9 +51,4 @@ describe("getImapWorkerQueueUrl", () => {
|
|
|
51
51
|
delete process.env.SQS_QUEUE_URL_IMAP_WORKER;
|
|
52
52
|
assert.equal(getImapWorkerQueueUrl(), undefined);
|
|
53
53
|
});
|
|
54
|
-
|
|
55
|
-
it("returns the configured value when set", () => {
|
|
56
|
-
process.env.SQS_QUEUE_URL_IMAP_WORKER = "http://queue/imap-worker";
|
|
57
|
-
assert.equal(getImapWorkerQueueUrl(), "http://queue/imap-worker");
|
|
58
|
-
});
|
|
59
54
|
});
|
package/src/config.ts
CHANGED
|
@@ -26,8 +26,6 @@ export const getSqsClient = (queueUrl: string): SQSClient => {
|
|
|
26
26
|
// no queue, so its Lambda doesn't carry the env vars. Eager evaluation here would
|
|
27
27
|
// crash finalize at module load — getters defer the read to the fanout-only call
|
|
28
28
|
// sites.
|
|
29
|
-
export const getSearchIndexQueueUrl = (): string =>
|
|
30
|
-
env.SQS_QUEUE_URL_SEARCH_INDEX;
|
|
31
29
|
// Optional, and read through `process.env` rather than expect-env so an unset
|
|
32
30
|
// var yields undefined instead of throwing. IMAP_WORKER_STOP is a no-op
|
|
33
31
|
// acknowledgement in the cascade contract — the account tombstone fence is what
|
|
@@ -48,7 +46,7 @@ let cascadeServicesPromise: Promise<CascadeServices> | null = null;
|
|
|
48
46
|
|
|
49
47
|
// Every cascade service is a `RemitClient` repository, so the whole enumeration
|
|
50
48
|
// runs on whatever backend `getClient()` resolved — DynamoDB (ElectroDB) or
|
|
51
|
-
// SQLite (Drizzle). Filter/
|
|
49
|
+
// SQLite (Drizzle). Filter/Label/MessageLabel (Smart Organize,
|
|
52
50
|
// RFC 034) are present on both via the client, so they need no
|
|
53
51
|
// backend-specific wiring here.
|
|
54
52
|
export const getCascadeServices = (): Promise<CascadeServices> => {
|
|
@@ -69,7 +67,6 @@ export const getCascadeServices = (): Promise<CascadeServices> => {
|
|
|
69
67
|
accountExportRequestService: remitClient.accountExportRequest,
|
|
70
68
|
accountSettingService: remitClient.accountSetting,
|
|
71
69
|
filterService: remitClient.filter,
|
|
72
|
-
filterAnchorService: remitClient.filterAnchor,
|
|
73
70
|
labelService: remitClient.label,
|
|
74
71
|
messageLabelService: remitClient.messageLabel,
|
|
75
72
|
}));
|
|
@@ -84,5 +81,3 @@ export const getStorageService = (): StorageService => {
|
|
|
84
81
|
}
|
|
85
82
|
return storageService;
|
|
86
83
|
};
|
|
87
|
-
|
|
88
|
-
export const dataBackend = process.env.DATA_BACKEND;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Logger } from "@remit/logger
|
|
1
|
+
import type { Logger } from "@remit/logger";
|
|
2
2
|
import type { CascadeEntity } from "./cascade.js";
|
|
3
3
|
|
|
4
4
|
/**
|
|
@@ -71,7 +71,7 @@ export const setDeletionCapabilities = (
|
|
|
71
71
|
const buildDeletionCapabilitiesFromEnv =
|
|
72
72
|
async (): Promise<DeletionCapabilities> => {
|
|
73
73
|
if (injectedCapabilities) return injectedCapabilities;
|
|
74
|
-
if (process.env.
|
|
74
|
+
if (process.env.SQLITE_APP_DB_PATH) {
|
|
75
75
|
const { buildRelationalDeletionCapabilities } = await import(
|
|
76
76
|
"./compose-relational.js"
|
|
77
77
|
);
|
package/src/events.ts
CHANGED
|
@@ -9,7 +9,7 @@ export type AccountDeleteEvent = {
|
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
11
|
* Per-account hard-delete. Purges ONE mail account's data (mailboxes,
|
|
12
|
-
* messages, body parts, thread messages, S3 objects
|
|
12
|
+
* messages, body parts, thread messages, S3 objects) while
|
|
13
13
|
* keeping the AccountConfig/tenant and any sibling accounts intact. Emitted
|
|
14
14
|
* by the API on `deleteAccount`; consumed by the fanout worker.
|
|
15
15
|
*/
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { Readable } from "node:stream";
|
|
3
3
|
import { describe, it } from "node:test";
|
|
4
|
-
import { noopLogger } from "@remit/logger
|
|
4
|
+
import { noopLogger } from "@remit/logger";
|
|
5
5
|
import type { StorageService } from "@remit/storage-service";
|
|
6
6
|
import type { CascadeServices } from "../cascade.js";
|
|
7
7
|
import type { AccountExportEvent } from "../events.js";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { PassThrough } from "node:stream";
|
|
2
|
-
import type { Logger } from "@remit/logger
|
|
2
|
+
import type { Logger } from "@remit/logger";
|
|
3
3
|
import type { StorageService } from "@remit/storage-service";
|
|
4
4
|
import { ZipArchive } from "archiver";
|
|
5
5
|
import type { CascadeServices } from "../cascade.js";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { afterEach, describe, it } from "node:test";
|
|
3
3
|
import type { SQSClient } from "@aws-sdk/client-sqs";
|
|
4
|
-
import { noopLogger } from "@remit/logger
|
|
4
|
+
import { noopLogger } from "@remit/logger";
|
|
5
5
|
import type { CascadeServices } from "../cascade.js";
|
|
6
6
|
import type { AccountDeleteEvent } from "../events.js";
|
|
7
7
|
import { processAccountFanout } from "./account-fanout.js";
|
|
@@ -32,7 +32,6 @@ const buildServices = (): CascadeServices =>
|
|
|
32
32
|
threadMessageService: { listAllByAccount: async () => [] },
|
|
33
33
|
accountSettingService: { listByAccountConfig: async () => [] },
|
|
34
34
|
filterService: { listByAccountConfig: async () => [] },
|
|
35
|
-
filterAnchorService: { get: async () => null },
|
|
36
35
|
labelService: { listByAccountConfig: async () => [] },
|
|
37
36
|
messageLabelService: { listByLabelId: async () => [] },
|
|
38
37
|
}) as unknown as CascadeServices;
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
|
|
2
|
+
import { createLogger, type Logger } from "@remit/logger";
|
|
2
3
|
import {
|
|
3
|
-
createLogger,
|
|
4
|
-
type Logger,
|
|
5
4
|
queueNameFromEventSource,
|
|
6
5
|
recordQueueEvent,
|
|
7
|
-
|
|
8
|
-
} from "@remit/logger-lambda";
|
|
6
|
+
} from "@remit/logger-lambda/metrics";
|
|
7
|
+
import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
|
|
9
8
|
import type { SQSBatchResponse, SQSEvent, SQSHandler } from "aws-lambda";
|
|
10
9
|
import type { CascadeServices } from "../cascade.js";
|
|
11
10
|
import { enumerateCascadeEntities } from "../cascade.js";
|
|
@@ -30,6 +29,8 @@ import { processAccountDataPurge } from "./account-purge.js";
|
|
|
30
29
|
import { processOrganizeJob } from "./organize-job.js";
|
|
31
30
|
import { processSenderCategoryBackApply } from "./sender-category-backapply.js";
|
|
32
31
|
|
|
32
|
+
installNodeLogger();
|
|
33
|
+
|
|
33
34
|
export interface ProcessAccountFanoutDeps {
|
|
34
35
|
services?: CascadeServices;
|
|
35
36
|
sqs?: SQSClient;
|
|
@@ -41,8 +42,8 @@ export interface ProcessAccountFanoutDeps {
|
|
|
41
42
|
|
|
42
43
|
/**
|
|
43
44
|
* Fanout step of the account-deletion cascade. Read-only: enumerates the
|
|
44
|
-
* AccountConfig's data and dispatches downstream work —
|
|
45
|
-
*
|
|
45
|
+
* AccountConfig's data and dispatches downstream work — per-account
|
|
46
|
+
* imap-worker stops, a sign-out, and the finalize
|
|
46
47
|
* enqueue. Throws on any non-swallowable error so SQS partial batch-failure
|
|
47
48
|
* retries the whole record.
|
|
48
49
|
*/
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { inspect } from "node:util";
|
|
2
|
+
import { createLogger, type Logger } from "@remit/logger";
|
|
2
3
|
import {
|
|
3
|
-
createLogger,
|
|
4
|
-
type Logger,
|
|
5
4
|
queueNameFromEventSource,
|
|
6
5
|
recordQueueEvent,
|
|
7
|
-
|
|
8
|
-
} from "@remit/logger-lambda";
|
|
6
|
+
} from "@remit/logger-lambda/metrics";
|
|
7
|
+
import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
|
|
9
8
|
import type { SQSEvent, SQSHandler } from "aws-lambda";
|
|
10
9
|
import {
|
|
11
10
|
type CascadeEntity,
|
|
@@ -24,6 +23,8 @@ import type {
|
|
|
24
23
|
AccountFinalizeEvent,
|
|
25
24
|
} from "../events.js";
|
|
26
25
|
|
|
26
|
+
installNodeLogger();
|
|
27
|
+
|
|
27
28
|
export interface ProcessFinalizeDeps {
|
|
28
29
|
capabilities?: DeletionCapabilities;
|
|
29
30
|
cascadeServices?: CascadeServices;
|
|
@@ -131,10 +132,7 @@ export const processAccountFinalize = async (
|
|
|
131
132
|
* the work; this worker only ever consumes.
|
|
132
133
|
*
|
|
133
134
|
* Idempotency: the cascade on a missing key is a no-op, so a redelivered batch
|
|
134
|
-
* (or container) no-ops on already-gone rows.
|
|
135
|
-
* from here: on DynamoDB the fanout step enqueues a REMOVE per message id up
|
|
136
|
-
* front; on the relational backends the cascade emits a `message.removed` outbox
|
|
137
|
-
* row the search-index worker relays.
|
|
135
|
+
* (or container) no-ops on already-gone rows.
|
|
138
136
|
*/
|
|
139
137
|
export const processAccountDataPurgeFinalize = async (
|
|
140
138
|
event: AccountDataPurgeFinalizeEvent,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { describe, it } from "node:test";
|
|
3
3
|
import type { SQSClient } from "@aws-sdk/client-sqs";
|
|
4
|
-
import { noopLogger } from "@remit/logger
|
|
4
|
+
import { noopLogger } from "@remit/logger";
|
|
5
5
|
import type { CascadeServices } from "../cascade.js";
|
|
6
6
|
import type { AccountDataPurgeEvent } from "../events.js";
|
|
7
7
|
import {
|
|
@@ -12,28 +12,19 @@ import {
|
|
|
12
12
|
interface Sent {
|
|
13
13
|
queueUrl: string | undefined;
|
|
14
14
|
body: Record<string, unknown>;
|
|
15
|
-
entries?: Array<{ Id: string; MessageBody: string }>;
|
|
16
15
|
}
|
|
17
16
|
|
|
18
|
-
const recordingSqs = (sent: Sent[]
|
|
17
|
+
const recordingSqs = (sent: Sent[]): SQSClient =>
|
|
19
18
|
({
|
|
20
19
|
send: async (command: {
|
|
21
|
-
input: {
|
|
22
|
-
QueueUrl?: string;
|
|
23
|
-
MessageBody?: string;
|
|
24
|
-
Entries?: Array<{ Id: string; MessageBody: string }>;
|
|
25
|
-
};
|
|
20
|
+
input: { QueueUrl?: string; MessageBody?: string };
|
|
26
21
|
}) => {
|
|
27
22
|
sent.push({
|
|
28
23
|
queueUrl: command.input.QueueUrl,
|
|
29
24
|
body: command.input.MessageBody
|
|
30
25
|
? JSON.parse(command.input.MessageBody)
|
|
31
26
|
: {},
|
|
32
|
-
entries: command.input.Entries,
|
|
33
27
|
});
|
|
34
|
-
if (command.input.Entries && failBatch) {
|
|
35
|
-
return { Failed: [{ Id: "0" }] };
|
|
36
|
-
}
|
|
37
28
|
return {};
|
|
38
29
|
},
|
|
39
30
|
}) as unknown as SQSClient;
|
|
@@ -73,19 +64,14 @@ const baseDeps = (
|
|
|
73
64
|
]),
|
|
74
65
|
sqs: recordingSqs(sent),
|
|
75
66
|
accountPurgeDeleteQueueUrl: "http://queue/purge",
|
|
76
|
-
searchIndexQueueUrl: "http://queue/search",
|
|
77
|
-
dataBackend: "dynamodb",
|
|
78
67
|
...overrides,
|
|
79
68
|
});
|
|
80
69
|
|
|
81
70
|
describe("processAccountDataPurge", () => {
|
|
82
|
-
it("enqueues
|
|
71
|
+
it("enqueues a subtrees batch and one container leftover", async () => {
|
|
83
72
|
const sent: Sent[] = [];
|
|
84
73
|
await processAccountDataPurge(event, noopLogger, baseDeps(sent));
|
|
85
74
|
|
|
86
|
-
const vectorBatch = sent.find((m) => m.queueUrl === "http://queue/search");
|
|
87
|
-
assert.equal(vectorBatch?.entries?.length, 2, "one entry per message");
|
|
88
|
-
|
|
89
75
|
const finalize = sent.filter((m) => m.queueUrl === "http://queue/purge");
|
|
90
76
|
const kinds = finalize.map((m) => m.body.kind);
|
|
91
77
|
assert.deepEqual(kinds, ["subtrees", "container"]);
|
|
@@ -96,27 +82,6 @@ describe("processAccountDataPurge", () => {
|
|
|
96
82
|
);
|
|
97
83
|
});
|
|
98
84
|
|
|
99
|
-
it("skips vector deletes on the relational backend, still enqueuing finalize", async () => {
|
|
100
|
-
const sent: Sent[] = [];
|
|
101
|
-
await processAccountDataPurge(
|
|
102
|
-
event,
|
|
103
|
-
noopLogger,
|
|
104
|
-
baseDeps(sent, { dataBackend: "sqlite" }),
|
|
105
|
-
);
|
|
106
|
-
|
|
107
|
-
assert.equal(
|
|
108
|
-
sent.some((m) => m.queueUrl === "http://queue/search"),
|
|
109
|
-
false,
|
|
110
|
-
"no search-index enqueue on the relational backend",
|
|
111
|
-
);
|
|
112
|
-
assert.deepEqual(
|
|
113
|
-
sent
|
|
114
|
-
.filter((m) => m.queueUrl === "http://queue/purge")
|
|
115
|
-
.map((m) => m.body.kind),
|
|
116
|
-
["subtrees", "container"],
|
|
117
|
-
);
|
|
118
|
-
});
|
|
119
|
-
|
|
120
85
|
it("drops manifest rows outside the account's mailbox set", async () => {
|
|
121
86
|
const sent: Sent[] = [];
|
|
122
87
|
await processAccountDataPurge(event, noopLogger, {
|
|
@@ -167,15 +132,4 @@ describe("processAccountDataPurge", () => {
|
|
|
167
132
|
/dynamo throttled/,
|
|
168
133
|
);
|
|
169
134
|
});
|
|
170
|
-
|
|
171
|
-
it("throws when a vector-delete batch reports failed entries", async () => {
|
|
172
|
-
const sent: Sent[] = [];
|
|
173
|
-
await assert.rejects(
|
|
174
|
-
processAccountDataPurge(event, noopLogger, {
|
|
175
|
-
...baseDeps(sent),
|
|
176
|
-
sqs: recordingSqs(sent, true),
|
|
177
|
-
}),
|
|
178
|
-
/failed entries/,
|
|
179
|
-
);
|
|
180
|
-
});
|
|
181
135
|
});
|
|
@@ -1,19 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
SendMessageBatchCommand,
|
|
4
|
-
SendMessageCommand,
|
|
5
|
-
type SQSClient,
|
|
6
|
-
} from "@aws-sdk/client-sqs";
|
|
7
|
-
import { Message } from "@remit/electrodb-entities";
|
|
8
|
-
import type { Logger } from "@remit/logger-lambda";
|
|
9
|
-
import type { SearchIndexMessage } from "@remit/search-index-worker";
|
|
10
|
-
import { Entity } from "electrodb";
|
|
1
|
+
import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
|
|
2
|
+
import type { Logger } from "@remit/logger";
|
|
11
3
|
import type { CascadeServices } from "../cascade.js";
|
|
12
4
|
import {
|
|
13
|
-
dataBackend as defaultDataBackend,
|
|
14
5
|
getAccountPurgeDeleteQueueUrl,
|
|
15
6
|
getCascadeServices,
|
|
16
|
-
getSearchIndexQueueUrl,
|
|
17
7
|
getSqsClient,
|
|
18
8
|
} from "../config.js";
|
|
19
9
|
import type {
|
|
@@ -26,12 +16,8 @@ export interface ProcessPurgeFanoutDeps {
|
|
|
26
16
|
services?: CascadeServices;
|
|
27
17
|
sqs?: SQSClient;
|
|
28
18
|
accountPurgeDeleteQueueUrl?: string;
|
|
29
|
-
searchIndexQueueUrl?: string;
|
|
30
|
-
dataBackend?: string;
|
|
31
19
|
}
|
|
32
20
|
|
|
33
|
-
const SQS_BATCH_SIZE = 10;
|
|
34
|
-
|
|
35
21
|
/**
|
|
36
22
|
* Message subtrees carried per FIFO finalize message. Each item is two ids
|
|
37
23
|
* (~70 bytes of JSON), so 100 stays far under the 256 KB SQS limit, and the
|
|
@@ -40,59 +26,6 @@ const SQS_BATCH_SIZE = 10;
|
|
|
40
26
|
*/
|
|
41
27
|
const SUBTREE_BATCH_SIZE = 100;
|
|
42
28
|
|
|
43
|
-
const messageEntity = new Entity(Message, { table: "purge-keys" });
|
|
44
|
-
|
|
45
|
-
const messageKeys = (messageId: string): { pk: string; sk: string } => {
|
|
46
|
-
const { Key } = messageEntity.delete({ messageId }).params<{
|
|
47
|
-
Key: { pk: string; sk: string };
|
|
48
|
-
}>();
|
|
49
|
-
return Key;
|
|
50
|
-
};
|
|
51
|
-
|
|
52
|
-
const buildVectorDeleteEvent = (
|
|
53
|
-
accountId: string,
|
|
54
|
-
messageId: string,
|
|
55
|
-
): SearchIndexMessage => ({
|
|
56
|
-
eventName: "REMOVE",
|
|
57
|
-
entity: "Message",
|
|
58
|
-
eventID: randomUUID(),
|
|
59
|
-
eventTimestamp: Date.now(),
|
|
60
|
-
accountId,
|
|
61
|
-
keys: messageKeys(messageId),
|
|
62
|
-
messageId,
|
|
63
|
-
});
|
|
64
|
-
|
|
65
|
-
const enqueueVectorDeletes = async (
|
|
66
|
-
sqs: SQSClient,
|
|
67
|
-
queueUrl: string,
|
|
68
|
-
accountId: string,
|
|
69
|
-
messageIds: string[],
|
|
70
|
-
): Promise<void> => {
|
|
71
|
-
for (let i = 0; i < messageIds.length; i += SQS_BATCH_SIZE) {
|
|
72
|
-
const batch = messageIds.slice(i, i + SQS_BATCH_SIZE);
|
|
73
|
-
const result = await sqs.send(
|
|
74
|
-
new SendMessageBatchCommand({
|
|
75
|
-
QueueUrl: queueUrl,
|
|
76
|
-
Entries: batch.map((messageId, idx) => ({
|
|
77
|
-
Id: `${i + idx}`,
|
|
78
|
-
MessageBody: JSON.stringify(
|
|
79
|
-
buildVectorDeleteEvent(accountId, messageId),
|
|
80
|
-
),
|
|
81
|
-
})),
|
|
82
|
-
}),
|
|
83
|
-
);
|
|
84
|
-
// SendMessageBatch returns HTTP 200 with per-entry failures in `Failed`;
|
|
85
|
-
// throw so the failed deletes surface to the DLQ and the whole fanout
|
|
86
|
-
// replays idempotently rather than silently skipping vector deletes.
|
|
87
|
-
if (result.Failed?.length) {
|
|
88
|
-
const failedIds = result.Failed.map((f) => f.Id).join(", ");
|
|
89
|
-
throw new Error(
|
|
90
|
-
`Search-index vector-delete batch had ${result.Failed.length} failed entries: ${failedIds}`,
|
|
91
|
-
);
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
};
|
|
95
|
-
|
|
96
29
|
/**
|
|
97
30
|
* Enqueue the destructive purge work onto the FIFO finalize queue, single
|
|
98
31
|
* message group per account so the worker processes everything strictly in
|
|
@@ -141,9 +74,8 @@ const enqueuePurgeFinalize = async (
|
|
|
141
74
|
* manifest (`pk = accountConfigId`, scoped to this account's mailbox set) —
|
|
142
75
|
* since the threading-visibility fix every persisted Message has exactly one
|
|
143
76
|
* manifest row, so the partition is a complete, describe-free index of the
|
|
144
|
-
* account's messages.
|
|
145
|
-
*
|
|
146
|
-
* worker as a stream of FIFO messages (subtree batches + one container
|
|
77
|
+
* account's messages. Hands the destructive DDB+S3+CloudFront work to the
|
|
78
|
+
* finalize worker as a stream of FIFO messages (subtree batches + one container
|
|
147
79
|
* leftover, single message group). No self-loop, no recursion: the producer
|
|
148
80
|
* enqueues all the work and the worker only ever consumes (#1069).
|
|
149
81
|
*
|
|
@@ -151,8 +83,8 @@ const enqueuePurgeFinalize = async (
|
|
|
151
83
|
* its sibling accounts, the tenant-shared Address rows, and the soft-deleted
|
|
152
84
|
* account row are untouched.
|
|
153
85
|
*
|
|
154
|
-
* Replay-safe: the manifest read is read-only and the
|
|
155
|
-
*
|
|
86
|
+
* Replay-safe: the manifest read is read-only and the finalize enqueues are
|
|
87
|
+
* idempotent. A redelivery after the account's rows are gone reads
|
|
156
88
|
* an empty manifest and enqueues only a container leftover that no-ops.
|
|
157
89
|
*/
|
|
158
90
|
export const processAccountDataPurge = async (
|
|
@@ -162,13 +94,8 @@ export const processAccountDataPurge = async (
|
|
|
162
94
|
): Promise<void> => {
|
|
163
95
|
const { accountId, accountConfigId } = event;
|
|
164
96
|
const services = deps.services ?? (await getCascadeServices());
|
|
165
|
-
const sqsFor = (queueUrl: string): SQSClient =>
|
|
166
|
-
deps.sqs ?? getSqsClient(queueUrl);
|
|
167
97
|
const accountPurgeDeleteQueueUrl =
|
|
168
98
|
deps.accountPurgeDeleteQueueUrl ?? getAccountPurgeDeleteQueueUrl();
|
|
169
|
-
const searchIndexQueueUrl =
|
|
170
|
-
deps.searchIndexQueueUrl ?? getSearchIndexQueueUrl();
|
|
171
|
-
const dataBackend = deps.dataBackend ?? defaultDataBackend;
|
|
172
99
|
|
|
173
100
|
let mailboxIds: Set<string>;
|
|
174
101
|
try {
|
|
@@ -194,27 +121,8 @@ export const processAccountDataPurge = async (
|
|
|
194
121
|
messageId: tm.messageId,
|
|
195
122
|
}));
|
|
196
123
|
|
|
197
|
-
const messageIds = [...new Set(items.map((i) => i.messageId))];
|
|
198
|
-
|
|
199
|
-
// On the relational backend the destructive delete emits a `message.removed`
|
|
200
|
-
// outbox row per message, which the search-index worker relays as a
|
|
201
|
-
// search-index REMOVE; enqueuing here too would double the removal. On
|
|
202
|
-
// DynamoDB this fanout is the sole producer of the vector deletes (#457).
|
|
203
|
-
if (dataBackend !== "sqlite") {
|
|
204
|
-
await enqueueVectorDeletes(
|
|
205
|
-
sqsFor(searchIndexQueueUrl),
|
|
206
|
-
searchIndexQueueUrl,
|
|
207
|
-
accountId,
|
|
208
|
-
messageIds,
|
|
209
|
-
);
|
|
210
|
-
log.info(
|
|
211
|
-
{ accountConfigId, accountId, count: messageIds.length },
|
|
212
|
-
"Enqueued search-index deletes for purged account",
|
|
213
|
-
);
|
|
214
|
-
}
|
|
215
|
-
|
|
216
124
|
await enqueuePurgeFinalize(
|
|
217
|
-
|
|
125
|
+
deps.sqs ?? getSqsClient(accountPurgeDeleteQueueUrl),
|
|
218
126
|
accountPurgeDeleteQueueUrl,
|
|
219
127
|
accountId,
|
|
220
128
|
accountConfigId,
|
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
|
-
import { readFileSync } from "node:fs";
|
|
4
3
|
import { afterEach, describe, it } from "node:test";
|
|
5
4
|
import type { RemitClient } from "@remit/backend/client";
|
|
6
5
|
import {
|
|
7
6
|
buildOrganizeMatchDeps,
|
|
8
7
|
type OrganizeMatchDeps,
|
|
9
|
-
type OrganizeSemanticDeps,
|
|
10
8
|
} from "@remit/backend/organize";
|
|
11
|
-
import type {
|
|
9
|
+
import type { FilterItem } from "@remit/data-ports";
|
|
12
10
|
import { NotFoundError } from "@remit/data-ports/errors";
|
|
13
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
DrizzleThreadMessageRepository,
|
|
13
|
+
SearchIndex,
|
|
14
|
+
} from "@remit/drizzle-service";
|
|
14
15
|
import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
|
|
15
|
-
import { noopLogger } from "@remit/logger
|
|
16
|
+
import { noopLogger } from "@remit/logger";
|
|
16
17
|
import type { OrganizeJobEvent } from "../events.js";
|
|
17
18
|
import {
|
|
18
19
|
type ProcessOrganizeJobDeps,
|
|
@@ -43,10 +44,8 @@ const jobClient = (
|
|
|
43
44
|
organizeJobId: "job-1",
|
|
44
45
|
accountConfigId: "cfg-1",
|
|
45
46
|
filterId: "None",
|
|
46
|
-
anchorMessageId: "None",
|
|
47
47
|
matchOperator: "And",
|
|
48
48
|
literalClauses,
|
|
49
|
-
similarityThreshold: 0.75,
|
|
50
49
|
actionLabelId: "lbl-1",
|
|
51
50
|
actionMailboxId: "None",
|
|
52
51
|
}),
|
|
@@ -63,20 +62,10 @@ const jobClient = (
|
|
|
63
62
|
*/
|
|
64
63
|
const explodingMatchDeps = (): OrganizeMatchDeps =>
|
|
65
64
|
({
|
|
66
|
-
semantic: () => {
|
|
67
|
-
throw new Error("the vector pipeline must not be reached");
|
|
68
|
-
},
|
|
69
65
|
listAccountFilterMessages: async () => {
|
|
70
66
|
throw new Error("SQLITE_BUSY");
|
|
71
67
|
},
|
|
72
|
-
|
|
73
|
-
get: async () => null,
|
|
74
|
-
listByAccountConfig: async () => [],
|
|
75
|
-
put: async () => {
|
|
76
|
-
throw new Error("unreachable");
|
|
77
|
-
},
|
|
78
|
-
},
|
|
79
|
-
}) as unknown as OrganizeMatchDeps;
|
|
68
|
+
}) as OrganizeMatchDeps;
|
|
80
69
|
|
|
81
70
|
describe("processOrganizeJob (reader #463)", () => {
|
|
82
71
|
it("fails the job on a refused rule and acknowledges the record", async () => {
|
|
@@ -136,7 +125,6 @@ const standingFilter = (over: Partial<FilterItem> = {}): FilterItem => ({
|
|
|
136
125
|
scope: "Standing",
|
|
137
126
|
state: "Active",
|
|
138
127
|
disabledReason: "None",
|
|
139
|
-
hasAnchor: false,
|
|
140
128
|
ruleChangedAt: 1,
|
|
141
129
|
actionChangedAt: 1,
|
|
142
130
|
matchOperator: "And",
|
|
@@ -187,28 +175,21 @@ interface World {
|
|
|
187
175
|
inbox: string;
|
|
188
176
|
archive: string;
|
|
189
177
|
updates: Update[];
|
|
178
|
+
filterUpdates: Array<{ filterId: string; patch: Record<string, unknown> }>;
|
|
190
179
|
moves: Move[];
|
|
191
180
|
labeled: string[];
|
|
192
181
|
close: () => void;
|
|
193
182
|
}
|
|
194
183
|
|
|
195
|
-
const world = async (
|
|
196
|
-
filters: FilterItem[],
|
|
197
|
-
anchors: FilterAnchorItem[] = [],
|
|
198
|
-
): Promise<World> => {
|
|
184
|
+
const world = async (filters: FilterItem[]): Promise<World> => {
|
|
199
185
|
const store = createShippedSqliteDb();
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
"../../../../npm-scripts/sqlite-search-index.sql",
|
|
204
|
-
import.meta.url,
|
|
205
|
-
),
|
|
206
|
-
"utf8",
|
|
207
|
-
),
|
|
186
|
+
const threadMessage = new DrizzleThreadMessageRepository(
|
|
187
|
+
store.db as never,
|
|
188
|
+
new SearchIndex(store.db as never, store.db as never),
|
|
208
189
|
);
|
|
209
|
-
const threadMessage = new DrizzleThreadMessageRepository(store.db as never);
|
|
210
190
|
const { inbox, archive } = await seedThreads(threadMessage);
|
|
211
191
|
const updates: Update[] = [];
|
|
192
|
+
const filterUpdates: World["filterUpdates"] = [];
|
|
212
193
|
const labeled: string[] = [];
|
|
213
194
|
const mailboxOf: Record<string, string> = {
|
|
214
195
|
[inbox]: INBOX,
|
|
@@ -221,10 +202,8 @@ const world = async (
|
|
|
221
202
|
organizeJobId: "job-1",
|
|
222
203
|
accountConfigId: CONFIG,
|
|
223
204
|
filterId: "flt-1",
|
|
224
|
-
anchorMessageId: "None",
|
|
225
205
|
matchOperator: "And",
|
|
226
206
|
literalClauses: [],
|
|
227
|
-
similarityThreshold: 0.75,
|
|
228
207
|
actionLabelId: "None",
|
|
229
208
|
actionMailboxId: "None",
|
|
230
209
|
}),
|
|
@@ -241,13 +220,21 @@ const world = async (
|
|
|
241
220
|
refreshExpiry: async (filter: FilterItem) => filter,
|
|
242
221
|
listByAccountAndState: async (_cfg: string, state: string) =>
|
|
243
222
|
filters.filter((f) => f.state === state),
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
223
|
+
update: async (
|
|
224
|
+
_cfg: string,
|
|
225
|
+
filterId: string,
|
|
226
|
+
patch: Record<string, unknown>,
|
|
227
|
+
) => {
|
|
228
|
+
filterUpdates.push({ filterId, patch });
|
|
229
|
+
return {};
|
|
230
|
+
},
|
|
248
231
|
},
|
|
249
232
|
account: {
|
|
250
233
|
listMailSourcesByAccountConfig: async () => [{ accountId: "acct-1" }],
|
|
234
|
+
get: async (accountId: string) => ({
|
|
235
|
+
accountId,
|
|
236
|
+
email: accountId === "acct-1" ? "me@example.com" : "me-too@example.com",
|
|
237
|
+
}),
|
|
251
238
|
},
|
|
252
239
|
mailboxSpecialUse: {
|
|
253
240
|
findInboxMailbox: async () => ({ mailboxId: INBOX, fullPath: "INBOX" }),
|
|
@@ -258,7 +245,18 @@ const world = async (
|
|
|
258
245
|
mailboxId: mailboxOf[messageId],
|
|
259
246
|
}),
|
|
260
247
|
},
|
|
261
|
-
mailbox: {
|
|
248
|
+
mailbox: {
|
|
249
|
+
resolveAccountId: async (mailboxId: string) =>
|
|
250
|
+
mailboxId === "mbx-gone"
|
|
251
|
+
? null
|
|
252
|
+
: mailboxId === "mbx-sibling-travel"
|
|
253
|
+
? "acct-2"
|
|
254
|
+
: "acct-1",
|
|
255
|
+
get: async (_accountId: string, mailboxId: string) => ({
|
|
256
|
+
mailboxId,
|
|
257
|
+
fullPath: mailboxId === "mbx-sibling-travel" ? "Travel" : "Invoices",
|
|
258
|
+
}),
|
|
259
|
+
},
|
|
262
260
|
messageLabel: {
|
|
263
261
|
apply: async (input: { messageId: string }) => {
|
|
264
262
|
labeled.push(input.messageId);
|
|
@@ -271,6 +269,7 @@ const world = async (
|
|
|
271
269
|
inbox,
|
|
272
270
|
archive,
|
|
273
271
|
updates,
|
|
272
|
+
filterUpdates,
|
|
274
273
|
moves: [],
|
|
275
274
|
labeled,
|
|
276
275
|
close: store.close,
|
|
@@ -295,50 +294,6 @@ const runFilterJob = (w: World) =>
|
|
|
295
294
|
moveService: recordingMoves(w.moves),
|
|
296
295
|
});
|
|
297
296
|
|
|
298
|
-
const EMBEDDING_ID = "test-model@4";
|
|
299
|
-
const ANCHOR_VECTOR = [1, 0, 0, 0];
|
|
300
|
-
const ORTHOGONAL_VECTOR = [0, 1, 0, 0];
|
|
301
|
-
|
|
302
|
-
const persistedAnchor: FilterAnchorItem = {
|
|
303
|
-
filterId: "flt-1",
|
|
304
|
-
accountConfigId: CONFIG,
|
|
305
|
-
anchorMessageId: "msg-anchor",
|
|
306
|
-
anchorEmbedding: ANCHOR_VECTOR,
|
|
307
|
-
anchorEmbeddingId: EMBEDDING_ID,
|
|
308
|
-
anchorSourceText: "Your invoice",
|
|
309
|
-
createdAt: 0,
|
|
310
|
-
updatedAt: 0,
|
|
311
|
-
};
|
|
312
|
-
|
|
313
|
-
/**
|
|
314
|
-
* The vector index answers the anchor with the inbox invoice, and only when
|
|
315
|
-
* asked about the inbox: the kNN read is where a similarity filter finds its
|
|
316
|
-
* candidates.
|
|
317
|
-
*/
|
|
318
|
-
const vectorMatchDeps = (w: World): OrganizeMatchDeps => ({
|
|
319
|
-
...buildOrganizeMatchDeps(w.client),
|
|
320
|
-
semantic: () => ({
|
|
321
|
-
buildAnchor: async () => {
|
|
322
|
-
throw new Error("a filter widens on its persisted anchor");
|
|
323
|
-
},
|
|
324
|
-
vectorStore: {
|
|
325
|
-
query: async (query: { filter?: { mailboxId?: string } }) =>
|
|
326
|
-
query.filter?.mailboxId === INBOX
|
|
327
|
-
? [
|
|
328
|
-
{
|
|
329
|
-
chunkId: `${w.inbox}#body`,
|
|
330
|
-
score: 0.9,
|
|
331
|
-
metadata: { messageId: w.inbox, mailboxIds: [INBOX] },
|
|
332
|
-
},
|
|
333
|
-
]
|
|
334
|
-
: [],
|
|
335
|
-
getByMessage: async () => [],
|
|
336
|
-
} as unknown as OrganizeSemanticDeps["vectorStore"],
|
|
337
|
-
embed: async () => ANCHOR_VECTOR,
|
|
338
|
-
embeddingId: EMBEDDING_ID,
|
|
339
|
-
}),
|
|
340
|
-
});
|
|
341
|
-
|
|
342
297
|
describe("processOrganizeJob for a filter job (#1354)", () => {
|
|
343
298
|
let current: World | undefined;
|
|
344
299
|
afterEach(() => current?.close());
|
|
@@ -390,6 +345,42 @@ describe("processOrganizeJob for a filter job (#1354)", () => {
|
|
|
390
345
|
assert.equal(current.updates.at(-1)?.state, "Complete");
|
|
391
346
|
});
|
|
392
347
|
|
|
348
|
+
it("skips another account's mail, as if the filter had not matched", async () => {
|
|
349
|
+
current = await world([
|
|
350
|
+
standingFilter({
|
|
351
|
+
actionLabelId: "lbl-trips",
|
|
352
|
+
actionMailboxId: "mbx-sibling-travel",
|
|
353
|
+
}),
|
|
354
|
+
]);
|
|
355
|
+
|
|
356
|
+
await runFilterJob(current);
|
|
357
|
+
|
|
358
|
+
assert.deepEqual(current.moves, []);
|
|
359
|
+
assert.deepEqual(current.labeled, []);
|
|
360
|
+
assert.deepEqual(current.filterUpdates, []);
|
|
361
|
+
const done = current.updates.at(-1);
|
|
362
|
+
assert.equal(done?.state, "Complete");
|
|
363
|
+
assert.equal(done?.appliedCount, 0);
|
|
364
|
+
assert.equal(done?.failedCount, 0);
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
it("turns the filter off and fails the job when no account holds its folder", async () => {
|
|
368
|
+
current = await world([standingFilter({ actionMailboxId: "mbx-gone" })]);
|
|
369
|
+
|
|
370
|
+
await runFilterJob(current);
|
|
371
|
+
|
|
372
|
+
assert.deepEqual(current.moves, []);
|
|
373
|
+
assert.deepEqual(current.filterUpdates, [
|
|
374
|
+
{
|
|
375
|
+
filterId: "flt-1",
|
|
376
|
+
patch: { state: "Disabled", disabledReason: "FolderMissing" },
|
|
377
|
+
},
|
|
378
|
+
]);
|
|
379
|
+
const done = current.updates.at(-1);
|
|
380
|
+
assert.equal(done?.state, "Failed");
|
|
381
|
+
assert.match(String(done?.errorMessage), /no longer exists/);
|
|
382
|
+
});
|
|
383
|
+
|
|
393
384
|
it("fails the job without a retry when the filter is turned off", async () => {
|
|
394
385
|
current = await world([
|
|
395
386
|
standingFilter({ state: "Disabled", disabledReason: "UserDisabled" }),
|
|
@@ -410,32 +401,4 @@ describe("processOrganizeJob for a filter job (#1354)", () => {
|
|
|
410
401
|
assert.equal(current.updates.at(-1)?.state, "Failed");
|
|
411
402
|
assert.match(String(current.updates.at(-1)?.errorMessage), /deleted/);
|
|
412
403
|
});
|
|
413
|
-
|
|
414
|
-
it("moves a message the vector query found, though the preview scores below the threshold", async () => {
|
|
415
|
-
current = await world(
|
|
416
|
-
[standingFilter({ hasAnchor: true, literalClauses: [] })],
|
|
417
|
-
[persistedAnchor],
|
|
418
|
-
);
|
|
419
|
-
|
|
420
|
-
// The index-time evaluator embeds subject and preview; this embedder puts
|
|
421
|
-
// that text orthogonal to the anchor, so evaluate alone would call the
|
|
422
|
-
// message a miss. It only orders the move, so the vector match stands.
|
|
423
|
-
await processOrganizeJob(event, noopLogger, {
|
|
424
|
-
client: current.client,
|
|
425
|
-
matchDeps: vectorMatchDeps(current),
|
|
426
|
-
moveService: recordingMoves(current.moves),
|
|
427
|
-
embedder: {
|
|
428
|
-
embed: async () => ORTHOGONAL_VECTOR,
|
|
429
|
-
embeddingId: EMBEDDING_ID,
|
|
430
|
-
},
|
|
431
|
-
});
|
|
432
|
-
|
|
433
|
-
assert.deepEqual(current.moves, [
|
|
434
|
-
{ messageId: current.inbox, destinationMailboxId: "mbx-invoices" },
|
|
435
|
-
]);
|
|
436
|
-
const done = current.updates.at(-1);
|
|
437
|
-
assert.equal(done?.state, "Complete");
|
|
438
|
-
assert.equal(done?.matchedCount, 1);
|
|
439
|
-
assert.equal(done?.appliedCount, 1);
|
|
440
|
-
});
|
|
441
404
|
});
|
|
@@ -5,7 +5,6 @@ import {
|
|
|
5
5
|
backApplyFilter,
|
|
6
6
|
buildOrganizeMatchDeps,
|
|
7
7
|
buildOrganizeMoveService,
|
|
8
|
-
type FilterBackApplyDeps,
|
|
9
8
|
isFilterJob,
|
|
10
9
|
matchOrganize,
|
|
11
10
|
ORGANIZE_MATCH_LIMIT,
|
|
@@ -13,14 +12,13 @@ import {
|
|
|
13
12
|
type OrganizeRejection,
|
|
14
13
|
predicateFromJob,
|
|
15
14
|
} from "@remit/backend/organize";
|
|
16
|
-
import type { Logger } from "@remit/logger
|
|
15
|
+
import type { Logger } from "@remit/logger";
|
|
17
16
|
import type { OrganizeJobEvent } from "../events.js";
|
|
18
17
|
|
|
19
18
|
export interface ProcessOrganizeJobDeps {
|
|
20
19
|
client?: RemitClient;
|
|
21
20
|
matchDeps?: OrganizeMatchDeps;
|
|
22
21
|
moveService?: ApplyOrganizeDeps["moveService"];
|
|
23
|
-
embedder?: FilterBackApplyDeps["embedder"];
|
|
24
22
|
}
|
|
25
23
|
|
|
26
24
|
interface OrganizeOutcome {
|
|
@@ -28,7 +26,7 @@ interface OrganizeOutcome {
|
|
|
28
26
|
matched: number;
|
|
29
27
|
applied: number;
|
|
30
28
|
failed: number;
|
|
31
|
-
|
|
29
|
+
failures: string[];
|
|
32
30
|
}
|
|
33
31
|
|
|
34
32
|
const rejectedOutcome = (rejected: OrganizeRejection): OrganizeOutcome => ({
|
|
@@ -36,7 +34,7 @@ const rejectedOutcome = (rejected: OrganizeRejection): OrganizeOutcome => ({
|
|
|
36
34
|
matched: 0,
|
|
37
35
|
applied: 0,
|
|
38
36
|
failed: 0,
|
|
39
|
-
|
|
37
|
+
failures: [],
|
|
40
38
|
});
|
|
41
39
|
|
|
42
40
|
/**
|
|
@@ -57,7 +55,7 @@ const runSnapshot = async (
|
|
|
57
55
|
ORGANIZE_MATCH_LIMIT,
|
|
58
56
|
);
|
|
59
57
|
if (match.rejected) return rejectedOutcome(match.rejected);
|
|
60
|
-
const { applied, failed } = await applyOrganize(
|
|
58
|
+
const { applied, failed, failures } = await applyOrganize(
|
|
61
59
|
{ client, moveService },
|
|
62
60
|
job.accountConfigId,
|
|
63
61
|
match.messageIds,
|
|
@@ -68,7 +66,7 @@ const runSnapshot = async (
|
|
|
68
66
|
matched: match.messageIds.length,
|
|
69
67
|
applied,
|
|
70
68
|
failed,
|
|
71
|
-
|
|
69
|
+
failures,
|
|
72
70
|
};
|
|
73
71
|
};
|
|
74
72
|
|
|
@@ -81,7 +79,7 @@ const runSnapshot = async (
|
|
|
81
79
|
*
|
|
82
80
|
* Reuses the shared matcher (so the applied set equals what preview returned)
|
|
83
81
|
* and the idempotent apply plumbing; `appliedByFilterId` is never written and no
|
|
84
|
-
* Filter
|
|
82
|
+
* Filter row is ever created. Both back-apply actions run here: the
|
|
85
83
|
* additive label upsert and the exclusive folder move, the latter through the
|
|
86
84
|
* same local-first placement mover body sync uses (`buildOrganizeMoveService`),
|
|
87
85
|
* so a redelivered job re-applies both idempotently.
|
|
@@ -114,7 +112,7 @@ export const processOrganizeJob = async (
|
|
|
114
112
|
const moveService = deps.moveService ?? buildOrganizeMoveService(client);
|
|
115
113
|
const outcome = isFilterJob(job)
|
|
116
114
|
? await backApplyFilter(
|
|
117
|
-
{ client, matchDeps, moveService
|
|
115
|
+
{ client, matchDeps, moveService },
|
|
118
116
|
accountConfigId,
|
|
119
117
|
job.filterId,
|
|
120
118
|
).then((result) =>
|
|
@@ -137,12 +135,13 @@ export const processOrganizeJob = async (
|
|
|
137
135
|
return;
|
|
138
136
|
}
|
|
139
137
|
|
|
140
|
-
const { matched, applied, failed,
|
|
138
|
+
const { matched, applied, failed, failures } = outcome;
|
|
141
139
|
await client.organizeJobRequest.update(organizeJobId, {
|
|
142
140
|
state: "Complete",
|
|
143
141
|
matchedCount: matched,
|
|
144
142
|
appliedCount: applied,
|
|
145
143
|
failedCount: failed,
|
|
144
|
+
errorMessage: failures[0] ?? "",
|
|
146
145
|
});
|
|
147
146
|
log.info(
|
|
148
147
|
{
|
|
@@ -152,7 +151,6 @@ export const processOrganizeJob = async (
|
|
|
152
151
|
matched,
|
|
153
152
|
applied,
|
|
154
153
|
failed,
|
|
155
|
-
semanticUnavailable,
|
|
156
154
|
},
|
|
157
155
|
"Organize back-apply complete",
|
|
158
156
|
);
|
|
@@ -2,7 +2,7 @@ import assert from "node:assert/strict";
|
|
|
2
2
|
import { describe, it } from "node:test";
|
|
3
3
|
import type { RemitClient } from "@remit/backend/client";
|
|
4
4
|
import { NotFoundError } from "@remit/data-ports/errors";
|
|
5
|
-
import { noopLogger } from "@remit/logger
|
|
5
|
+
import { noopLogger } from "@remit/logger";
|
|
6
6
|
import type { SenderCategoryBackApplyEvent } from "../events.js";
|
|
7
7
|
import { processSenderCategoryBackApply } from "./sender-category-backapply.js";
|
|
8
8
|
|
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
SENDER_CATEGORY_BACKAPPLY_LIMIT,
|
|
5
5
|
} from "@remit/backend/sender-category";
|
|
6
6
|
import { NotFoundError } from "@remit/data-ports/errors";
|
|
7
|
-
import type { Logger } from "@remit/logger
|
|
7
|
+
import type { Logger } from "@remit/logger";
|
|
8
8
|
import type { SenderCategoryBackApplyEvent } from "../events.js";
|
|
9
9
|
|
|
10
10
|
export interface ProcessSenderCategoryBackApplyDeps {
|
package/src/poller.ts
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
|
-
import { createLogger
|
|
1
|
+
import { createLogger } from "@remit/logger";
|
|
2
|
+
import { startMetricsServer } from "@remit/logger-lambda/metrics";
|
|
3
|
+
import { installNodeLogger } from "@remit/logger-lambda/node";
|
|
2
4
|
import { runQueuePoller } from "@remit/sqs-client/poller";
|
|
3
5
|
import { env } from "expect-env";
|
|
4
6
|
import { fanoutHandler, finalizeHandler } from "./index.js";
|
|
5
7
|
|
|
8
|
+
installNodeLogger();
|
|
9
|
+
|
|
6
10
|
/**
|
|
7
11
|
* Production queue poller, and the standalone entrypoint for the dedicated
|
|
8
12
|
* image. The sibling workers carry a separate `e2e-processor-shim.ts` because
|
package/src/cascade.test.ts
DELETED
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
import assert from "node:assert";
|
|
2
|
-
import { existsSync, readdirSync } from "node:fs";
|
|
3
|
-
import { resolve } from "node:path";
|
|
4
|
-
import { describe, it } from "node:test";
|
|
5
|
-
import { COVERED_ENTITY_TYPES } from "./cascade.js";
|
|
6
|
-
|
|
7
|
-
// The snapshot enumerates electrodb-service's models; that package is stripped
|
|
8
|
-
// from the open-core tree, so the check skips there and runs where it ships.
|
|
9
|
-
const modelsDir = resolve(
|
|
10
|
-
import.meta.dirname,
|
|
11
|
-
"../../remit-electrodb-service/src/models",
|
|
12
|
-
);
|
|
13
|
-
|
|
14
|
-
describe("cascade entity coverage snapshot", () => {
|
|
15
|
-
it("covers every entity model in remit-electrodb-service", {
|
|
16
|
-
skip: !existsSync(modelsDir),
|
|
17
|
-
}, () => {
|
|
18
|
-
const modelFiles = readdirSync(modelsDir)
|
|
19
|
-
.filter(
|
|
20
|
-
(f) => f.endsWith(".ts") && !f.endsWith(".test.ts") && f !== "index.ts",
|
|
21
|
-
)
|
|
22
|
-
.map((f) => f.replace(".ts", ""));
|
|
23
|
-
|
|
24
|
-
// Map model file names to the entity types they contain
|
|
25
|
-
const entityTypesByFile: Record<string, string[]> = {
|
|
26
|
-
"account-config": ["AccountConfig"],
|
|
27
|
-
"account-export-request": [],
|
|
28
|
-
"account-setting": ["AccountSetting"],
|
|
29
|
-
"account-setting-registry": [],
|
|
30
|
-
account: ["Account"],
|
|
31
|
-
address: ["Address", "EnvelopeAddress"],
|
|
32
|
-
envelope: [
|
|
33
|
-
"Envelope",
|
|
34
|
-
"MessageReference",
|
|
35
|
-
"BodyPart",
|
|
36
|
-
"BodyPartParameter",
|
|
37
|
-
"RawMessageStorage",
|
|
38
|
-
"BodyPartStorage",
|
|
39
|
-
"BodyPartContent",
|
|
40
|
-
],
|
|
41
|
-
filter: ["Filter"],
|
|
42
|
-
"filter-anchor": ["FilterAnchor"],
|
|
43
|
-
label: ["Label"],
|
|
44
|
-
mailbox: ["Mailbox"],
|
|
45
|
-
"mailbox-lock": ["MailboxLock"],
|
|
46
|
-
"mailbox-special-use": [],
|
|
47
|
-
message: ["Message"],
|
|
48
|
-
"message-flag": ["MessageFlag"],
|
|
49
|
-
"message-flag-push": ["MessageFlagPush"],
|
|
50
|
-
"message-label": ["MessageLabel"],
|
|
51
|
-
"message-placement-move": ["MessagePlacementMove"],
|
|
52
|
-
"organize-job-request": [],
|
|
53
|
-
"outbox-message": ["OutboxMessage"],
|
|
54
|
-
"thread-message": ["ThreadMessage"],
|
|
55
|
-
"wellknown-rule": [],
|
|
56
|
-
};
|
|
57
|
-
|
|
58
|
-
const allEntityTypes = Object.values(entityTypesByFile).flat().sort();
|
|
59
|
-
const coveredSorted = [...COVERED_ENTITY_TYPES].sort();
|
|
60
|
-
|
|
61
|
-
assert.deepStrictEqual(
|
|
62
|
-
coveredSorted,
|
|
63
|
-
allEntityTypes,
|
|
64
|
-
`Cascade does not cover all entity types.\n` +
|
|
65
|
-
`Missing: ${allEntityTypes.filter((t) => !(coveredSorted as readonly string[]).includes(t)).join(", ")}\n` +
|
|
66
|
-
`Extra: ${(coveredSorted as readonly string[]).filter((t) => !allEntityTypes.includes(t)).join(", ")}`,
|
|
67
|
-
);
|
|
68
|
-
|
|
69
|
-
// Verify all model files are accounted for
|
|
70
|
-
const unmappedFiles = modelFiles.filter((f) => !(f in entityTypesByFile));
|
|
71
|
-
assert.deepStrictEqual(
|
|
72
|
-
unmappedFiles,
|
|
73
|
-
[],
|
|
74
|
-
`Model files not mapped to entity types: ${unmappedFiles.join(", ")}`,
|
|
75
|
-
);
|
|
76
|
-
});
|
|
77
|
-
});
|