@remit/account-worker 0.0.26 → 0.0.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@remit/account-worker",
3
- "version": "0.0.26",
3
+ "version": "0.0.27",
4
4
  "type": "module",
5
5
  "main": "dist/index.js",
6
6
  "exports": {
@@ -33,6 +33,7 @@
33
33
  "dev": "node --import tsx src/poller.ts"
34
34
  },
35
35
  "dependencies": {
36
+ "@remit/logger": "*",
36
37
  "@remit/data-ports": "*",
37
38
  "@remit/logger-lambda": "*",
38
39
  "@remit/drizzle-service": "*",
@@ -40,7 +41,6 @@
40
41
  "@remit/sqs-client": "*",
41
42
  "@remit/storage-service": "*",
42
43
  "@remit/electrodb-entities": "*",
43
- "@remit/search-index-worker": "*",
44
44
  "@aws-sdk/client-sqs": "*",
45
45
  "archiver": "*",
46
46
  "electrodb": "*",
@@ -1,7 +1,7 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { describe, it } from "node:test";
3
3
  import type { MessageDescription } from "@remit/data-ports";
4
- import { noopLogger } from "@remit/logger-lambda/noop-logger";
4
+ import { noopLogger } from "@remit/logger";
5
5
  import {
6
6
  type CascadeEntity,
7
7
  type CascadeServices,
@@ -18,11 +18,10 @@ const messageWithChildren = {
18
18
  bodyPartParameter: [{ bodyPartParameterId: "bpp-1" }],
19
19
  rawMessageStorage: [{ rawStorageId: "rms-1" }],
20
20
  bodyPartStorage: [{ bodyPartStorageId: "bps-1" }],
21
- bodyPartContent: [{ bodyPartContentId: "bpc-1" }],
22
21
  } as unknown as MessageDescription;
23
22
 
24
23
  describe("collectMessageChildEntities", () => {
25
- it("expands the nine child entities of one message", () => {
24
+ it("expands the eight child entities of one message", () => {
26
25
  const entities: CascadeEntity[] = [];
27
26
  collectMessageChildEntities(entities, messageWithChildren);
28
27
 
@@ -36,13 +35,7 @@ describe("collectMessageChildEntities", () => {
36
35
  "BodyPartParameter",
37
36
  "RawMessageStorage",
38
37
  "BodyPartStorage",
39
- "BodyPartContent",
40
38
  ]);
41
- assert.equal(
42
- entities.find((e) => e.entityType === "BodyPartContent")?.key
43
- .bodyPartContentId,
44
- "bpc-1",
45
- );
46
39
  });
47
40
 
48
41
  it("appends nothing when every child list is empty", () => {
@@ -56,7 +49,6 @@ describe("collectMessageChildEntities", () => {
56
49
  bodyPartParameter: [],
57
50
  rawMessageStorage: [],
58
51
  bodyPartStorage: [],
59
- bodyPartContent: [],
60
52
  } as unknown as MessageDescription);
61
53
  assert.equal(entities.length, 0);
62
54
  });
@@ -96,13 +88,7 @@ const fullServices = (): CascadeServices =>
96
88
  listByAccountConfig: async () => [{ accountSettingId: "set-1" }],
97
89
  },
98
90
  filterService: {
99
- listByAccountConfig: async () => [
100
- { filterId: "flt-1", hasAnchor: true },
101
- { filterId: "flt-2", hasAnchor: false },
102
- ],
103
- },
104
- filterAnchorService: {
105
- get: async () => ({ filterId: "flt-1" }),
91
+ listByAccountConfig: async () => [{ filterId: "flt-1" }],
106
92
  },
107
93
  labelService: {
108
94
  listByAccountConfig: async () => [{ labelId: "lbl-1" }],
@@ -128,7 +114,6 @@ describe("enumerateCascadeEntities", () => {
128
114
  "Mailbox",
129
115
  "Message",
130
116
  "MessageFlag",
131
- "BodyPartContent",
132
117
  "OutboxMessage",
133
118
  "MailboxLock",
134
119
  "MessagePlacementMove",
@@ -136,7 +121,6 @@ describe("enumerateCascadeEntities", () => {
136
121
  "ThreadMessage",
137
122
  "AccountSetting",
138
123
  "Filter",
139
- "FilterAnchor",
140
124
  "Label",
141
125
  "MessageLabel",
142
126
  "Address",
@@ -148,40 +132,6 @@ describe("enumerateCascadeEntities", () => {
148
132
  assert.equal(types[types.length - 1], "AccountConfig");
149
133
  });
150
134
 
151
- it("emits a Filter without a FilterAnchor when the filter has no anchor", async () => {
152
- const services = fullServices();
153
- services.filterService.listByAccountConfig = async () =>
154
- [{ filterId: "flt-2", hasAnchor: false }] as never;
155
-
156
- const { entities } = await enumerateCascadeEntities(
157
- "cfg-1",
158
- services,
159
- noopLogger,
160
- );
161
-
162
- assert.equal(entities.filter((e) => e.entityType === "Filter").length, 1);
163
- assert.equal(
164
- entities.filter((e) => e.entityType === "FilterAnchor").length,
165
- 0,
166
- );
167
- });
168
-
169
- it("skips the FilterAnchor when the anchor lookup returns null", async () => {
170
- const services = fullServices();
171
- services.filterAnchorService.get = async () => null as never;
172
-
173
- const { entities } = await enumerateCascadeEntities(
174
- "cfg-1",
175
- services,
176
- noopLogger,
177
- );
178
-
179
- assert.equal(
180
- entities.filter((e) => e.entityType === "FilterAnchor").length,
181
- 0,
182
- );
183
- });
184
-
185
135
  it("yields only the AccountConfig row for an empty tenant", async () => {
186
136
  const services = fullServices();
187
137
  services.accountConfigService.describe = async () =>
package/src/cascade.ts CHANGED
@@ -5,7 +5,6 @@ import type {
5
5
  IAccountSettingRepository,
6
6
  IAddressRepository,
7
7
  IEnvelopeRepository,
8
- IFilterAnchorRepository,
9
8
  IFilterRepository,
10
9
  ILabelRepository,
11
10
  IMailboxLockRepository,
@@ -19,7 +18,7 @@ import type {
19
18
  IThreadMessageRepository,
20
19
  MessageDescription,
21
20
  } from "@remit/data-ports";
22
- import type { Logger } from "@remit/logger-lambda";
21
+ import type { Logger } from "@remit/logger";
23
22
 
24
23
  export interface CascadeEntity {
25
24
  entityType: string;
@@ -42,7 +41,6 @@ export interface CascadeServices {
42
41
  accountExportRequestService: IAccountExportRequestRepository;
43
42
  accountSettingService: IAccountSettingRepository;
44
43
  filterService: IFilterRepository;
45
- filterAnchorService: IFilterAnchorRepository;
46
44
  labelService: ILabelRepository;
47
45
  messageLabelService: IMessageLabelRepository;
48
46
  }
@@ -111,12 +109,6 @@ export const collectMessageChildEntities = (
111
109
  key: { bodyPartStorageId: bps.bodyPartStorageId },
112
110
  });
113
111
  }
114
- for (const bpc of messageData.bodyPartContent) {
115
- entities.push({
116
- entityType: "BodyPartContent",
117
- key: { bodyPartContentId: bpc.bodyPartContentId },
118
- });
119
- }
120
112
  };
121
113
 
122
114
  export const enumerateCascadeEntities = async (
@@ -138,7 +130,6 @@ export const enumerateCascadeEntities = async (
138
130
  messageFlagPushService,
139
131
  accountSettingService,
140
132
  filterService,
141
- filterAnchorService,
142
133
  labelService,
143
134
  messageLabelService,
144
135
  } = services;
@@ -225,13 +216,6 @@ export const enumerateCascadeEntities = async (
225
216
  key: { bodyPartStorageId: bps.bodyPartStorageId },
226
217
  });
227
218
  }
228
-
229
- for (const bpc of messageData.bodyPartContent) {
230
- entities.push({
231
- entityType: "BodyPartContent",
232
- key: { bodyPartContentId: bpc.bodyPartContentId },
233
- });
234
- }
235
219
  }
236
220
  }
237
221
 
@@ -292,7 +276,7 @@ export const enumerateCascadeEntities = async (
292
276
  });
293
277
  }
294
278
 
295
- // Filter/FilterAnchor/Label/MessageLabel are accountConfig-scoped (RFC 034),
279
+ // Filter/Label/MessageLabel are accountConfig-scoped (RFC 034),
296
280
  // not per sub-account, so — like AccountSetting and ThreadMessage above —
297
281
  // they are enumerated once here rather than inside the per-account loop.
298
282
  const filters = await filterService.listByAccountConfig(accountConfigId);
@@ -301,16 +285,6 @@ export const enumerateCascadeEntities = async (
301
285
  entityType: "Filter",
302
286
  key: { accountConfigId, filterId: filter.filterId },
303
287
  });
304
- if (!filter.hasAnchor) continue;
305
- const anchor = await filterAnchorService.get(
306
- accountConfigId,
307
- filter.filterId,
308
- );
309
- if (!anchor) continue;
310
- entities.push({
311
- entityType: "FilterAnchor",
312
- key: { accountConfigId, filterId: filter.filterId },
313
- });
314
288
  }
315
289
 
316
290
  const labels = await labelService.listByAccountConfig(accountConfigId);
@@ -354,30 +328,3 @@ export const enumerateCascadeEntities = async (
354
328
 
355
329
  return { entities, messageIds };
356
330
  };
357
-
358
- export const COVERED_ENTITY_TYPES = [
359
- "Account",
360
- "AccountConfig",
361
- "AccountSetting",
362
- "Address",
363
- "BodyPart",
364
- "BodyPartContent",
365
- "BodyPartParameter",
366
- "BodyPartStorage",
367
- "Envelope",
368
- "EnvelopeAddress",
369
- "Filter",
370
- "FilterAnchor",
371
- "Label",
372
- "Mailbox",
373
- "MailboxLock",
374
- "Message",
375
- "MessageFlag",
376
- "MessageFlagPush",
377
- "MessageLabel",
378
- "MessagePlacementMove",
379
- "MessageReference",
380
- "OutboxMessage",
381
- "RawMessageStorage",
382
- "ThreadMessage",
383
- ] as const;
@@ -3,7 +3,7 @@ import { mkdir, mkdtemp, stat, writeFile } from "node:fs/promises";
3
3
  import { tmpdir } from "node:os";
4
4
  import { dirname, join } from "node:path";
5
5
  import { afterEach, beforeEach, describe, it } from "node:test";
6
- import { noopLogger } from "@remit/logger-lambda/noop-logger";
6
+ import { noopLogger } from "@remit/logger";
7
7
  import { buildRelationalDeletionCapabilities } from "./compose-relational.js";
8
8
 
9
9
  const exists = (path: string): Promise<boolean> =>
@@ -79,16 +79,3 @@ describe("relational deletion capabilities — filesystem storage cleanup", () =
79
79
  await caps.deleteStoragePrefix("accounts/never-existed/", noopLogger);
80
80
  });
81
81
  });
82
-
83
- describe("relational deletion capabilities — CDN and sign-out are no-ops", () => {
84
- it("invalidateContent resolves without a CDN", async () => {
85
- const caps = buildRelationalDeletionCapabilities();
86
- await caps.invalidateContent("cfg-1", noopLogger);
87
- await caps.invalidateContent("cfg-1/acc-1", noopLogger);
88
- });
89
-
90
- it("signOut resolves without a federated session store", async () => {
91
- const caps = buildRelationalDeletionCapabilities();
92
- await caps.signOut("user-123", noopLogger);
93
- });
94
- });
@@ -3,9 +3,9 @@ import { join } from "node:path";
3
3
  import {
4
4
  type CascadeDeleter,
5
5
  createSqliteCascadeDeleter,
6
+ sqliteDatabasePathsFromEnv,
6
7
  } from "@remit/drizzle-service";
7
- import type { Logger } from "@remit/logger-lambda";
8
- import { env } from "expect-env";
8
+ import type { Logger } from "@remit/logger";
9
9
  import type { DeletionCapabilities } from "./deletion-capabilities.js";
10
10
 
11
11
  // Mirrors `@remit/storage-service`'s filesystem-backend default: bodies
@@ -30,14 +30,14 @@ const deleteStoragePrefix = async (
30
30
  };
31
31
 
32
32
  // The Drizzle cascade deleter is opened once, on first use, and reused: it
33
- // opens the shared file at `SQLITE_DB_PATH` (the native binding loads there,
33
+ // opens the app, mail and search databases (the native binding loads there,
34
34
  // hence async). Built lazily so the fanout worker — which signs out but never
35
35
  // cascade-deletes — does not open a database handle it will not use.
36
36
  let deleterPromise: Promise<CascadeDeleter> | undefined;
37
37
 
38
38
  const getDeleter = (): Promise<CascadeDeleter> => {
39
39
  if (!deleterPromise)
40
- deleterPromise = createSqliteCascadeDeleter(env.SQLITE_DB_PATH);
40
+ deleterPromise = createSqliteCascadeDeleter(sqliteDatabasePathsFromEnv());
41
41
  return deleterPromise;
42
42
  };
43
43
 
@@ -52,8 +52,7 @@ const getDeleter = (): Promise<CascadeDeleter> => {
52
52
  * backend with no response cache (`deploy/vps/caddy/routes.caddy`), so there
53
53
  * is nothing to invalidate.
54
54
  * - `deleteStoragePrefix`: recursive filesystem removal.
55
- * - `cascadeDelete`: the Drizzle cascade over SQLite. Message
56
- * subtrees emit `message.removed` outbox rows the search-index worker relays.
55
+ * - `cascadeDelete`: the Drizzle cascade over SQLite.
57
56
  */
58
57
  export const buildRelationalDeletionCapabilities =
59
58
  (): DeletionCapabilities => ({
@@ -51,9 +51,4 @@ describe("getImapWorkerQueueUrl", () => {
51
51
  delete process.env.SQS_QUEUE_URL_IMAP_WORKER;
52
52
  assert.equal(getImapWorkerQueueUrl(), undefined);
53
53
  });
54
-
55
- it("returns the configured value when set", () => {
56
- process.env.SQS_QUEUE_URL_IMAP_WORKER = "http://queue/imap-worker";
57
- assert.equal(getImapWorkerQueueUrl(), "http://queue/imap-worker");
58
- });
59
54
  });
package/src/config.ts CHANGED
@@ -26,8 +26,6 @@ export const getSqsClient = (queueUrl: string): SQSClient => {
26
26
  // no queue, so its Lambda doesn't carry the env vars. Eager evaluation here would
27
27
  // crash finalize at module load — getters defer the read to the fanout-only call
28
28
  // sites.
29
- export const getSearchIndexQueueUrl = (): string =>
30
- env.SQS_QUEUE_URL_SEARCH_INDEX;
31
29
  // Optional, and read through `process.env` rather than expect-env so an unset
32
30
  // var yields undefined instead of throwing. IMAP_WORKER_STOP is a no-op
33
31
  // acknowledgement in the cascade contract — the account tombstone fence is what
@@ -48,7 +46,7 @@ let cascadeServicesPromise: Promise<CascadeServices> | null = null;
48
46
 
49
47
  // Every cascade service is a `RemitClient` repository, so the whole enumeration
50
48
  // runs on whatever backend `getClient()` resolved — DynamoDB (ElectroDB) or
51
- // SQLite (Drizzle). Filter/FilterAnchor/Label/MessageLabel (Smart Organize,
49
+ // SQLite (Drizzle). Filter/Label/MessageLabel (Smart Organize,
52
50
  // RFC 034) are present on both via the client, so they need no
53
51
  // backend-specific wiring here.
54
52
  export const getCascadeServices = (): Promise<CascadeServices> => {
@@ -69,7 +67,6 @@ export const getCascadeServices = (): Promise<CascadeServices> => {
69
67
  accountExportRequestService: remitClient.accountExportRequest,
70
68
  accountSettingService: remitClient.accountSetting,
71
69
  filterService: remitClient.filter,
72
- filterAnchorService: remitClient.filterAnchor,
73
70
  labelService: remitClient.label,
74
71
  messageLabelService: remitClient.messageLabel,
75
72
  }));
@@ -84,5 +81,3 @@ export const getStorageService = (): StorageService => {
84
81
  }
85
82
  return storageService;
86
83
  };
87
-
88
- export const dataBackend = process.env.DATA_BACKEND;
@@ -1,4 +1,4 @@
1
- import type { Logger } from "@remit/logger-lambda";
1
+ import type { Logger } from "@remit/logger";
2
2
  import type { CascadeEntity } from "./cascade.js";
3
3
 
4
4
  /**
@@ -71,7 +71,7 @@ export const setDeletionCapabilities = (
71
71
  const buildDeletionCapabilitiesFromEnv =
72
72
  async (): Promise<DeletionCapabilities> => {
73
73
  if (injectedCapabilities) return injectedCapabilities;
74
- if (process.env.SQLITE_DB_PATH) {
74
+ if (process.env.SQLITE_APP_DB_PATH) {
75
75
  const { buildRelationalDeletionCapabilities } = await import(
76
76
  "./compose-relational.js"
77
77
  );
package/src/events.ts CHANGED
@@ -9,7 +9,7 @@ export type AccountDeleteEvent = {
9
9
 
10
10
  /**
11
11
  * Per-account hard-delete. Purges ONE mail account's data (mailboxes,
12
- * messages, body parts, thread messages, S3 objects, search vectors) while
12
+ * messages, body parts, thread messages, S3 objects) while
13
13
  * keeping the AccountConfig/tenant and any sibling accounts intact. Emitted
14
14
  * by the API on `deleteAccount`; consumed by the fanout worker.
15
15
  */
@@ -1,7 +1,7 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { Readable } from "node:stream";
3
3
  import { describe, it } from "node:test";
4
- import { noopLogger } from "@remit/logger-lambda/noop-logger";
4
+ import { noopLogger } from "@remit/logger";
5
5
  import type { StorageService } from "@remit/storage-service";
6
6
  import type { CascadeServices } from "../cascade.js";
7
7
  import type { AccountExportEvent } from "../events.js";
@@ -1,5 +1,5 @@
1
1
  import { PassThrough } from "node:stream";
2
- import type { Logger } from "@remit/logger-lambda";
2
+ import type { Logger } from "@remit/logger";
3
3
  import type { StorageService } from "@remit/storage-service";
4
4
  import { ZipArchive } from "archiver";
5
5
  import type { CascadeServices } from "../cascade.js";
@@ -1,7 +1,7 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { afterEach, describe, it } from "node:test";
3
3
  import type { SQSClient } from "@aws-sdk/client-sqs";
4
- import { noopLogger } from "@remit/logger-lambda/noop-logger";
4
+ import { noopLogger } from "@remit/logger";
5
5
  import type { CascadeServices } from "../cascade.js";
6
6
  import type { AccountDeleteEvent } from "../events.js";
7
7
  import { processAccountFanout } from "./account-fanout.js";
@@ -32,7 +32,6 @@ const buildServices = (): CascadeServices =>
32
32
  threadMessageService: { listAllByAccount: async () => [] },
33
33
  accountSettingService: { listByAccountConfig: async () => [] },
34
34
  filterService: { listByAccountConfig: async () => [] },
35
- filterAnchorService: { get: async () => null },
36
35
  labelService: { listByAccountConfig: async () => [] },
37
36
  messageLabelService: { listByLabelId: async () => [] },
38
37
  }) as unknown as CascadeServices;
@@ -1,11 +1,10 @@
1
1
  import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
2
+ import { createLogger, type Logger } from "@remit/logger";
2
3
  import {
3
- createLogger,
4
- type Logger,
5
4
  queueNameFromEventSource,
6
5
  recordQueueEvent,
7
- withTelemetry,
8
- } from "@remit/logger-lambda";
6
+ } from "@remit/logger-lambda/metrics";
7
+ import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
9
8
  import type { SQSBatchResponse, SQSEvent, SQSHandler } from "aws-lambda";
10
9
  import type { CascadeServices } from "../cascade.js";
11
10
  import { enumerateCascadeEntities } from "../cascade.js";
@@ -30,6 +29,8 @@ import { processAccountDataPurge } from "./account-purge.js";
30
29
  import { processOrganizeJob } from "./organize-job.js";
31
30
  import { processSenderCategoryBackApply } from "./sender-category-backapply.js";
32
31
 
32
+ installNodeLogger();
33
+
33
34
  export interface ProcessAccountFanoutDeps {
34
35
  services?: CascadeServices;
35
36
  sqs?: SQSClient;
@@ -41,8 +42,8 @@ export interface ProcessAccountFanoutDeps {
41
42
 
42
43
  /**
43
44
  * Fanout step of the account-deletion cascade. Read-only: enumerates the
44
- * AccountConfig's data and dispatches downstream work — search-index
45
- * deletes, per-account imap-worker stops, a sign-out, and the finalize
45
+ * AccountConfig's data and dispatches downstream work — per-account
46
+ * imap-worker stops, a sign-out, and the finalize
46
47
  * enqueue. Throws on any non-swallowable error so SQS partial batch-failure
47
48
  * retries the whole record.
48
49
  */
@@ -1,11 +1,10 @@
1
1
  import { inspect } from "node:util";
2
+ import { createLogger, type Logger } from "@remit/logger";
2
3
  import {
3
- createLogger,
4
- type Logger,
5
4
  queueNameFromEventSource,
6
5
  recordQueueEvent,
7
- withTelemetry,
8
- } from "@remit/logger-lambda";
6
+ } from "@remit/logger-lambda/metrics";
7
+ import { installNodeLogger, withTelemetry } from "@remit/logger-lambda/node";
9
8
  import type { SQSEvent, SQSHandler } from "aws-lambda";
10
9
  import {
11
10
  type CascadeEntity,
@@ -24,6 +23,8 @@ import type {
24
23
  AccountFinalizeEvent,
25
24
  } from "../events.js";
26
25
 
26
+ installNodeLogger();
27
+
27
28
  export interface ProcessFinalizeDeps {
28
29
  capabilities?: DeletionCapabilities;
29
30
  cascadeServices?: CascadeServices;
@@ -131,10 +132,7 @@ export const processAccountFinalize = async (
131
132
  * the work; this worker only ever consumes.
132
133
  *
133
134
  * Idempotency: the cascade on a missing key is a no-op, so a redelivered batch
134
- * (or container) no-ops on already-gone rows. Search-index cleanup is not driven
135
- * from here: on DynamoDB the fanout step enqueues a REMOVE per message id up
136
- * front; on the relational backends the cascade emits a `message.removed` outbox
137
- * row the search-index worker relays.
135
+ * (or container) no-ops on already-gone rows.
138
136
  */
139
137
  export const processAccountDataPurgeFinalize = async (
140
138
  event: AccountDataPurgeFinalizeEvent,
@@ -1,7 +1,7 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { describe, it } from "node:test";
3
3
  import type { SQSClient } from "@aws-sdk/client-sqs";
4
- import { noopLogger } from "@remit/logger-lambda/noop-logger";
4
+ import { noopLogger } from "@remit/logger";
5
5
  import type { CascadeServices } from "../cascade.js";
6
6
  import type { AccountDataPurgeEvent } from "../events.js";
7
7
  import {
@@ -12,28 +12,19 @@ import {
12
12
  interface Sent {
13
13
  queueUrl: string | undefined;
14
14
  body: Record<string, unknown>;
15
- entries?: Array<{ Id: string; MessageBody: string }>;
16
15
  }
17
16
 
18
- const recordingSqs = (sent: Sent[], failBatch = false): SQSClient =>
17
+ const recordingSqs = (sent: Sent[]): SQSClient =>
19
18
  ({
20
19
  send: async (command: {
21
- input: {
22
- QueueUrl?: string;
23
- MessageBody?: string;
24
- Entries?: Array<{ Id: string; MessageBody: string }>;
25
- };
20
+ input: { QueueUrl?: string; MessageBody?: string };
26
21
  }) => {
27
22
  sent.push({
28
23
  queueUrl: command.input.QueueUrl,
29
24
  body: command.input.MessageBody
30
25
  ? JSON.parse(command.input.MessageBody)
31
26
  : {},
32
- entries: command.input.Entries,
33
27
  });
34
- if (command.input.Entries && failBatch) {
35
- return { Failed: [{ Id: "0" }] };
36
- }
37
28
  return {};
38
29
  },
39
30
  }) as unknown as SQSClient;
@@ -73,19 +64,14 @@ const baseDeps = (
73
64
  ]),
74
65
  sqs: recordingSqs(sent),
75
66
  accountPurgeDeleteQueueUrl: "http://queue/purge",
76
- searchIndexQueueUrl: "http://queue/search",
77
- dataBackend: "dynamodb",
78
67
  ...overrides,
79
68
  });
80
69
 
81
70
  describe("processAccountDataPurge", () => {
82
- it("enqueues vector deletes then a subtrees batch and one container leftover", async () => {
71
+ it("enqueues a subtrees batch and one container leftover", async () => {
83
72
  const sent: Sent[] = [];
84
73
  await processAccountDataPurge(event, noopLogger, baseDeps(sent));
85
74
 
86
- const vectorBatch = sent.find((m) => m.queueUrl === "http://queue/search");
87
- assert.equal(vectorBatch?.entries?.length, 2, "one entry per message");
88
-
89
75
  const finalize = sent.filter((m) => m.queueUrl === "http://queue/purge");
90
76
  const kinds = finalize.map((m) => m.body.kind);
91
77
  assert.deepEqual(kinds, ["subtrees", "container"]);
@@ -96,27 +82,6 @@ describe("processAccountDataPurge", () => {
96
82
  );
97
83
  });
98
84
 
99
- it("skips vector deletes on the relational backend, still enqueuing finalize", async () => {
100
- const sent: Sent[] = [];
101
- await processAccountDataPurge(
102
- event,
103
- noopLogger,
104
- baseDeps(sent, { dataBackend: "sqlite" }),
105
- );
106
-
107
- assert.equal(
108
- sent.some((m) => m.queueUrl === "http://queue/search"),
109
- false,
110
- "no search-index enqueue on the relational backend",
111
- );
112
- assert.deepEqual(
113
- sent
114
- .filter((m) => m.queueUrl === "http://queue/purge")
115
- .map((m) => m.body.kind),
116
- ["subtrees", "container"],
117
- );
118
- });
119
-
120
85
  it("drops manifest rows outside the account's mailbox set", async () => {
121
86
  const sent: Sent[] = [];
122
87
  await processAccountDataPurge(event, noopLogger, {
@@ -167,15 +132,4 @@ describe("processAccountDataPurge", () => {
167
132
  /dynamo throttled/,
168
133
  );
169
134
  });
170
-
171
- it("throws when a vector-delete batch reports failed entries", async () => {
172
- const sent: Sent[] = [];
173
- await assert.rejects(
174
- processAccountDataPurge(event, noopLogger, {
175
- ...baseDeps(sent),
176
- sqs: recordingSqs(sent, true),
177
- }),
178
- /failed entries/,
179
- );
180
- });
181
135
  });
@@ -1,19 +1,9 @@
1
- import { randomUUID } from "node:crypto";
2
- import {
3
- SendMessageBatchCommand,
4
- SendMessageCommand,
5
- type SQSClient,
6
- } from "@aws-sdk/client-sqs";
7
- import { Message } from "@remit/electrodb-entities";
8
- import type { Logger } from "@remit/logger-lambda";
9
- import type { SearchIndexMessage } from "@remit/search-index-worker";
10
- import { Entity } from "electrodb";
1
+ import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
2
+ import type { Logger } from "@remit/logger";
11
3
  import type { CascadeServices } from "../cascade.js";
12
4
  import {
13
- dataBackend as defaultDataBackend,
14
5
  getAccountPurgeDeleteQueueUrl,
15
6
  getCascadeServices,
16
- getSearchIndexQueueUrl,
17
7
  getSqsClient,
18
8
  } from "../config.js";
19
9
  import type {
@@ -26,12 +16,8 @@ export interface ProcessPurgeFanoutDeps {
26
16
  services?: CascadeServices;
27
17
  sqs?: SQSClient;
28
18
  accountPurgeDeleteQueueUrl?: string;
29
- searchIndexQueueUrl?: string;
30
- dataBackend?: string;
31
19
  }
32
20
 
33
- const SQS_BATCH_SIZE = 10;
34
-
35
21
  /**
36
22
  * Message subtrees carried per FIFO finalize message. Each item is two ids
37
23
  * (~70 bytes of JSON), so 100 stays far under the 256 KB SQS limit, and the
@@ -40,59 +26,6 @@ const SQS_BATCH_SIZE = 10;
40
26
  */
41
27
  const SUBTREE_BATCH_SIZE = 100;
42
28
 
43
- const messageEntity = new Entity(Message, { table: "purge-keys" });
44
-
45
- const messageKeys = (messageId: string): { pk: string; sk: string } => {
46
- const { Key } = messageEntity.delete({ messageId }).params<{
47
- Key: { pk: string; sk: string };
48
- }>();
49
- return Key;
50
- };
51
-
52
- const buildVectorDeleteEvent = (
53
- accountId: string,
54
- messageId: string,
55
- ): SearchIndexMessage => ({
56
- eventName: "REMOVE",
57
- entity: "Message",
58
- eventID: randomUUID(),
59
- eventTimestamp: Date.now(),
60
- accountId,
61
- keys: messageKeys(messageId),
62
- messageId,
63
- });
64
-
65
- const enqueueVectorDeletes = async (
66
- sqs: SQSClient,
67
- queueUrl: string,
68
- accountId: string,
69
- messageIds: string[],
70
- ): Promise<void> => {
71
- for (let i = 0; i < messageIds.length; i += SQS_BATCH_SIZE) {
72
- const batch = messageIds.slice(i, i + SQS_BATCH_SIZE);
73
- const result = await sqs.send(
74
- new SendMessageBatchCommand({
75
- QueueUrl: queueUrl,
76
- Entries: batch.map((messageId, idx) => ({
77
- Id: `${i + idx}`,
78
- MessageBody: JSON.stringify(
79
- buildVectorDeleteEvent(accountId, messageId),
80
- ),
81
- })),
82
- }),
83
- );
84
- // SendMessageBatch returns HTTP 200 with per-entry failures in `Failed`;
85
- // throw so the failed deletes surface to the DLQ and the whole fanout
86
- // replays idempotently rather than silently skipping vector deletes.
87
- if (result.Failed?.length) {
88
- const failedIds = result.Failed.map((f) => f.Id).join(", ");
89
- throw new Error(
90
- `Search-index vector-delete batch had ${result.Failed.length} failed entries: ${failedIds}`,
91
- );
92
- }
93
- }
94
- };
95
-
96
29
  /**
97
30
  * Enqueue the destructive purge work onto the FIFO finalize queue, single
98
31
  * message group per account so the worker processes everything strictly in
@@ -141,9 +74,8 @@ const enqueuePurgeFinalize = async (
141
74
  * manifest (`pk = accountConfigId`, scoped to this account's mailbox set) —
142
75
  * since the threading-visibility fix every persisted Message has exactly one
143
76
  * manifest row, so the partition is a complete, describe-free index of the
144
- * account's messages. Enqueues the per-message search-index (vector) deletes
145
- * (#457), then hands the destructive DDB+S3+CloudFront work to the finalize
146
- * worker as a stream of FIFO messages (subtree batches + one container
77
+ * account's messages. Hands the destructive DDB+S3+CloudFront work to the
78
+ * finalize worker as a stream of FIFO messages (subtree batches + one container
147
79
  * leftover, single message group). No self-loop, no recursion: the producer
148
80
  * enqueues all the work and the worker only ever consumes (#1069).
149
81
  *
@@ -151,8 +83,8 @@ const enqueuePurgeFinalize = async (
151
83
  * its sibling accounts, the tenant-shared Address rows, and the soft-deleted
152
84
  * account row are untouched.
153
85
  *
154
- * Replay-safe: the manifest read is read-only and the search-index/finalize
155
- * enqueues are idempotent. A redelivery after the account's rows are gone reads
86
+ * Replay-safe: the manifest read is read-only and the finalize enqueues are
87
+ * idempotent. A redelivery after the account's rows are gone reads
156
88
  * an empty manifest and enqueues only a container leftover that no-ops.
157
89
  */
158
90
  export const processAccountDataPurge = async (
@@ -162,13 +94,8 @@ export const processAccountDataPurge = async (
162
94
  ): Promise<void> => {
163
95
  const { accountId, accountConfigId } = event;
164
96
  const services = deps.services ?? (await getCascadeServices());
165
- const sqsFor = (queueUrl: string): SQSClient =>
166
- deps.sqs ?? getSqsClient(queueUrl);
167
97
  const accountPurgeDeleteQueueUrl =
168
98
  deps.accountPurgeDeleteQueueUrl ?? getAccountPurgeDeleteQueueUrl();
169
- const searchIndexQueueUrl =
170
- deps.searchIndexQueueUrl ?? getSearchIndexQueueUrl();
171
- const dataBackend = deps.dataBackend ?? defaultDataBackend;
172
99
 
173
100
  let mailboxIds: Set<string>;
174
101
  try {
@@ -194,27 +121,8 @@ export const processAccountDataPurge = async (
194
121
  messageId: tm.messageId,
195
122
  }));
196
123
 
197
- const messageIds = [...new Set(items.map((i) => i.messageId))];
198
-
199
- // On the relational backend the destructive delete emits a `message.removed`
200
- // outbox row per message, which the search-index worker relays as a
201
- // search-index REMOVE; enqueuing here too would double the removal. On
202
- // DynamoDB this fanout is the sole producer of the vector deletes (#457).
203
- if (dataBackend !== "sqlite") {
204
- await enqueueVectorDeletes(
205
- sqsFor(searchIndexQueueUrl),
206
- searchIndexQueueUrl,
207
- accountId,
208
- messageIds,
209
- );
210
- log.info(
211
- { accountConfigId, accountId, count: messageIds.length },
212
- "Enqueued search-index deletes for purged account",
213
- );
214
- }
215
-
216
124
  await enqueuePurgeFinalize(
217
- sqsFor(accountPurgeDeleteQueueUrl),
125
+ deps.sqs ?? getSqsClient(accountPurgeDeleteQueueUrl),
218
126
  accountPurgeDeleteQueueUrl,
219
127
  accountId,
220
128
  accountConfigId,
@@ -1,18 +1,19 @@
1
1
  import assert from "node:assert/strict";
2
2
  import { randomUUID } from "node:crypto";
3
- import { readFileSync } from "node:fs";
4
3
  import { afterEach, describe, it } from "node:test";
5
4
  import type { RemitClient } from "@remit/backend/client";
6
5
  import {
7
6
  buildOrganizeMatchDeps,
8
7
  type OrganizeMatchDeps,
9
- type OrganizeSemanticDeps,
10
8
  } from "@remit/backend/organize";
11
- import type { FilterAnchorItem, FilterItem } from "@remit/data-ports";
9
+ import type { FilterItem } from "@remit/data-ports";
12
10
  import { NotFoundError } from "@remit/data-ports/errors";
13
- import { DrizzleThreadMessageRepository } from "@remit/drizzle-service";
11
+ import {
12
+ DrizzleThreadMessageRepository,
13
+ SearchIndex,
14
+ } from "@remit/drizzle-service";
14
15
  import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
15
- import { noopLogger } from "@remit/logger-lambda/noop-logger";
16
+ import { noopLogger } from "@remit/logger";
16
17
  import type { OrganizeJobEvent } from "../events.js";
17
18
  import {
18
19
  type ProcessOrganizeJobDeps,
@@ -43,10 +44,8 @@ const jobClient = (
43
44
  organizeJobId: "job-1",
44
45
  accountConfigId: "cfg-1",
45
46
  filterId: "None",
46
- anchorMessageId: "None",
47
47
  matchOperator: "And",
48
48
  literalClauses,
49
- similarityThreshold: 0.75,
50
49
  actionLabelId: "lbl-1",
51
50
  actionMailboxId: "None",
52
51
  }),
@@ -63,20 +62,10 @@ const jobClient = (
63
62
  */
64
63
  const explodingMatchDeps = (): OrganizeMatchDeps =>
65
64
  ({
66
- semantic: () => {
67
- throw new Error("the vector pipeline must not be reached");
68
- },
69
65
  listAccountFilterMessages: async () => {
70
66
  throw new Error("SQLITE_BUSY");
71
67
  },
72
- filterAnchors: {
73
- get: async () => null,
74
- listByAccountConfig: async () => [],
75
- put: async () => {
76
- throw new Error("unreachable");
77
- },
78
- },
79
- }) as unknown as OrganizeMatchDeps;
68
+ }) as OrganizeMatchDeps;
80
69
 
81
70
  describe("processOrganizeJob (reader #463)", () => {
82
71
  it("fails the job on a refused rule and acknowledges the record", async () => {
@@ -136,7 +125,6 @@ const standingFilter = (over: Partial<FilterItem> = {}): FilterItem => ({
136
125
  scope: "Standing",
137
126
  state: "Active",
138
127
  disabledReason: "None",
139
- hasAnchor: false,
140
128
  ruleChangedAt: 1,
141
129
  actionChangedAt: 1,
142
130
  matchOperator: "And",
@@ -187,28 +175,21 @@ interface World {
187
175
  inbox: string;
188
176
  archive: string;
189
177
  updates: Update[];
178
+ filterUpdates: Array<{ filterId: string; patch: Record<string, unknown> }>;
190
179
  moves: Move[];
191
180
  labeled: string[];
192
181
  close: () => void;
193
182
  }
194
183
 
195
- const world = async (
196
- filters: FilterItem[],
197
- anchors: FilterAnchorItem[] = [],
198
- ): Promise<World> => {
184
+ const world = async (filters: FilterItem[]): Promise<World> => {
199
185
  const store = createShippedSqliteDb();
200
- store.sqlite.exec(
201
- readFileSync(
202
- new URL(
203
- "../../../../npm-scripts/sqlite-search-index.sql",
204
- import.meta.url,
205
- ),
206
- "utf8",
207
- ),
186
+ const threadMessage = new DrizzleThreadMessageRepository(
187
+ store.db as never,
188
+ new SearchIndex(store.db as never, store.db as never),
208
189
  );
209
- const threadMessage = new DrizzleThreadMessageRepository(store.db as never);
210
190
  const { inbox, archive } = await seedThreads(threadMessage);
211
191
  const updates: Update[] = [];
192
+ const filterUpdates: World["filterUpdates"] = [];
212
193
  const labeled: string[] = [];
213
194
  const mailboxOf: Record<string, string> = {
214
195
  [inbox]: INBOX,
@@ -221,10 +202,8 @@ const world = async (
221
202
  organizeJobId: "job-1",
222
203
  accountConfigId: CONFIG,
223
204
  filterId: "flt-1",
224
- anchorMessageId: "None",
225
205
  matchOperator: "And",
226
206
  literalClauses: [],
227
- similarityThreshold: 0.75,
228
207
  actionLabelId: "None",
229
208
  actionMailboxId: "None",
230
209
  }),
@@ -241,13 +220,21 @@ const world = async (
241
220
  refreshExpiry: async (filter: FilterItem) => filter,
242
221
  listByAccountAndState: async (_cfg: string, state: string) =>
243
222
  filters.filter((f) => f.state === state),
244
- },
245
- filterAnchor: {
246
- get: async (_cfg: string, filterId: string) =>
247
- anchors.find((anchor) => anchor.filterId === filterId) ?? null,
223
+ update: async (
224
+ _cfg: string,
225
+ filterId: string,
226
+ patch: Record<string, unknown>,
227
+ ) => {
228
+ filterUpdates.push({ filterId, patch });
229
+ return {};
230
+ },
248
231
  },
249
232
  account: {
250
233
  listMailSourcesByAccountConfig: async () => [{ accountId: "acct-1" }],
234
+ get: async (accountId: string) => ({
235
+ accountId,
236
+ email: accountId === "acct-1" ? "me@example.com" : "me-too@example.com",
237
+ }),
251
238
  },
252
239
  mailboxSpecialUse: {
253
240
  findInboxMailbox: async () => ({ mailboxId: INBOX, fullPath: "INBOX" }),
@@ -258,7 +245,18 @@ const world = async (
258
245
  mailboxId: mailboxOf[messageId],
259
246
  }),
260
247
  },
261
- mailbox: { resolveAccountId: async () => "acct-1" },
248
+ mailbox: {
249
+ resolveAccountId: async (mailboxId: string) =>
250
+ mailboxId === "mbx-gone"
251
+ ? null
252
+ : mailboxId === "mbx-sibling-travel"
253
+ ? "acct-2"
254
+ : "acct-1",
255
+ get: async (_accountId: string, mailboxId: string) => ({
256
+ mailboxId,
257
+ fullPath: mailboxId === "mbx-sibling-travel" ? "Travel" : "Invoices",
258
+ }),
259
+ },
262
260
  messageLabel: {
263
261
  apply: async (input: { messageId: string }) => {
264
262
  labeled.push(input.messageId);
@@ -271,6 +269,7 @@ const world = async (
271
269
  inbox,
272
270
  archive,
273
271
  updates,
272
+ filterUpdates,
274
273
  moves: [],
275
274
  labeled,
276
275
  close: store.close,
@@ -295,50 +294,6 @@ const runFilterJob = (w: World) =>
295
294
  moveService: recordingMoves(w.moves),
296
295
  });
297
296
 
298
- const EMBEDDING_ID = "test-model@4";
299
- const ANCHOR_VECTOR = [1, 0, 0, 0];
300
- const ORTHOGONAL_VECTOR = [0, 1, 0, 0];
301
-
302
- const persistedAnchor: FilterAnchorItem = {
303
- filterId: "flt-1",
304
- accountConfigId: CONFIG,
305
- anchorMessageId: "msg-anchor",
306
- anchorEmbedding: ANCHOR_VECTOR,
307
- anchorEmbeddingId: EMBEDDING_ID,
308
- anchorSourceText: "Your invoice",
309
- createdAt: 0,
310
- updatedAt: 0,
311
- };
312
-
313
- /**
314
- * The vector index answers the anchor with the inbox invoice, and only when
315
- * asked about the inbox: the kNN read is where a similarity filter finds its
316
- * candidates.
317
- */
318
- const vectorMatchDeps = (w: World): OrganizeMatchDeps => ({
319
- ...buildOrganizeMatchDeps(w.client),
320
- semantic: () => ({
321
- buildAnchor: async () => {
322
- throw new Error("a filter widens on its persisted anchor");
323
- },
324
- vectorStore: {
325
- query: async (query: { filter?: { mailboxId?: string } }) =>
326
- query.filter?.mailboxId === INBOX
327
- ? [
328
- {
329
- chunkId: `${w.inbox}#body`,
330
- score: 0.9,
331
- metadata: { messageId: w.inbox, mailboxIds: [INBOX] },
332
- },
333
- ]
334
- : [],
335
- getByMessage: async () => [],
336
- } as unknown as OrganizeSemanticDeps["vectorStore"],
337
- embed: async () => ANCHOR_VECTOR,
338
- embeddingId: EMBEDDING_ID,
339
- }),
340
- });
341
-
342
297
  describe("processOrganizeJob for a filter job (#1354)", () => {
343
298
  let current: World | undefined;
344
299
  afterEach(() => current?.close());
@@ -390,6 +345,42 @@ describe("processOrganizeJob for a filter job (#1354)", () => {
390
345
  assert.equal(current.updates.at(-1)?.state, "Complete");
391
346
  });
392
347
 
348
+ it("skips another account's mail, as if the filter had not matched", async () => {
349
+ current = await world([
350
+ standingFilter({
351
+ actionLabelId: "lbl-trips",
352
+ actionMailboxId: "mbx-sibling-travel",
353
+ }),
354
+ ]);
355
+
356
+ await runFilterJob(current);
357
+
358
+ assert.deepEqual(current.moves, []);
359
+ assert.deepEqual(current.labeled, []);
360
+ assert.deepEqual(current.filterUpdates, []);
361
+ const done = current.updates.at(-1);
362
+ assert.equal(done?.state, "Complete");
363
+ assert.equal(done?.appliedCount, 0);
364
+ assert.equal(done?.failedCount, 0);
365
+ });
366
+
367
+ it("turns the filter off and fails the job when no account holds its folder", async () => {
368
+ current = await world([standingFilter({ actionMailboxId: "mbx-gone" })]);
369
+
370
+ await runFilterJob(current);
371
+
372
+ assert.deepEqual(current.moves, []);
373
+ assert.deepEqual(current.filterUpdates, [
374
+ {
375
+ filterId: "flt-1",
376
+ patch: { state: "Disabled", disabledReason: "FolderMissing" },
377
+ },
378
+ ]);
379
+ const done = current.updates.at(-1);
380
+ assert.equal(done?.state, "Failed");
381
+ assert.match(String(done?.errorMessage), /no longer exists/);
382
+ });
383
+
393
384
  it("fails the job without a retry when the filter is turned off", async () => {
394
385
  current = await world([
395
386
  standingFilter({ state: "Disabled", disabledReason: "UserDisabled" }),
@@ -410,32 +401,4 @@ describe("processOrganizeJob for a filter job (#1354)", () => {
410
401
  assert.equal(current.updates.at(-1)?.state, "Failed");
411
402
  assert.match(String(current.updates.at(-1)?.errorMessage), /deleted/);
412
403
  });
413
-
414
- it("moves a message the vector query found, though the preview scores below the threshold", async () => {
415
- current = await world(
416
- [standingFilter({ hasAnchor: true, literalClauses: [] })],
417
- [persistedAnchor],
418
- );
419
-
420
- // The index-time evaluator embeds subject and preview; this embedder puts
421
- // that text orthogonal to the anchor, so evaluate alone would call the
422
- // message a miss. It only orders the move, so the vector match stands.
423
- await processOrganizeJob(event, noopLogger, {
424
- client: current.client,
425
- matchDeps: vectorMatchDeps(current),
426
- moveService: recordingMoves(current.moves),
427
- embedder: {
428
- embed: async () => ORTHOGONAL_VECTOR,
429
- embeddingId: EMBEDDING_ID,
430
- },
431
- });
432
-
433
- assert.deepEqual(current.moves, [
434
- { messageId: current.inbox, destinationMailboxId: "mbx-invoices" },
435
- ]);
436
- const done = current.updates.at(-1);
437
- assert.equal(done?.state, "Complete");
438
- assert.equal(done?.matchedCount, 1);
439
- assert.equal(done?.appliedCount, 1);
440
- });
441
404
  });
@@ -5,7 +5,6 @@ import {
5
5
  backApplyFilter,
6
6
  buildOrganizeMatchDeps,
7
7
  buildOrganizeMoveService,
8
- type FilterBackApplyDeps,
9
8
  isFilterJob,
10
9
  matchOrganize,
11
10
  ORGANIZE_MATCH_LIMIT,
@@ -13,14 +12,13 @@ import {
13
12
  type OrganizeRejection,
14
13
  predicateFromJob,
15
14
  } from "@remit/backend/organize";
16
- import type { Logger } from "@remit/logger-lambda";
15
+ import type { Logger } from "@remit/logger";
17
16
  import type { OrganizeJobEvent } from "../events.js";
18
17
 
19
18
  export interface ProcessOrganizeJobDeps {
20
19
  client?: RemitClient;
21
20
  matchDeps?: OrganizeMatchDeps;
22
21
  moveService?: ApplyOrganizeDeps["moveService"];
23
- embedder?: FilterBackApplyDeps["embedder"];
24
22
  }
25
23
 
26
24
  interface OrganizeOutcome {
@@ -28,7 +26,7 @@ interface OrganizeOutcome {
28
26
  matched: number;
29
27
  applied: number;
30
28
  failed: number;
31
- semanticUnavailable: boolean;
29
+ failures: string[];
32
30
  }
33
31
 
34
32
  const rejectedOutcome = (rejected: OrganizeRejection): OrganizeOutcome => ({
@@ -36,7 +34,7 @@ const rejectedOutcome = (rejected: OrganizeRejection): OrganizeOutcome => ({
36
34
  matched: 0,
37
35
  applied: 0,
38
36
  failed: 0,
39
- semanticUnavailable: false,
37
+ failures: [],
40
38
  });
41
39
 
42
40
  /**
@@ -57,7 +55,7 @@ const runSnapshot = async (
57
55
  ORGANIZE_MATCH_LIMIT,
58
56
  );
59
57
  if (match.rejected) return rejectedOutcome(match.rejected);
60
- const { applied, failed } = await applyOrganize(
58
+ const { applied, failed, failures } = await applyOrganize(
61
59
  { client, moveService },
62
60
  job.accountConfigId,
63
61
  match.messageIds,
@@ -68,7 +66,7 @@ const runSnapshot = async (
68
66
  matched: match.messageIds.length,
69
67
  applied,
70
68
  failed,
71
- semanticUnavailable: match.semanticUnavailable,
69
+ failures,
72
70
  };
73
71
  };
74
72
 
@@ -81,7 +79,7 @@ const runSnapshot = async (
81
79
  *
82
80
  * Reuses the shared matcher (so the applied set equals what preview returned)
83
81
  * and the idempotent apply plumbing; `appliedByFilterId` is never written and no
84
- * Filter/FilterAnchor row is ever created. Both back-apply actions run here: the
82
+ * Filter row is ever created. Both back-apply actions run here: the
85
83
  * additive label upsert and the exclusive folder move, the latter through the
86
84
  * same local-first placement mover body sync uses (`buildOrganizeMoveService`),
87
85
  * so a redelivered job re-applies both idempotently.
@@ -114,7 +112,7 @@ export const processOrganizeJob = async (
114
112
  const moveService = deps.moveService ?? buildOrganizeMoveService(client);
115
113
  const outcome = isFilterJob(job)
116
114
  ? await backApplyFilter(
117
- { client, matchDeps, moveService, embedder: deps.embedder },
115
+ { client, matchDeps, moveService },
118
116
  accountConfigId,
119
117
  job.filterId,
120
118
  ).then((result) =>
@@ -137,12 +135,13 @@ export const processOrganizeJob = async (
137
135
  return;
138
136
  }
139
137
 
140
- const { matched, applied, failed, semanticUnavailable } = outcome;
138
+ const { matched, applied, failed, failures } = outcome;
141
139
  await client.organizeJobRequest.update(organizeJobId, {
142
140
  state: "Complete",
143
141
  matchedCount: matched,
144
142
  appliedCount: applied,
145
143
  failedCount: failed,
144
+ errorMessage: failures[0] ?? "",
146
145
  });
147
146
  log.info(
148
147
  {
@@ -152,7 +151,6 @@ export const processOrganizeJob = async (
152
151
  matched,
153
152
  applied,
154
153
  failed,
155
- semanticUnavailable,
156
154
  },
157
155
  "Organize back-apply complete",
158
156
  );
@@ -2,7 +2,7 @@ import assert from "node:assert/strict";
2
2
  import { describe, it } from "node:test";
3
3
  import type { RemitClient } from "@remit/backend/client";
4
4
  import { NotFoundError } from "@remit/data-ports/errors";
5
- import { noopLogger } from "@remit/logger-lambda/noop-logger";
5
+ import { noopLogger } from "@remit/logger";
6
6
  import type { SenderCategoryBackApplyEvent } from "../events.js";
7
7
  import { processSenderCategoryBackApply } from "./sender-category-backapply.js";
8
8
 
@@ -4,7 +4,7 @@ import {
4
4
  SENDER_CATEGORY_BACKAPPLY_LIMIT,
5
5
  } from "@remit/backend/sender-category";
6
6
  import { NotFoundError } from "@remit/data-ports/errors";
7
- import type { Logger } from "@remit/logger-lambda";
7
+ import type { Logger } from "@remit/logger";
8
8
  import type { SenderCategoryBackApplyEvent } from "../events.js";
9
9
 
10
10
  export interface ProcessSenderCategoryBackApplyDeps {
package/src/poller.ts CHANGED
@@ -1,8 +1,12 @@
1
- import { createLogger, startMetricsServer } from "@remit/logger-lambda";
1
+ import { createLogger } from "@remit/logger";
2
+ import { startMetricsServer } from "@remit/logger-lambda/metrics";
3
+ import { installNodeLogger } from "@remit/logger-lambda/node";
2
4
  import { runQueuePoller } from "@remit/sqs-client/poller";
3
5
  import { env } from "expect-env";
4
6
  import { fanoutHandler, finalizeHandler } from "./index.js";
5
7
 
8
+ installNodeLogger();
9
+
6
10
  /**
7
11
  * Production queue poller, and the standalone entrypoint for the dedicated
8
12
  * image. The sibling workers carry a separate `e2e-processor-shim.ts` because
@@ -1,77 +0,0 @@
1
- import assert from "node:assert";
2
- import { existsSync, readdirSync } from "node:fs";
3
- import { resolve } from "node:path";
4
- import { describe, it } from "node:test";
5
- import { COVERED_ENTITY_TYPES } from "./cascade.js";
6
-
7
- // The snapshot enumerates electrodb-service's models; that package is stripped
8
- // from the open-core tree, so the check skips there and runs where it ships.
9
- const modelsDir = resolve(
10
- import.meta.dirname,
11
- "../../remit-electrodb-service/src/models",
12
- );
13
-
14
- describe("cascade entity coverage snapshot", () => {
15
- it("covers every entity model in remit-electrodb-service", {
16
- skip: !existsSync(modelsDir),
17
- }, () => {
18
- const modelFiles = readdirSync(modelsDir)
19
- .filter(
20
- (f) => f.endsWith(".ts") && !f.endsWith(".test.ts") && f !== "index.ts",
21
- )
22
- .map((f) => f.replace(".ts", ""));
23
-
24
- // Map model file names to the entity types they contain
25
- const entityTypesByFile: Record<string, string[]> = {
26
- "account-config": ["AccountConfig"],
27
- "account-export-request": [],
28
- "account-setting": ["AccountSetting"],
29
- "account-setting-registry": [],
30
- account: ["Account"],
31
- address: ["Address", "EnvelopeAddress"],
32
- envelope: [
33
- "Envelope",
34
- "MessageReference",
35
- "BodyPart",
36
- "BodyPartParameter",
37
- "RawMessageStorage",
38
- "BodyPartStorage",
39
- "BodyPartContent",
40
- ],
41
- filter: ["Filter"],
42
- "filter-anchor": ["FilterAnchor"],
43
- label: ["Label"],
44
- mailbox: ["Mailbox"],
45
- "mailbox-lock": ["MailboxLock"],
46
- "mailbox-special-use": [],
47
- message: ["Message"],
48
- "message-flag": ["MessageFlag"],
49
- "message-flag-push": ["MessageFlagPush"],
50
- "message-label": ["MessageLabel"],
51
- "message-placement-move": ["MessagePlacementMove"],
52
- "organize-job-request": [],
53
- "outbox-message": ["OutboxMessage"],
54
- "thread-message": ["ThreadMessage"],
55
- "wellknown-rule": [],
56
- };
57
-
58
- const allEntityTypes = Object.values(entityTypesByFile).flat().sort();
59
- const coveredSorted = [...COVERED_ENTITY_TYPES].sort();
60
-
61
- assert.deepStrictEqual(
62
- coveredSorted,
63
- allEntityTypes,
64
- `Cascade does not cover all entity types.\n` +
65
- `Missing: ${allEntityTypes.filter((t) => !(coveredSorted as readonly string[]).includes(t)).join(", ")}\n` +
66
- `Extra: ${(coveredSorted as readonly string[]).filter((t) => !allEntityTypes.includes(t)).join(", ")}`,
67
- );
68
-
69
- // Verify all model files are accounted for
70
- const unmappedFiles = modelFiles.filter((f) => !(f in entityTypesByFile));
71
- assert.deepStrictEqual(
72
- unmappedFiles,
73
- [],
74
- `Model files not mapped to entity types: ${unmappedFiles.join(", ")}`,
75
- );
76
- });
77
- });