@remit/account-worker 0.0.25 → 0.0.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,19 +1,9 @@
1
- import { randomUUID } from "node:crypto";
2
- import {
3
- SendMessageBatchCommand,
4
- SendMessageCommand,
5
- type SQSClient,
6
- } from "@aws-sdk/client-sqs";
7
- import { Message } from "@remit/electrodb-entities";
8
- import type { Logger } from "@remit/logger-lambda";
9
- import type { SearchIndexMessage } from "@remit/search-index-worker";
10
- import { Entity } from "electrodb";
1
+ import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
2
+ import type { Logger } from "@remit/logger";
11
3
  import type { CascadeServices } from "../cascade.js";
12
4
  import {
13
- dataBackend as defaultDataBackend,
14
5
  getAccountPurgeDeleteQueueUrl,
15
6
  getCascadeServices,
16
- getSearchIndexQueueUrl,
17
7
  getSqsClient,
18
8
  } from "../config.js";
19
9
  import type {
@@ -26,12 +16,8 @@ export interface ProcessPurgeFanoutDeps {
26
16
  services?: CascadeServices;
27
17
  sqs?: SQSClient;
28
18
  accountPurgeDeleteQueueUrl?: string;
29
- searchIndexQueueUrl?: string;
30
- dataBackend?: string;
31
19
  }
32
20
 
33
- const SQS_BATCH_SIZE = 10;
34
-
35
21
  /**
36
22
  * Message subtrees carried per FIFO finalize message. Each item is two ids
37
23
  * (~70 bytes of JSON), so 100 stays far under the 256 KB SQS limit, and the
@@ -40,59 +26,6 @@ const SQS_BATCH_SIZE = 10;
40
26
  */
41
27
  const SUBTREE_BATCH_SIZE = 100;
42
28
 
43
- const messageEntity = new Entity(Message, { table: "purge-keys" });
44
-
45
- const messageKeys = (messageId: string): { pk: string; sk: string } => {
46
- const { Key } = messageEntity.delete({ messageId }).params<{
47
- Key: { pk: string; sk: string };
48
- }>();
49
- return Key;
50
- };
51
-
52
- const buildVectorDeleteEvent = (
53
- accountId: string,
54
- messageId: string,
55
- ): SearchIndexMessage => ({
56
- eventName: "REMOVE",
57
- entity: "Message",
58
- eventID: randomUUID(),
59
- eventTimestamp: Date.now(),
60
- accountId,
61
- keys: messageKeys(messageId),
62
- messageId,
63
- });
64
-
65
- const enqueueVectorDeletes = async (
66
- sqs: SQSClient,
67
- queueUrl: string,
68
- accountId: string,
69
- messageIds: string[],
70
- ): Promise<void> => {
71
- for (let i = 0; i < messageIds.length; i += SQS_BATCH_SIZE) {
72
- const batch = messageIds.slice(i, i + SQS_BATCH_SIZE);
73
- const result = await sqs.send(
74
- new SendMessageBatchCommand({
75
- QueueUrl: queueUrl,
76
- Entries: batch.map((messageId, idx) => ({
77
- Id: `${i + idx}`,
78
- MessageBody: JSON.stringify(
79
- buildVectorDeleteEvent(accountId, messageId),
80
- ),
81
- })),
82
- }),
83
- );
84
- // SendMessageBatch returns HTTP 200 with per-entry failures in `Failed`;
85
- // throw so the failed deletes surface to the DLQ and the whole fanout
86
- // replays idempotently rather than silently skipping vector deletes.
87
- if (result.Failed?.length) {
88
- const failedIds = result.Failed.map((f) => f.Id).join(", ");
89
- throw new Error(
90
- `Search-index vector-delete batch had ${result.Failed.length} failed entries: ${failedIds}`,
91
- );
92
- }
93
- }
94
- };
95
-
96
29
  /**
97
30
  * Enqueue the destructive purge work onto the FIFO finalize queue, single
98
31
  * message group per account so the worker processes everything strictly in
@@ -141,9 +74,8 @@ const enqueuePurgeFinalize = async (
141
74
  * manifest (`pk = accountConfigId`, scoped to this account's mailbox set) —
142
75
  * since the threading-visibility fix every persisted Message has exactly one
143
76
  * manifest row, so the partition is a complete, describe-free index of the
144
- * account's messages. Enqueues the per-message search-index (vector) deletes
145
- * (#457), then hands the destructive DDB+S3+CloudFront work to the finalize
146
- * worker as a stream of FIFO messages (subtree batches + one container
77
+ * account's messages. Hands the destructive DDB+S3+CloudFront work to the
78
+ * finalize worker as a stream of FIFO messages (subtree batches + one container
147
79
  * leftover, single message group). No self-loop, no recursion: the producer
148
80
  * enqueues all the work and the worker only ever consumes (#1069).
149
81
  *
@@ -151,8 +83,8 @@ const enqueuePurgeFinalize = async (
151
83
  * its sibling accounts, the tenant-shared Address rows, and the soft-deleted
152
84
  * account row are untouched.
153
85
  *
154
- * Replay-safe: the manifest read is read-only and the search-index/finalize
155
- * enqueues are idempotent. A redelivery after the account's rows are gone reads
86
+ * Replay-safe: the manifest read is read-only and the finalize enqueues are
87
+ * idempotent. A redelivery after the account's rows are gone reads
156
88
  * an empty manifest and enqueues only a container leftover that no-ops.
157
89
  */
158
90
  export const processAccountDataPurge = async (
@@ -162,13 +94,8 @@ export const processAccountDataPurge = async (
162
94
  ): Promise<void> => {
163
95
  const { accountId, accountConfigId } = event;
164
96
  const services = deps.services ?? (await getCascadeServices());
165
- const sqsFor = (queueUrl: string): SQSClient =>
166
- deps.sqs ?? getSqsClient(queueUrl);
167
97
  const accountPurgeDeleteQueueUrl =
168
98
  deps.accountPurgeDeleteQueueUrl ?? getAccountPurgeDeleteQueueUrl();
169
- const searchIndexQueueUrl =
170
- deps.searchIndexQueueUrl ?? getSearchIndexQueueUrl();
171
- const dataBackend = deps.dataBackend ?? defaultDataBackend;
172
99
 
173
100
  let mailboxIds: Set<string>;
174
101
  try {
@@ -194,27 +121,8 @@ export const processAccountDataPurge = async (
194
121
  messageId: tm.messageId,
195
122
  }));
196
123
 
197
- const messageIds = [...new Set(items.map((i) => i.messageId))];
198
-
199
- // On the relational backend the destructive delete emits a `message.removed`
200
- // outbox row per message, which the search-index worker relays as a
201
- // search-index REMOVE; enqueuing here too would double the removal. On
202
- // DynamoDB this fanout is the sole producer of the vector deletes (#457).
203
- if (dataBackend !== "sqlite") {
204
- await enqueueVectorDeletes(
205
- sqsFor(searchIndexQueueUrl),
206
- searchIndexQueueUrl,
207
- accountId,
208
- messageIds,
209
- );
210
- log.info(
211
- { accountConfigId, accountId, count: messageIds.length },
212
- "Enqueued search-index deletes for purged account",
213
- );
214
- }
215
-
216
124
  await enqueuePurgeFinalize(
217
- sqsFor(accountPurgeDeleteQueueUrl),
125
+ deps.sqs ?? getSqsClient(accountPurgeDeleteQueueUrl),
218
126
  accountPurgeDeleteQueueUrl,
219
127
  accountId,
220
128
  accountConfigId,
@@ -1,10 +1,24 @@
1
1
  import assert from "node:assert/strict";
2
- import { describe, it } from "node:test";
2
+ import { randomUUID } from "node:crypto";
3
+ import { afterEach, describe, it } from "node:test";
3
4
  import type { RemitClient } from "@remit/backend/client";
4
- import type { OrganizeMatchDeps } from "@remit/backend/organize";
5
- import { noopLogger } from "@remit/logger-lambda/noop-logger";
5
+ import {
6
+ buildOrganizeMatchDeps,
7
+ type OrganizeMatchDeps,
8
+ } from "@remit/backend/organize";
9
+ import type { FilterItem } from "@remit/data-ports";
10
+ import { NotFoundError } from "@remit/data-ports/errors";
11
+ import {
12
+ DrizzleThreadMessageRepository,
13
+ SearchIndex,
14
+ } from "@remit/drizzle-service";
15
+ import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
16
+ import { noopLogger } from "@remit/logger";
6
17
  import type { OrganizeJobEvent } from "../events.js";
7
- import { processOrganizeJob } from "./organize-job.js";
18
+ import {
19
+ type ProcessOrganizeJobDeps,
20
+ processOrganizeJob,
21
+ } from "./organize-job.js";
8
22
 
9
23
  interface Update {
10
24
  state: string;
@@ -29,10 +43,9 @@ const jobClient = (
29
43
  get: async () => ({
30
44
  organizeJobId: "job-1",
31
45
  accountConfigId: "cfg-1",
32
- anchorMessageId: "None",
46
+ filterId: "None",
33
47
  matchOperator: "And",
34
48
  literalClauses,
35
- similarityThreshold: 0.75,
36
49
  actionLabelId: "lbl-1",
37
50
  actionMailboxId: "None",
38
51
  }),
@@ -49,19 +62,10 @@ const jobClient = (
49
62
  */
50
63
  const explodingMatchDeps = (): OrganizeMatchDeps =>
51
64
  ({
52
- semantic: () => {
53
- throw new Error("the vector pipeline must not be reached");
54
- },
55
65
  listAccountFilterMessages: async () => {
56
66
  throw new Error("SQLITE_BUSY");
57
67
  },
58
- filterAnchors: {
59
- listByAccountConfig: async () => [],
60
- put: async () => {
61
- throw new Error("unreachable");
62
- },
63
- },
64
- }) as unknown as OrganizeMatchDeps;
68
+ }) as OrganizeMatchDeps;
65
69
 
66
70
  describe("processOrganizeJob (reader #463)", () => {
67
71
  it("fails the job on a refused rule and acknowledges the record", async () => {
@@ -104,3 +108,297 @@ describe("processOrganizeJob (reader #463)", () => {
104
108
  assert.equal(updates[1]?.errorMessage, "SQLITE_BUSY");
105
109
  });
106
110
  });
111
+
112
+ interface Move {
113
+ messageId: string;
114
+ destinationMailboxId: string;
115
+ }
116
+
117
+ const CONFIG = "cfg-1";
118
+ const INBOX = "mbx-inbox";
119
+ const ARCHIVE = "mbx-archive";
120
+
121
+ const standingFilter = (over: Partial<FilterItem> = {}): FilterItem => ({
122
+ filterId: "flt-1",
123
+ accountConfigId: CONFIG,
124
+ name: "Invoices",
125
+ scope: "Standing",
126
+ state: "Active",
127
+ disabledReason: "None",
128
+ ruleChangedAt: 1,
129
+ actionChangedAt: 1,
130
+ matchOperator: "And",
131
+ literalClauses: [{ field: "From", value: "billing@example.com" }],
132
+ actionLabelId: "None",
133
+ actionMailboxId: "mbx-invoices",
134
+ createdAt: 0,
135
+ updatedAt: 0,
136
+ ...over,
137
+ });
138
+
139
+ /**
140
+ * The thread rows live in the shipped SQLite schema, read through the real
141
+ * repository, so the inbox narrowing is the store's own query. One invoice
142
+ * sits in the inbox and one in the archive, both from the filter's sender
143
+ * and both saying "overdue" in their preview.
144
+ */
145
+ const seedThreads = async (
146
+ threadMessage: DrizzleThreadMessageRepository,
147
+ ): Promise<{ inbox: string; archive: string }> => {
148
+ const seed = async (mailboxId: string, uid: number): Promise<string> => {
149
+ const messageId = randomUUID();
150
+ await threadMessage.create({
151
+ accountConfigId: CONFIG,
152
+ threadId: randomUUID(),
153
+ messageId,
154
+ mailboxId,
155
+ uid,
156
+ referenceOrder: 0,
157
+ internalDate: uid,
158
+ sentDate: uid,
159
+ subject: "Your invoice",
160
+ fromEmail: "billing@example.com",
161
+ fromName: "Billing",
162
+ snippet: "This invoice is overdue",
163
+ isRead: false,
164
+ isDeleted: false,
165
+ hasAttachment: false,
166
+ hasStars: false,
167
+ });
168
+ return messageId;
169
+ };
170
+ return { inbox: await seed(INBOX, 1), archive: await seed(ARCHIVE, 2) };
171
+ };
172
+
173
+ interface World {
174
+ client: RemitClient;
175
+ inbox: string;
176
+ archive: string;
177
+ updates: Update[];
178
+ filterUpdates: Array<{ filterId: string; patch: Record<string, unknown> }>;
179
+ moves: Move[];
180
+ labeled: string[];
181
+ close: () => void;
182
+ }
183
+
184
+ const world = async (filters: FilterItem[]): Promise<World> => {
185
+ const store = createShippedSqliteDb();
186
+ const threadMessage = new DrizzleThreadMessageRepository(
187
+ store.db as never,
188
+ new SearchIndex(store.db as never, store.db as never),
189
+ );
190
+ const { inbox, archive } = await seedThreads(threadMessage);
191
+ const updates: Update[] = [];
192
+ const filterUpdates: World["filterUpdates"] = [];
193
+ const labeled: string[] = [];
194
+ const mailboxOf: Record<string, string> = {
195
+ [inbox]: INBOX,
196
+ [archive]: ARCHIVE,
197
+ };
198
+ const client = {
199
+ threadMessage,
200
+ organizeJobRequest: {
201
+ get: async () => ({
202
+ organizeJobId: "job-1",
203
+ accountConfigId: CONFIG,
204
+ filterId: "flt-1",
205
+ matchOperator: "And",
206
+ literalClauses: [],
207
+ actionLabelId: "None",
208
+ actionMailboxId: "None",
209
+ }),
210
+ update: async (_id: string, patch: Update) => {
211
+ updates.push(patch);
212
+ },
213
+ },
214
+ filter: {
215
+ get: async (_cfg: string, filterId: string) => {
216
+ const found = filters.find((f) => f.filterId === filterId);
217
+ if (!found) throw new NotFoundError(`Filter not found: ${filterId}`);
218
+ return found;
219
+ },
220
+ refreshExpiry: async (filter: FilterItem) => filter,
221
+ listByAccountAndState: async (_cfg: string, state: string) =>
222
+ filters.filter((f) => f.state === state),
223
+ update: async (
224
+ _cfg: string,
225
+ filterId: string,
226
+ patch: Record<string, unknown>,
227
+ ) => {
228
+ filterUpdates.push({ filterId, patch });
229
+ return {};
230
+ },
231
+ },
232
+ account: {
233
+ listMailSourcesByAccountConfig: async () => [{ accountId: "acct-1" }],
234
+ get: async (accountId: string) => ({
235
+ accountId,
236
+ email: accountId === "acct-1" ? "me@example.com" : "me-too@example.com",
237
+ }),
238
+ },
239
+ mailboxSpecialUse: {
240
+ findInboxMailbox: async () => ({ mailboxId: INBOX, fullPath: "INBOX" }),
241
+ },
242
+ message: {
243
+ get: async (messageId: string) => ({
244
+ messageId,
245
+ mailboxId: mailboxOf[messageId],
246
+ }),
247
+ },
248
+ mailbox: {
249
+ resolveAccountId: async (mailboxId: string) =>
250
+ mailboxId === "mbx-gone"
251
+ ? null
252
+ : mailboxId === "mbx-sibling-travel"
253
+ ? "acct-2"
254
+ : "acct-1",
255
+ get: async (_accountId: string, mailboxId: string) => ({
256
+ mailboxId,
257
+ fullPath: mailboxId === "mbx-sibling-travel" ? "Travel" : "Invoices",
258
+ }),
259
+ },
260
+ messageLabel: {
261
+ apply: async (input: { messageId: string }) => {
262
+ labeled.push(input.messageId);
263
+ return {};
264
+ },
265
+ },
266
+ } as unknown as RemitClient;
267
+ return {
268
+ client,
269
+ inbox,
270
+ archive,
271
+ updates,
272
+ filterUpdates,
273
+ moves: [],
274
+ labeled,
275
+ close: store.close,
276
+ };
277
+ };
278
+
279
+ const recordingMoves = (moves: Move[]) =>
280
+ ({
281
+ moveMessage: async (
282
+ _cfg: string,
283
+ messageId: string,
284
+ destinationMailboxId: string,
285
+ ) => {
286
+ moves.push({ messageId, destinationMailboxId });
287
+ },
288
+ }) as unknown as NonNullable<ProcessOrganizeJobDeps["moveService"]>;
289
+
290
+ const runFilterJob = (w: World) =>
291
+ processOrganizeJob(event, noopLogger, {
292
+ client: w.client,
293
+ matchDeps: buildOrganizeMatchDeps(w.client),
294
+ moveService: recordingMoves(w.moves),
295
+ });
296
+
297
+ describe("processOrganizeJob for a filter job (#1354)", () => {
298
+ let current: World | undefined;
299
+ afterEach(() => current?.close());
300
+
301
+ it("moves the pre-existing matching inbox message and leaves the archive alone", async () => {
302
+ current = await world([standingFilter()]);
303
+
304
+ await runFilterJob(current);
305
+
306
+ assert.deepEqual(current.moves, [
307
+ { messageId: current.inbox, destinationMailboxId: "mbx-invoices" },
308
+ ]);
309
+ const done = current.updates.at(-1);
310
+ assert.equal(done?.state, "Complete");
311
+ assert.equal(done?.matchedCount, 1);
312
+ assert.equal(done?.appliedCount, 1);
313
+ });
314
+
315
+ it("matches a free-text rule from search on the stored preview", async () => {
316
+ current = await world([
317
+ standingFilter({
318
+ literalClauses: [{ field: "HasWords", value: "overdue" }],
319
+ }),
320
+ ]);
321
+
322
+ await runFilterJob(current);
323
+
324
+ assert.deepEqual(current.moves, [
325
+ { messageId: current.inbox, destinationMailboxId: "mbx-invoices" },
326
+ ]);
327
+ assert.equal(current.updates.at(-1)?.state, "Complete");
328
+ });
329
+
330
+ it("leaves the move to a newer filter that also matches, and still labels", async () => {
331
+ current = await world([
332
+ standingFilter({ actionLabelId: "lbl-bills" }),
333
+ standingFilter({
334
+ filterId: "flt-2",
335
+ name: "Receipts",
336
+ actionChangedAt: 2,
337
+ actionMailboxId: "mbx-receipts",
338
+ }),
339
+ ]);
340
+
341
+ await runFilterJob(current);
342
+
343
+ assert.deepEqual(current.moves, []);
344
+ assert.deepEqual(current.labeled, [current.inbox]);
345
+ assert.equal(current.updates.at(-1)?.state, "Complete");
346
+ });
347
+
348
+ it("skips another account's mail, as if the filter had not matched", async () => {
349
+ current = await world([
350
+ standingFilter({
351
+ actionLabelId: "lbl-trips",
352
+ actionMailboxId: "mbx-sibling-travel",
353
+ }),
354
+ ]);
355
+
356
+ await runFilterJob(current);
357
+
358
+ assert.deepEqual(current.moves, []);
359
+ assert.deepEqual(current.labeled, []);
360
+ assert.deepEqual(current.filterUpdates, []);
361
+ const done = current.updates.at(-1);
362
+ assert.equal(done?.state, "Complete");
363
+ assert.equal(done?.appliedCount, 0);
364
+ assert.equal(done?.failedCount, 0);
365
+ });
366
+
367
+ it("turns the filter off and fails the job when no account holds its folder", async () => {
368
+ current = await world([standingFilter({ actionMailboxId: "mbx-gone" })]);
369
+
370
+ await runFilterJob(current);
371
+
372
+ assert.deepEqual(current.moves, []);
373
+ assert.deepEqual(current.filterUpdates, [
374
+ {
375
+ filterId: "flt-1",
376
+ patch: { state: "Disabled", disabledReason: "FolderMissing" },
377
+ },
378
+ ]);
379
+ const done = current.updates.at(-1);
380
+ assert.equal(done?.state, "Failed");
381
+ assert.match(String(done?.errorMessage), /no longer exists/);
382
+ });
383
+
384
+ it("fails the job without a retry when the filter is turned off", async () => {
385
+ current = await world([
386
+ standingFilter({ state: "Disabled", disabledReason: "UserDisabled" }),
387
+ ]);
388
+
389
+ await runFilterJob(current);
390
+
391
+ assert.deepEqual(current.moves, []);
392
+ assert.equal(current.updates.at(-1)?.state, "Failed");
393
+ assert.match(String(current.updates.at(-1)?.errorMessage), /turned off/);
394
+ });
395
+
396
+ it("fails the job without a retry when the filter was deleted", async () => {
397
+ current = await world([]);
398
+
399
+ await runFilterJob(current);
400
+
401
+ assert.equal(current.updates.at(-1)?.state, "Failed");
402
+ assert.match(String(current.updates.at(-1)?.errorMessage), /deleted/);
403
+ });
404
+ });
@@ -1,30 +1,85 @@
1
1
  import { getClient, type RemitClient } from "@remit/backend/client";
2
2
  import {
3
+ type ApplyOrganizeDeps,
3
4
  applyOrganize,
5
+ backApplyFilter,
4
6
  buildOrganizeMatchDeps,
5
7
  buildOrganizeMoveService,
8
+ isFilterJob,
6
9
  matchOrganize,
7
10
  ORGANIZE_MATCH_LIMIT,
8
11
  type OrganizeMatchDeps,
12
+ type OrganizeRejection,
9
13
  predicateFromJob,
10
14
  } from "@remit/backend/organize";
11
- import type { Logger } from "@remit/logger-lambda";
15
+ import type { Logger } from "@remit/logger";
12
16
  import type { OrganizeJobEvent } from "../events.js";
13
17
 
14
18
  export interface ProcessOrganizeJobDeps {
15
19
  client?: RemitClient;
16
20
  matchDeps?: OrganizeMatchDeps;
21
+ moveService?: ApplyOrganizeDeps["moveService"];
17
22
  }
18
23
 
24
+ interface OrganizeOutcome {
25
+ rejected: OrganizeRejection | null;
26
+ matched: number;
27
+ applied: number;
28
+ failed: number;
29
+ failures: string[];
30
+ }
31
+
32
+ const rejectedOutcome = (rejected: OrganizeRejection): OrganizeOutcome => ({
33
+ rejected,
34
+ matched: 0,
35
+ applied: 0,
36
+ failed: 0,
37
+ failures: [],
38
+ });
39
+
40
+ /**
41
+ * The "all like these" pass: the predicate snapshotted on the row, over every
42
+ * folder, every requested move applied (reader #497).
43
+ */
44
+ const runSnapshot = async (
45
+ client: RemitClient,
46
+ matchDeps: OrganizeMatchDeps,
47
+ moveService: ApplyOrganizeDeps["moveService"],
48
+ job: Awaited<ReturnType<RemitClient["organizeJobRequest"]["get"]>>,
49
+ ): Promise<OrganizeOutcome> => {
50
+ const predicate = predicateFromJob(job);
51
+ const match = await matchOrganize(
52
+ matchDeps,
53
+ job.accountConfigId,
54
+ predicate,
55
+ ORGANIZE_MATCH_LIMIT,
56
+ );
57
+ if (match.rejected) return rejectedOutcome(match.rejected);
58
+ const { applied, failed, failures } = await applyOrganize(
59
+ { client, moveService },
60
+ job.accountConfigId,
61
+ match.messageIds,
62
+ predicate,
63
+ );
64
+ return {
65
+ rejected: null,
66
+ matched: match.messageIds.length,
67
+ applied,
68
+ failed,
69
+ failures,
70
+ };
71
+ };
72
+
19
73
  /**
20
- * Run a "all like these" back-apply job (RFC 034, #1278): match the corpus
21
- * against the job's snapshotted predicate and apply the action to every match,
22
- * in one pass, then record the counts. Mirrors the export job's lifecycle —
74
+ * Run a back-apply job (RFC 034, #1278): an "all like these" pass matches the
75
+ * corpus against the job's snapshotted predicate; a filter job (#1354) runs the
76
+ * named standing filter over the inbox with index-time move precedence. Either
77
+ * applies the action to every match in one pass, then records the counts. Mirrors the export job's lifecycle —
23
78
  * Running, then Complete/Failed — on the same fanout seam.
24
79
  *
25
80
  * Reuses the shared matcher (so the applied set equals what preview returned)
26
81
  * and the idempotent apply plumbing; `appliedByFilterId` is never written and no
27
- * Filter/FilterAnchor row is ever created. Both back-apply actions run here: the
82
+ * Filter row is ever created. Both back-apply actions run here: the
28
83
  * additive label upsert and the exclusive folder move, the latter through the
29
84
  * same local-first placement mover body sync uses (`buildOrganizeMoveService`),
30
85
  * so a redelivered job re-applies both idempotently.
@@ -53,52 +108,49 @@ export const processOrganizeJob = async (
53
108
  );
54
109
 
55
110
  try {
56
- const predicate = predicateFromJob(job);
57
111
  const matchDeps = deps.matchDeps ?? buildOrganizeMatchDeps(client);
58
- const match = await matchOrganize(
59
- matchDeps,
60
- accountConfigId,
61
- predicate,
62
- ORGANIZE_MATCH_LIMIT,
63
- );
112
+ const moveService = deps.moveService ?? buildOrganizeMoveService(client);
113
+ const outcome = isFilterJob(job)
114
+ ? await backApplyFilter(
115
+ { client, matchDeps, moveService },
116
+ accountConfigId,
117
+ job.filterId,
118
+ ).then((result) =>
119
+ result.rejected ? rejectedOutcome(result.rejected) : result,
120
+ )
121
+ : await runSnapshot(client, matchDeps, moveService, job);
64
122
 
65
- if (match.rejected) {
123
+ if (outcome.rejected) {
66
124
  await client.organizeJobRequest.update(organizeJobId, {
67
125
  state: "Failed",
68
126
  matchedCount: 0,
69
127
  appliedCount: 0,
70
128
  failedCount: 0,
71
- errorMessage: match.rejected.message,
129
+ errorMessage: outcome.rejected.message,
72
130
  });
73
131
  log.warn(
74
- { accountConfigId, organizeJobId, reason: match.rejected.reason },
132
+ { accountConfigId, organizeJobId, reason: outcome.rejected.reason },
75
133
  "Organize back-apply refused the job's rule; failing the job without a retry",
76
134
  );
77
135
  return;
78
136
  }
79
137
 
80
- const { messageIds, semanticUnavailable } = match;
81
- const { applied, failed } = await applyOrganize(
82
- { client, moveService: buildOrganizeMoveService(client) },
83
- accountConfigId,
84
- messageIds,
85
- predicate,
86
- );
87
-
138
+ const { matched, applied, failed, failures } = outcome;
88
139
  await client.organizeJobRequest.update(organizeJobId, {
89
140
  state: "Complete",
90
- matchedCount: messageIds.length,
141
+ matchedCount: matched,
91
142
  appliedCount: applied,
92
143
  failedCount: failed,
144
+ errorMessage: failures[0] ?? "",
93
145
  });
94
146
  log.info(
95
147
  {
96
148
  accountConfigId,
97
149
  organizeJobId,
98
- matched: messageIds.length,
150
+ filterId: job.filterId,
151
+ matched,
99
152
  applied,
100
153
  failed,
101
- semanticUnavailable,
102
154
  },
103
155
  "Organize back-apply complete",
104
156
  );
@@ -2,7 +2,7 @@ import assert from "node:assert/strict";
2
2
  import { describe, it } from "node:test";
3
3
  import type { RemitClient } from "@remit/backend/client";
4
4
  import { NotFoundError } from "@remit/data-ports/errors";
5
- import { noopLogger } from "@remit/logger-lambda/noop-logger";
5
+ import { noopLogger } from "@remit/logger";
6
6
  import type { SenderCategoryBackApplyEvent } from "../events.js";
7
7
  import { processSenderCategoryBackApply } from "./sender-category-backapply.js";
8
8
 
@@ -4,7 +4,7 @@ import {
4
4
  SENDER_CATEGORY_BACKAPPLY_LIMIT,
5
5
  } from "@remit/backend/sender-category";
6
6
  import { NotFoundError } from "@remit/data-ports/errors";
7
- import type { Logger } from "@remit/logger-lambda";
7
+ import type { Logger } from "@remit/logger";
8
8
  import type { SenderCategoryBackApplyEvent } from "../events.js";
9
9
 
10
10
  export interface ProcessSenderCategoryBackApplyDeps {