@remit/backend 0.0.138 → 0.0.139

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@remit/backend",
3
- "version": "0.0.138",
3
+ "version": "0.0.139",
4
4
  "description": "Remit Mail Inspector API backend",
5
5
  "license": "MIT",
6
6
  "author": "",
@@ -1,11 +1,13 @@
1
1
  import assert from "node:assert/strict";
2
- import { describe, it, mock } from "node:test";
2
+ import { afterEach, describe, it, mock } from "node:test";
3
+ import type { SendMessageCommand } from "@aws-sdk/client-sqs";
3
4
  import type {
4
5
  CreateFilterInput,
5
6
  UpdateFilterInput as UpdateFilterRequestBody,
6
7
  } from "@remit/api-openapi-types";
7
8
  import type {
8
9
  CreateFilterAnchorInput,
10
+ CreateOrganizeJobRequestInput,
9
11
  CreateFilterInput as FilterAnchorTxCreateInput,
10
12
  FilterItem,
11
13
  } from "@remit/data-ports";
@@ -16,9 +18,19 @@ import {
16
18
  FilterState,
17
19
  } from "@remit/domain-enums";
18
20
  import type { AnchorPayload } from "@remit/search-service";
21
+ import type { APIGatewayProxyEvent } from "aws-lambda";
22
+ import type { Context } from "openapi-backend";
23
+ import { deriveAccountConfigId } from "../auth.js";
24
+ import {
25
+ _resetForTest,
26
+ type RemitClient,
27
+ setClient,
28
+ } from "../service/data-client.js";
29
+ import { sqsClient } from "../service/sqs.js";
19
30
  import {
20
31
  createFilterWithAnchor,
21
32
  type FilterCrudDeps,
33
+ FilterOperations,
22
34
  pickFilterUpdate,
23
35
  rejectAnchorMutation,
24
36
  resolveFilterScopeExpiry,
@@ -516,3 +528,117 @@ describe("createFilterWithAnchor (#351)", () => {
516
528
  );
517
529
  });
518
530
  });
531
+
532
+ describe("createFilter queues the new filter's pass over the inbox (#1354)", () => {
533
+ const SUB = "cognito-sub-1354";
534
+ const ACCOUNT_CONFIG_ID = deriveAccountConfigId(SUB);
535
+ const ACCOUNT_ID = "acc-1354";
536
+
537
+ const created: FilterItem = {
538
+ filterId: "flt-new",
539
+ accountConfigId: ACCOUNT_CONFIG_ID,
540
+ name: "Invoices",
541
+ scope: FilterScope.Standing,
542
+ state: FilterState.Active,
543
+ disabledReason: "None",
544
+ hasAnchor: false,
545
+ ruleChangedAt: 1,
546
+ actionChangedAt: 1,
547
+ matchOperator: FilterMatchOperator.And,
548
+ literalClauses: [{ field: "From", value: "billing@example.com" }],
549
+ actionLabelId: "None",
550
+ actionMailboxId: "mbx-invoices",
551
+ createdAt: 0,
552
+ updatedAt: 0,
553
+ };
554
+
555
+ afterEach(() => {
556
+ mock.restoreAll();
557
+ _resetForTest();
558
+ });
559
+
560
+ const world = (filter: FilterItem) => {
561
+ process.env.SQS_QUEUE_URL_ACCOUNT_FANOUT =
562
+ "http://localhost:9324/queue/account-fanout-test";
563
+ const jobs: CreateOrganizeJobRequestInput[] = [];
564
+ const enqueued: SendMessageCommand[] = [];
565
+ mock.method(sqsClient, "send", async (command: SendMessageCommand) => {
566
+ enqueued.push(command);
567
+ return {};
568
+ });
569
+ setClient({
570
+ account: {
571
+ get: async () => ({
572
+ accountId: ACCOUNT_ID,
573
+ accountConfigId: ACCOUNT_CONFIG_ID,
574
+ }),
575
+ },
576
+ mailbox: {
577
+ get: async () => ({
578
+ mailboxId: "mbx-invoices",
579
+ accountId: ACCOUNT_ID,
580
+ fullPath: "Invoices",
581
+ syncStatus: "synced",
582
+ }),
583
+ },
584
+ filterAnchorTransaction: { createWithAnchor: async () => filter },
585
+ organizeJobRequest: {
586
+ create: async (input: CreateOrganizeJobRequestInput) => {
587
+ jobs.push(input);
588
+ return { ...input, organizeJobId: "job-new", state: "Pending" };
589
+ },
590
+ },
591
+ } as unknown as RemitClient);
592
+ return { jobs, enqueued };
593
+ };
594
+
595
+ const createFilter = (body: Partial<CreateFilterInput>) =>
596
+ (
597
+ FilterOperations.FilterOperations_createFilter as (
598
+ context: Context,
599
+ event: APIGatewayProxyEvent,
600
+ ) => Promise<Record<string, unknown>>
601
+ )(
602
+ {
603
+ request: {
604
+ params: { accountId: ACCOUNT_ID },
605
+ requestBody: {
606
+ name: created.name,
607
+ scope: created.scope,
608
+ matchOperator: created.matchOperator,
609
+ literalClauses: created.literalClauses,
610
+ actionLabelId: created.actionLabelId,
611
+ actionMailboxId: created.actionMailboxId,
612
+ ...body,
613
+ },
614
+ },
615
+ } as unknown as Context,
616
+ {
617
+ requestContext: { authorizer: { claims: { sub: SUB } } },
618
+ } as unknown as APIGatewayProxyEvent,
619
+ );
620
+
621
+ it("queues an organize job for the created filter and returns its id", async () => {
622
+ const { jobs, enqueued } = world(created);
623
+
624
+ const response = await createFilter({});
625
+
626
+ assert.equal(jobs.length, 1);
627
+ assert.equal(jobs[0]?.filterId, "flt-new");
628
+ assert.equal(enqueued.length, 1);
629
+ assert.match(String(enqueued[0]?.input.MessageBody), /job-new/);
630
+ assert.equal(response.organizeJobId, "job-new");
631
+ assert.equal(response.filterId, "flt-new");
632
+ });
633
+
634
+ it("names only the filter: the predicate columns stay at their sentinels", async () => {
635
+ const { jobs } = world(created);
636
+
637
+ await createFilter({});
638
+
639
+ assert.equal(jobs[0]?.anchorMessageId, "None");
640
+ assert.deepEqual(jobs[0]?.literalClauses, []);
641
+ assert.equal(jobs[0]?.actionLabelId, "None");
642
+ assert.equal(jobs[0]?.actionMailboxId, "None");
643
+ });
644
+ });
@@ -1,5 +1,6 @@
1
1
  import type {
2
2
  CreateFilterInput,
3
+ CreateFilterResponse,
3
4
  FilterResponse,
4
5
  UpdateFilterInput as UpdateFilterRequestBody,
5
6
  } from "@remit/api-openapi-types";
@@ -19,9 +20,10 @@ import {
19
20
  } from "@remit/domain-enums";
20
21
  import type { AnchorPayload } from "@remit/search-service";
21
22
  import type { APIGatewayProxyEvent } from "aws-lambda";
22
- import { getAccountConfigIdFromEvent } from "../auth.js";
23
+ import { getAccountConfigIdFromEvent, getSubFromEvent } from "../auth.js";
23
24
  import { getClient } from "../service/data-client.js";
24
25
  import { buildFilterAnchor } from "../service/filter.js";
26
+ import { queueOrganizeJob } from "../service/organize-queue.js";
25
27
  import type {
26
28
  FilterDetailOperationIds,
27
29
  FilterOperationIds,
@@ -340,6 +342,16 @@ export const createFilterWithAnchor = async (
340
342
  );
341
343
  };
342
344
 
345
+ const requireUserId = (event: APIGatewayProxyEvent): string => {
346
+ const userId = getSubFromEvent(event);
347
+ if (!userId) {
348
+ throw new Error(
349
+ "Missing Cognito `sub`: cannot attribute an organize job to a user",
350
+ );
351
+ }
352
+ return userId;
353
+ };
354
+
343
355
  /**
344
356
  * A filter's `actionMailboxId` is a durable reference, so its target has to be
345
357
  * a folder the mail server has settled (D12, first row) — 422 otherwise. The
@@ -393,11 +405,15 @@ export const FilterOperations: Record<
393
405
  };
394
406
  },
395
407
 
396
- FilterOperations_createFilter: async (context, ...args: unknown[]) => {
408
+ FilterOperations_createFilter: async (
409
+ context,
410
+ ...args: unknown[]
411
+ ): Promise<CreateFilterResponse> => {
397
412
  const event = args[0] as APIGatewayProxyEvent;
398
413
  const accountConfigId = getAccountConfigIdFromEvent(event);
399
414
  const { accountId } = context.request.params as { accountId: string };
400
415
  const input = context.request.requestBody as CreateFilterInput;
416
+ const userId = requireUserId(event);
401
417
 
402
418
  const client = await getClient();
403
419
  const account = await client.account.get(accountId);
@@ -412,7 +428,14 @@ export const FilterOperations: Record<
412
428
  accountConfigId,
413
429
  input,
414
430
  );
415
- return toFilterResponse(filter);
431
+ // The new filter's first pass over the inbox. The worker reads the filter
432
+ // when it runs and refuses one that has nothing to run (#1354).
433
+ const job = await queueOrganizeJob(client, {
434
+ accountConfigId,
435
+ userId,
436
+ filterId: filter.filterId,
437
+ });
438
+ return { ...toFilterResponse(filter), organizeJobId: job.organizeJobId };
416
439
  },
417
440
  };
418
441
 
@@ -9,7 +9,7 @@
9
9
  * only removed by the user, so a `mailboxId` that names nothing never exists.
10
10
  */
11
11
  import assert from "node:assert/strict";
12
- import { afterEach, describe, it } from "node:test";
12
+ import { afterEach, describe, it, mock } from "node:test";
13
13
  import type { MailboxItem } from "@remit/data-ports";
14
14
  import { CanonicalMailboxRole, MailboxSyncStatus } from "@remit/domain-enums";
15
15
  import type { APIGatewayProxyEvent } from "aws-lambda";
@@ -20,6 +20,7 @@ import {
20
20
  type RemitClient,
21
21
  setClient,
22
22
  } from "../service/data-client.js";
23
+ import { sqsClient } from "../service/sqs.js";
23
24
  import { FilterDetailOperations, FilterOperations } from "./filter.js";
24
25
  import { FolderRoleOperations } from "./folder-role.js";
25
26
  import { MessageBulkOperations } from "./message.js";
@@ -55,6 +56,9 @@ const world = (syncStatus: MailboxItem["syncStatus"]): Writes => {
55
56
  copies: [],
56
57
  };
57
58
  const row = target(syncStatus);
59
+ process.env.SQS_QUEUE_URL_ACCOUNT_FANOUT =
60
+ "http://localhost:9324/queue/account-fanout-test";
61
+ mock.method(sqsClient, "send", async () => ({}));
58
62
 
59
63
  setClient({
60
64
  account: {
@@ -85,6 +89,13 @@ const world = (syncStatus: MailboxItem["syncStatus"]): Writes => {
85
89
  return { filterId: "flt-1" };
86
90
  },
87
91
  },
92
+ organizeJobRequest: {
93
+ create: async (input: Record<string, unknown>) => ({
94
+ ...input,
95
+ organizeJobId: "job-1",
96
+ state: "Pending",
97
+ }),
98
+ },
88
99
  accountSetting: {
89
100
  get: async () => null,
90
101
  upsert: async (item: unknown) => {
@@ -168,6 +179,7 @@ const SITES = [
168
179
 
169
180
  describe("a write that binds to a folder requires a settled folder", () => {
170
181
  afterEach(() => {
182
+ mock.restoreAll();
171
183
  _resetForTest();
172
184
  });
173
185
 
@@ -4,11 +4,12 @@ import type { SendMessageCommand } from "@aws-sdk/client-sqs";
4
4
  import type { OrganizeInput } from "@remit/api-openapi-types";
5
5
  import type {
6
6
  CreateOrganizeJobRequestInput,
7
+ FilterItem,
7
8
  IAccountRepository,
8
9
  IOrganizeJobRequestRepository,
9
10
  OrganizeJobRequestItem,
10
11
  } from "@remit/data-ports";
11
- import { BadRequestError } from "@remit/data-ports/errors";
12
+ import { BadRequestError, NotFoundError } from "@remit/data-ports/errors";
12
13
  import { FilterMatchOperator } from "@remit/domain-enums";
13
14
  import type { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda";
14
15
  import type { Context } from "openapi-backend";
@@ -186,3 +187,91 @@ describe("createOrganizeJob rejected-rule refusal (reader #995)", () => {
186
187
  );
187
188
  });
188
189
  });
190
+
191
+ describe("createOrganizeJob by filterId — Run now (#1354)", () => {
192
+ afterEach(() => {
193
+ mock.restoreAll();
194
+ _resetForTest();
195
+ });
196
+
197
+ const standing = (state: FilterItem["state"]): FilterItem => ({
198
+ filterId: "flt-1",
199
+ accountConfigId: ACCOUNT_CONFIG_ID,
200
+ name: "Invoices",
201
+ scope: "Standing",
202
+ state,
203
+ disabledReason: state === "Active" ? "None" : "UserDisabled",
204
+ hasAnchor: false,
205
+ ruleChangedAt: 1,
206
+ actionChangedAt: 1,
207
+ matchOperator: FilterMatchOperator.And,
208
+ literalClauses: [{ field: "From", value: "billing@example.com" }],
209
+ actionLabelId: "None",
210
+ actionMailboxId: "mbx-invoices",
211
+ createdAt: 0,
212
+ updatedAt: 0,
213
+ });
214
+
215
+ const withFilter = (filter: FilterItem): void => {
216
+ process.env.SQS_QUEUE_URL_ACCOUNT_FANOUT =
217
+ "http://localhost:9324/queue/account-fanout-test";
218
+ enqueued.length = 0;
219
+ createdRows.length = 0;
220
+ mock.method(sqsClient, "send", async (command: SendMessageCommand) => {
221
+ enqueued.push(command);
222
+ return {};
223
+ });
224
+ setClient({
225
+ account: {
226
+ get: async () => ({
227
+ accountId: ACCOUNT_ID,
228
+ accountConfigId: ACCOUNT_CONFIG_ID,
229
+ }),
230
+ },
231
+ filter: {
232
+ get: async () => filter,
233
+ },
234
+ organizeJobRequest: {
235
+ create: async (row: CreateOrganizeJobRequestInput) => {
236
+ createdRows.push(row);
237
+ return { ...row, organizeJobId: "job-run", state: "Pending" };
238
+ },
239
+ },
240
+ } as unknown as RemitClient);
241
+ };
242
+
243
+ it("queues a job that names only the filter", async () => {
244
+ withFilter(standing("Active"));
245
+
246
+ const response = await postOrganize(input({ filterId: "flt-1" }));
247
+
248
+ assert.equal(response.statusCode, 202);
249
+ assert.equal(createdRows[0]?.filterId, "flt-1");
250
+ assert.equal(createdRows[0]?.actionMailboxId, "None");
251
+ assert.deepEqual(createdRows[0]?.literalClauses, []);
252
+ assert.equal(enqueued.length, 1);
253
+ });
254
+
255
+ it("answers an unknown filter with a 404 and queues nothing", async () => {
256
+ withFilter(standing("Active"));
257
+ setClient({
258
+ account: {
259
+ get: async () => ({
260
+ accountId: ACCOUNT_ID,
261
+ accountConfigId: ACCOUNT_CONFIG_ID,
262
+ }),
263
+ },
264
+ filter: {
265
+ get: async () => {
266
+ throw new NotFoundError("Filter not found: flt-gone");
267
+ },
268
+ },
269
+ } as unknown as RemitClient);
270
+
271
+ const response = await postOrganize(input({ filterId: "flt-gone" }));
272
+
273
+ assert.equal(response.statusCode, 404);
274
+ assert.deepEqual(createdRows, []);
275
+ assert.deepEqual(enqueued, []);
276
+ });
277
+ });
@@ -1,4 +1,3 @@
1
- import { SendMessageCommand } from "@aws-sdk/client-sqs";
2
1
  import type {
3
2
  CreateOrganizeJobResponse,
4
3
  OrganizeInput,
@@ -6,9 +5,7 @@ import type {
6
5
  OrganizePreviewResponse,
7
6
  } from "@remit/api-openapi-types";
8
7
  import { BadRequestError, NotFoundError } from "@remit/data-ports/errors";
9
- import { logger } from "@remit/logger-lambda";
10
8
  import type { APIGatewayProxyEvent } from "aws-lambda";
11
- import { env } from "expect-env";
12
9
  import type { Context } from "openapi-backend";
13
10
  import { getAccountConfigIdFromEvent, getSubFromEvent } from "../auth.js";
14
11
  import { getClient, type RemitClient } from "../service/data-client.js";
@@ -19,7 +16,10 @@ import {
19
16
  type OrganizePredicate,
20
17
  organizePredicateRejection,
21
18
  } from "../service/organize.js";
22
- import { sqsClient } from "../service/sqs.js";
19
+ import {
20
+ type OrganizeJobRequest,
21
+ queueOrganizeJob,
22
+ } from "../service/organize-queue.js";
23
23
  import type {
24
24
  OperationHandler,
25
25
  OrganizeJobDetailOperationIds,
@@ -29,13 +29,6 @@ import { assertAccountOwnership } from "./account-ownership.js";
29
29
 
30
30
  const NONE = "None";
31
31
 
32
- /**
33
- * How long a finished (or abandoned) job row lingers before the table-wide TTL
34
- * reclaims it (RFC 034 Decision 1). A back-apply runs once and needs no standing
35
- * lifetime; a week is ample for a client to poll the result.
36
- */
37
- const ORGANIZE_JOB_TTL_SECONDS = 7 * 24 * 60 * 60;
38
-
39
32
  /**
40
33
  * Normalize the request body into the flattened predicate the job row and the
41
34
  * matcher share. `anchorMessageId` collapses to the `"None"` sentinel when
@@ -70,6 +63,7 @@ const toOrganizeJobResponse = (
70
63
  accountConfigId: job.accountConfigId,
71
64
  userId: job.userId,
72
65
  state: job.state,
66
+ filterId: job.filterId,
73
67
  anchorMessageId: job.anchorMessageId,
74
68
  matchOperator: job.matchOperator,
75
69
  literalClauses: job.literalClauses,
@@ -84,6 +78,27 @@ const toOrganizeJobResponse = (
84
78
  updatedAt: job.updatedAt,
85
79
  });
86
80
 
81
+ /**
82
+ * What a create asks the worker to run. A saved filter is named and read when
83
+ * the job runs, so Run now and a filter's create share one path (#1354); an
84
+ * "all like these" predicate the matcher can never honour is answered here,
85
+ * not with a 202 the worker has to fail later (reader #463).
86
+ */
87
+ const requestedJob = async (
88
+ client: RemitClient,
89
+ owner: { accountConfigId: string; userId: string },
90
+ input: OrganizeInput,
91
+ ): Promise<OrganizeJobRequest> => {
92
+ if (input.filterId) {
93
+ await client.filter.get(owner.accountConfigId, input.filterId);
94
+ return { ...owner, filterId: input.filterId };
95
+ }
96
+ const predicate = predicateFromInput(input);
97
+ const rejection = organizePredicateRejection(predicate);
98
+ if (rejection) throw new BadRequestError(rejection.message);
99
+ return { ...owner, predicate };
100
+ };
101
+
87
102
  export const OrganizeOperations: Record<
88
103
  OrganizeOperationIds,
89
104
  OperationHandler<OrganizeOperationIds>
@@ -107,42 +122,9 @@ export const OrganizeOperations: Record<
107
122
  const client = await getClient();
108
123
  await assertAccount(client, accountId, accountConfigId, "act");
109
124
 
110
- const predicate = predicateFromInput(input);
111
-
112
- // A rule the matcher can never honour is answered here, not with a 202
113
- // the worker has to fail later (reader #463).
114
- const rejection = organizePredicateRejection(predicate);
115
- if (rejection) throw new BadRequestError(rejection.message);
116
-
117
- const ttl = Math.floor(Date.now() / 1000) + ORGANIZE_JOB_TTL_SECONDS;
118
-
119
- const job = await client.organizeJobRequest.create({
120
- accountConfigId,
121
- userId,
122
- anchorMessageId: predicate.anchorMessageId,
123
- matchOperator: predicate.matchOperator,
124
- literalClauses: predicate.literalClauses,
125
- similarityThreshold: predicate.similarityThreshold,
126
- actionLabelId: predicate.actionLabelId,
127
- actionMailboxId: predicate.actionMailboxId,
128
- ttl,
129
- });
130
-
131
- await sqsClient.send(
132
- new SendMessageCommand({
133
- QueueUrl: env.SQS_QUEUE_URL_ACCOUNT_FANOUT,
134
- MessageBody: JSON.stringify({
135
- type: "OrganizeJob",
136
- accountConfigId,
137
- organizeJobId: job.organizeJobId,
138
- }),
139
- }),
140
- );
141
-
142
- // biome-ignore lint/plugin/no-logger-info: a back-apply is an audit-grade signal
143
- logger.info(
144
- { accountConfigId, organizeJobId: job.organizeJobId },
145
- "Organize back-apply job initiated",
125
+ const job = await queueOrganizeJob(
126
+ client,
127
+ await requestedJob(client, { accountConfigId, userId }, input),
146
128
  );
147
129
 
148
130
  return {
@@ -164,6 +146,11 @@ export const OrganizeOperations: Record<
164
146
  const client = await getClient();
165
147
  await assertAccount(client, accountId, accountConfigId, "read");
166
148
 
149
+ if (input.filterId) {
150
+ throw new BadRequestError(
151
+ "Preview takes a rule, not a saved filter. Run the filter to apply it.",
152
+ );
153
+ }
167
154
  const result = await matchOrganize(
168
155
  buildOrganizeMatchDeps(client),
169
156
  accountConfigId,
@@ -0,0 +1,98 @@
1
+ import { SendMessageCommand } from "@aws-sdk/client-sqs";
2
+ import type {
3
+ CreateOrganizeJobRequestInput,
4
+ OrganizeJobRequestItem,
5
+ } from "@remit/data-ports";
6
+ import { FilterMatchOperator } from "@remit/domain-enums";
7
+ import { logger } from "@remit/logger-lambda";
8
+ import {
9
+ DEFAULT_SEMANTIC_MATCH_THRESHOLD,
10
+ NO_ACTION,
11
+ } from "@remit/mailbox-service";
12
+ import { env } from "expect-env";
13
+ import type { RemitClient } from "./data-client.js";
14
+ import type { OrganizePredicate } from "./organize.js";
15
+ import { sqsClient } from "./sqs.js";
16
+
17
+ /**
18
+ * How long a finished (or abandoned) job row lingers before the table-wide TTL
19
+ * reclaims it (RFC 034 Decision 1). A back-apply runs once and needs no standing
20
+ * lifetime; a week is ample for a client to poll the result.
21
+ */
22
+ const ORGANIZE_JOB_TTL_SECONDS = 7 * 24 * 60 * 60;
23
+
24
+ interface JobOwner {
25
+ accountConfigId: string;
26
+ userId: string;
27
+ }
28
+
29
+ /**
30
+ * An "all like these" pass carries its predicate; a filter job names only the
31
+ * filter, which the worker reads when it runs.
32
+ */
33
+ export type OrganizeJobRequest = JobOwner &
34
+ ({ predicate: OrganizePredicate } | { filterId: string });
35
+
36
+ /** A filter job's row: the filter id, every predicate column at its sentinel. */
37
+ const filterJobColumns = (
38
+ filterId: string,
39
+ ): Omit<CreateOrganizeJobRequestInput, keyof JobOwner | "ttl"> => ({
40
+ filterId,
41
+ anchorMessageId: NO_ACTION,
42
+ matchOperator: FilterMatchOperator.And,
43
+ literalClauses: [],
44
+ similarityThreshold: DEFAULT_SEMANTIC_MATCH_THRESHOLD,
45
+ actionLabelId: NO_ACTION,
46
+ actionMailboxId: NO_ACTION,
47
+ });
48
+
49
+ const predicateColumns = (
50
+ predicate: OrganizePredicate,
51
+ ): Omit<CreateOrganizeJobRequestInput, keyof JobOwner | "ttl"> => ({
52
+ filterId: NO_ACTION,
53
+ ...predicate,
54
+ });
55
+
56
+ /**
57
+ * Write a Pending job row and hand it to the account-fanout worker. Every
58
+ * back-apply goes through here: "all like these", a new filter's first pass,
59
+ * and Run now on a saved filter.
60
+ */
61
+ export const queueOrganizeJob = async (
62
+ client: Pick<RemitClient, "organizeJobRequest">,
63
+ request: OrganizeJobRequest,
64
+ ): Promise<OrganizeJobRequestItem> => {
65
+ const { accountConfigId, userId } = request;
66
+ const columns =
67
+ "filterId" in request
68
+ ? filterJobColumns(request.filterId)
69
+ : predicateColumns(request.predicate);
70
+ const job = await client.organizeJobRequest.create({
71
+ accountConfigId,
72
+ userId,
73
+ ...columns,
74
+ ttl: Math.floor(Date.now() / 1000) + ORGANIZE_JOB_TTL_SECONDS,
75
+ });
76
+
77
+ await sqsClient.send(
78
+ new SendMessageCommand({
79
+ QueueUrl: env.SQS_QUEUE_URL_ACCOUNT_FANOUT,
80
+ MessageBody: JSON.stringify({
81
+ type: "OrganizeJob",
82
+ accountConfigId,
83
+ organizeJobId: job.organizeJobId,
84
+ }),
85
+ }),
86
+ );
87
+
88
+ // biome-ignore lint/plugin/no-logger-info: a back-apply is an audit-grade signal
89
+ logger.info(
90
+ {
91
+ accountConfigId,
92
+ organizeJobId: job.organizeJobId,
93
+ filterId: job.filterId,
94
+ },
95
+ "Organize back-apply job initiated",
96
+ );
97
+ return job;
98
+ };
@@ -289,6 +289,7 @@ const matchDeps = (
289
289
  },
290
290
  listAccountFilterMessages: async () => ({ items: corpus }),
291
291
  filterAnchors: {
292
+ get: async () => null,
292
293
  listByAccountConfig: async () => filterAnchorRows,
293
294
  put: async () => {
294
295
  throw new Error("matchDeps must not repair an anchor");
@@ -330,6 +331,7 @@ const vectorlessDeps = (
330
331
  },
331
332
  listAccountFilterMessages: async () => ({ items: corpus }),
332
333
  filterAnchors: {
334
+ get: async () => null,
333
335
  listByAccountConfig: async () => [],
334
336
  put: async () => {
335
337
  throw moduleNotFound();
@@ -471,6 +473,7 @@ describe("matchOrganize honors the persisted FilterAnchor (reader #350)", () =>
471
473
  }),
472
474
  listAccountFilterMessages: async () => ({ items: [] }),
473
475
  filterAnchors: {
476
+ get: async () => null,
474
477
  listByAccountConfig: async () => [persistedAnchor],
475
478
  put: async () => {
476
479
  throw new Error("a current anchor must not be rewritten");
@@ -757,6 +760,7 @@ describe("matchOrganize on a deployment without the vector pipeline", () => {
757
760
  }),
758
761
  listAccountFilterMessages: async () => ({ items: [] }),
759
762
  filterAnchors: {
763
+ get: async () => null,
760
764
  listByAccountConfig: async () => [drifted],
761
765
  put: async () => {
762
766
  throw new Error("an unrepairable anchor must not be rewritten");
@@ -798,6 +802,7 @@ describe("matchOrganize on a deployment without the vector pipeline", () => {
798
802
  }),
799
803
  listAccountFilterMessages: async () => ({ items: [] }),
800
804
  filterAnchors: {
805
+ get: async () => null,
801
806
  listByAccountConfig: async () => [],
802
807
  put: async () => {
803
808
  throw new Error("unreachable");
@@ -1120,6 +1125,7 @@ describe("back-apply repairs a drifted anchor the way index-time matching does (
1120
1125
  }),
1121
1126
  listAccountFilterMessages: async () => ({ items: [] }),
1122
1127
  filterAnchors: {
1128
+ get: async () => null,
1123
1129
  listByAccountConfig: async () => anchors,
1124
1130
  put: async (input: CreateFilterAnchorInput) => {
1125
1131
  anchorPuts.push(input);
@@ -1319,6 +1325,7 @@ const storeBackedDeps = (
1319
1325
  };
1320
1326
  },
1321
1327
  filterAnchors: {
1328
+ get: async () => null,
1322
1329
  listByAccountConfig: async () => [],
1323
1330
  put: async () => {
1324
1331
  throw new Error("unreachable");
@@ -5,18 +5,22 @@ import type {
5
5
  IFilterAnchorRepository,
6
6
  OrganizeJobRequestItem,
7
7
  } from "@remit/data-ports";
8
- import { FilterClauseField } from "@remit/domain-enums";
8
+ import { NotFoundError } from "@remit/data-ports/errors";
9
+ import { FilterClauseField, FilterState } from "@remit/domain-enums";
9
10
  import { logger } from "@remit/logger-lambda";
10
11
  import {
11
12
  DEFAULT_SEMANTIC_MATCH_THRESHOLD,
12
13
  type FilterMessage,
14
+ FilterPipeline,
13
15
  type LiteralClauseNarrowing,
14
16
  literalClausesMatch,
15
17
  literalClauseTerms,
18
+ type MessageEmbedder,
16
19
  NO_ACTION,
17
20
  PlacementMoveService,
18
21
  refreshAnchorForEmbedder,
19
22
  } from "@remit/mailbox-service";
23
+ import { buildFilterConfig } from "@remit/mailbox-service/filter-config";
20
24
  import {
21
25
  type AnchorPayload,
22
26
  buildMessageAnchor,
@@ -135,6 +139,18 @@ export interface OrganizeCandidatePage {
135
139
  export interface OrganizeCandidateQuery extends LiteralClauseNarrowing {
136
140
  limit: number;
137
141
  continuationToken?: string;
142
+ mailboxIds?: readonly string[];
143
+ }
144
+
145
+ /**
146
+ * Where a pass looks and which standing filter it runs for. A filter
147
+ * back-apply reads that filter's own FilterAnchor row by id, narrows both the
148
+ * literal corpus and the vector query to the inbox, and matches a body-content
149
+ * clause on the stored preview; an "all like these" pass sets neither.
150
+ */
151
+ export interface OrganizeMatchScope {
152
+ filterId?: string;
153
+ mailboxIds?: readonly string[];
138
154
  }
139
155
 
140
156
  export interface OrganizeMatchDeps {
@@ -159,16 +175,19 @@ export interface OrganizeMatchDeps {
159
175
  /**
160
176
  * Every persisted FilterAnchor for the account — read-only, always
161
177
  * available (never gated behind {@link semantic}, since it never touches
162
- * the vector store). A back-apply predicate carries only a bare
163
- * `anchorMessageId`, never a `filterId` (RFC 034 recap: this job is
164
- * deliberately not a Filter), so this is how {@link matchSemantic} finds
165
- * "the standing filter this anchor came from," if one still exists, to
166
- * read its fixed-at-save-time vector instead of re-deriving one from the
167
- * anchor message's current chunks (reader #350 / RFC 039 Decision 1).
178
+ * the vector store). A filter back-apply reads its own row by `filterId`
179
+ * (#1354); an "all like these" predicate carries only a bare
180
+ * `anchorMessageId`, so this is how {@link matchSemantic} finds "the
181
+ * standing filter this anchor came from," if one still exists, to read its
182
+ * fixed-at-save-time vector instead of re-deriving one from the anchor
183
+ * message's current chunks (reader #350 / RFC 039 Decision 1).
168
184
  * `put` is the write half of the lazy anchor-drift repair
169
185
  * ({@link refreshAnchorForEmbedder}), never a new standing rule.
170
186
  */
171
- filterAnchors: Pick<IFilterAnchorRepository, "listByAccountConfig" | "put">;
187
+ filterAnchors: Pick<
188
+ IFilterAnchorRepository,
189
+ "get" | "listByAccountConfig" | "put"
190
+ >;
172
191
  }
173
192
 
174
193
  /**
@@ -179,7 +198,10 @@ export interface OrganizeMatchDeps {
179
198
  * make it (reader #463).
180
199
  */
181
200
  export interface OrganizeRejection {
182
- reason: "BodyContentWithoutVectorPipeline";
201
+ reason:
202
+ | "BodyContentWithoutVectorPipeline"
203
+ | "FilterNotActive"
204
+ | "FilterMissing";
183
205
  message: string;
184
206
  }
185
207
 
@@ -246,9 +268,9 @@ const filterMessageFromChunks = (
246
268
  /**
247
269
  * The persisted FilterAnchor whose `anchorMessageId` matches this predicate's
248
270
  * anchor, if the account has a standing filter built from that message (RFC
249
- * 034 Decision 2 / RFC 039 Decision 1, reader #350). A back-apply predicate
250
- * never carries a `filterId` — this job structurally is not a Filter — so the
251
- * only way to recover "the filter this anchor came from" is by the value it
271
+ * 034 Decision 2 / RFC 039 Decision 1, reader #350). An "all like these"
272
+ * predicate carries no `filterId` — that job structurally is not a Filter — so
273
+ * the only way to recover "the filter this anchor came from" is by the value it
252
274
  * was anchored on. The scan is over one account's (small, bounded) anchor
253
275
  * rows, read-only, and never touches the vector store. Returns `undefined`
254
276
  * when no standing filter was ever anchored on this message (or it has since
@@ -331,24 +353,38 @@ const matchSemantic = async (
331
353
  accountConfigId: string,
332
354
  predicate: OrganizePredicate,
333
355
  limit: number,
356
+ scope: OrganizeMatchScope,
334
357
  ): Promise<{ messageIds: string[]; indexEmpty: boolean }> => {
335
358
  const semantic = deps.semantic();
336
- const persisted = await findPersistedAnchor(
337
- deps.filterAnchors,
338
- accountConfigId,
339
- predicate.anchorMessageId,
340
- );
359
+ const persisted = scope.filterId
360
+ ? ((await deps.filterAnchors.get(accountConfigId, scope.filterId)) ??
361
+ undefined)
362
+ : await findPersistedAnchor(
363
+ deps.filterAnchors,
364
+ accountConfigId,
365
+ predicate.anchorMessageId,
366
+ );
341
367
  const anchor: AnchorPayload | null = persisted
342
368
  ? await anchorForWiden(deps, semantic, persisted)
343
369
  : await semantic.buildAnchor(accountConfigId, predicate.anchorMessageId);
344
370
  if (!anchor) return { messageIds: [], indexEmpty: true };
345
371
  const threshold =
346
372
  predicate.similarityThreshold ?? DEFAULT_SEMANTIC_MATCH_THRESHOLD;
347
- const matches = await semantic.vectorStore.query({
348
- vector: anchor.anchorEmbedding,
349
- topK: limit * VECTOR_CHUNK_FACTOR,
350
- filter: { accountConfigId },
351
- });
373
+ const folders: readonly (string | undefined)[] = scope.mailboxIds ?? [
374
+ undefined,
375
+ ];
376
+ const pages = await Promise.all(
377
+ folders.map((mailboxId) =>
378
+ semantic.vectorStore.query({
379
+ vector: anchor.anchorEmbedding,
380
+ topK: limit * VECTOR_CHUNK_FACTOR,
381
+ filter: mailboxId
382
+ ? { accountConfigId, mailboxId }
383
+ : { accountConfigId },
384
+ }),
385
+ ),
386
+ );
387
+ const matches = pages.flat();
352
388
  if (matches.length === 0) return { messageIds: [], indexEmpty: true };
353
389
  const bestScore = new Map<string, number>();
354
390
  for (const match of matches) {
@@ -436,6 +472,7 @@ const matchLiteral = async (
436
472
  accountConfigId: string,
437
473
  predicate: OrganizePredicate,
438
474
  limit: number,
475
+ scope: OrganizeMatchScope,
439
476
  ): Promise<string[]> => {
440
477
  const clauses = predicate.literalClauses;
441
478
  const narrowing = literalClauseTerms(clauses, predicate.matchOperator);
@@ -449,6 +486,7 @@ const matchLiteral = async (
449
486
  ...narrowing,
450
487
  limit,
451
488
  continuationToken,
489
+ mailboxIds: scope.mailboxIds,
452
490
  });
453
491
  for (const { messageId, message } of page.items) {
454
492
  if (seen.has(messageId)) continue;
@@ -488,6 +526,7 @@ export const matchOrganize = async (
488
526
  accountConfigId: string,
489
527
  predicate: OrganizePredicate,
490
528
  limit: number = ORGANIZE_MATCH_LIMIT,
529
+ scope: OrganizeMatchScope = {},
491
530
  ): Promise<OrganizeMatchResult> => {
492
531
  const anchored = hasAnchor(predicate);
493
532
  const clauses = predicate.literalClauses;
@@ -501,13 +540,16 @@ export const matchOrganize = async (
501
540
  }
502
541
 
503
542
  if (!anchored) {
504
- const rejection = organizePredicateRejection(predicate);
543
+ const rejection = scope.filterId
544
+ ? null
545
+ : organizePredicateRejection(predicate);
505
546
  if (rejection) return { rejected: rejection };
506
547
  const messageIds = await matchLiteral(
507
548
  deps,
508
549
  accountConfigId,
509
550
  predicate,
510
551
  limit,
552
+ scope,
511
553
  );
512
554
  return {
513
555
  rejected: null,
@@ -523,6 +565,7 @@ export const matchOrganize = async (
523
565
  accountConfigId,
524
566
  predicate,
525
567
  limit,
568
+ scope,
526
569
  );
527
570
  return {
528
571
  rejected: null,
@@ -543,13 +586,14 @@ export const matchOrganize = async (
543
586
  semanticIndexEmpty: false,
544
587
  };
545
588
  }
546
- const rejection = bodyContentRejection(clauses);
589
+ const rejection = scope.filterId ? null : bodyContentRejection(clauses);
547
590
  if (rejection) return { rejected: rejection };
548
591
  const messageIds = await matchLiteral(
549
592
  deps,
550
593
  accountConfigId,
551
594
  predicate,
552
595
  limit,
596
+ scope,
553
597
  );
554
598
  return {
555
599
  rejected: null,
@@ -591,12 +635,11 @@ const buildSemanticFromEnv = (): OrganizeSemanticDeps => {
591
635
  *
592
636
  * `From`/`Subject`/`listId` come from the row verbatim (full fidelity), so a
593
637
  * `From`, `Subject`, `FromDomain` or `ListId` clause matches here exactly as it
594
- * does at index time. `text` (the body) is left empty: the thread row carries
595
- * only a truncated `snippet`, and matching
596
- * a body-content clause against a preview would silently diverge from the live
597
- * index-time filter's full-body match. Body-content (`HasWords`) clauses are
598
- * rejected before this is read (see {@link bodyContentRejection}), so the
599
- * empty `text` is never matched against.
638
+ * does at index time. `text` is the row's stored body preview (`snippet`),
639
+ * the same text search matches free words against. An "all like these" pass
640
+ * refuses a body-content (`HasWords`) clause before reading it (see
641
+ * {@link bodyContentRejection}); a filter back-apply matches it on the
642
+ * preview, the mail search showed the user when the rule was made (#1354).
600
643
  */
601
644
  const listAccountFilterMessagesFromClient =
602
645
  (client: RemitClient): OrganizeMatchDeps["listAccountFilterMessages"] =>
@@ -609,6 +652,7 @@ const listAccountFilterMessagesFromClient =
609
652
  limit: query.limit,
610
653
  continuationToken: query.continuationToken,
611
654
  excludeDeleted: true,
655
+ mailboxIds: query.mailboxIds,
612
656
  },
613
657
  );
614
658
  return {
@@ -618,7 +662,7 @@ const listAccountFilterMessagesFromClient =
618
662
  from: row.fromEmail ?? "",
619
663
  fromName: row.fromName ?? "",
620
664
  subject: row.subject ?? "",
621
- text: "",
665
+ text: row.snippet ?? "",
622
666
  listId: row.listId ?? "",
623
667
  },
624
668
  })),
@@ -754,3 +798,228 @@ export const applyOrganize = async (
754
798
  }
755
799
  return { applied, failed };
756
800
  };
801
+
802
+ const FILTER_NOT_ACTIVE_MESSAGE =
803
+ "This filter is turned off or has expired. Turn it on to run it over the inbox.";
804
+
805
+ const FILTER_MISSING_MESSAGE =
806
+ "This filter was deleted before it could run over the inbox.";
807
+
808
+ const rejectFilter = (
809
+ reason: OrganizeRejection["reason"],
810
+ message: string,
811
+ ): OrganizeMatchRejected => ({ rejected: { reason, message } });
812
+
813
+ /**
814
+ * A standing filter's predicate and action, read the way index time reads
815
+ * them. `matchSemantic` reads the anchor vector by `filterId`; the anchor
816
+ * message id here only says the filter widens.
817
+ */
818
+ const predicateFromFilter = (
819
+ filter: FilterItem,
820
+ anchorMessageId: string,
821
+ ): OrganizePredicate => ({
822
+ anchorMessageId,
823
+ matchOperator: filter.matchOperator,
824
+ literalClauses: filter.literalClauses,
825
+ similarityThreshold: DEFAULT_SEMANTIC_MATCH_THRESHOLD,
826
+ actionLabelId: filter.actionLabelId,
827
+ actionMailboxId: filter.actionMailboxId,
828
+ });
829
+
830
+ interface LoadedFilter {
831
+ rejected: null;
832
+ filter: FilterItem;
833
+ predicate: OrganizePredicate;
834
+ /**
835
+ * Marked `hasAnchor` with its FilterAnchor row gone. Index time matches
836
+ * such a filter against nothing, so the back-apply does too.
837
+ */
838
+ matchesNothing: boolean;
839
+ }
840
+
841
+ /**
842
+ * Read the filter a job names: refused when it was deleted or is not Active,
843
+ * since index time evaluates neither.
844
+ */
845
+ const loadFilter = async (
846
+ client: Pick<RemitClient, "filter" | "filterAnchor">,
847
+ accountConfigId: string,
848
+ filterId: string,
849
+ ): Promise<LoadedFilter | OrganizeMatchRejected> => {
850
+ const stored = await client.filter
851
+ .get(accountConfigId, filterId)
852
+ .catch((error: unknown) => {
853
+ if (error instanceof NotFoundError) return null;
854
+ throw error;
855
+ });
856
+ if (!stored) return rejectFilter("FilterMissing", FILTER_MISSING_MESSAGE);
857
+ const filter = await client.filter.refreshExpiry(stored);
858
+ if (filter.state !== FilterState.Active) {
859
+ return rejectFilter("FilterNotActive", FILTER_NOT_ACTIVE_MESSAGE);
860
+ }
861
+ if (!filter.hasAnchor) {
862
+ return {
863
+ rejected: null,
864
+ filter,
865
+ predicate: predicateFromFilter(filter, NO_ACTION),
866
+ matchesNothing: false,
867
+ };
868
+ }
869
+ const anchor = await client.filterAnchor.get(accountConfigId, filterId);
870
+ return {
871
+ rejected: null,
872
+ filter,
873
+ predicate: predicateFromFilter(
874
+ filter,
875
+ anchor?.anchorMessageId ?? NO_ACTION,
876
+ ),
877
+ matchesNothing: anchor === null,
878
+ };
879
+ };
880
+
881
+ /** Every inbox under the config — the folders a filter back-apply runs over. */
882
+ const inboxMailboxIds = async (
883
+ client: Pick<RemitClient, "account" | "mailboxSpecialUse">,
884
+ accountConfigId: string,
885
+ ): Promise<string[]> => {
886
+ const accounts =
887
+ await client.account.listMailSourcesByAccountConfig(accountConfigId);
888
+ const inboxes = await Promise.all(
889
+ accounts.map((account) =>
890
+ client.mailboxSpecialUse.findInboxMailbox(account.accountId),
891
+ ),
892
+ );
893
+ return inboxes.flatMap((inbox) => (inbox ? [inbox.mailboxId] : []));
894
+ };
895
+
896
+ /**
897
+ * The message as the index-time pipeline reads it, from its thread row. The
898
+ * body is the stored preview, so a `HasWords` clause matches what search
899
+ * matched. A message deleted since its vector was written has no row.
900
+ */
901
+ const filterMessageOf = async (
902
+ client: Pick<RemitClient, "threadMessage">,
903
+ accountConfigId: string,
904
+ messageId: string,
905
+ ): Promise<FilterMessage | null> => {
906
+ const row = await client.threadMessage
907
+ .getByMessageId(accountConfigId, messageId)
908
+ .catch((error: unknown) => {
909
+ if (error instanceof NotFoundError) return null;
910
+ throw error;
911
+ });
912
+ if (!row) return null;
913
+ return {
914
+ from: row.fromEmail ?? "",
915
+ fromName: row.fromName ?? "",
916
+ subject: row.subject ?? "",
917
+ text: row.snippet ?? "",
918
+ listId: row.listId ?? "",
919
+ };
920
+ };
921
+
922
+ export interface FilterBackApplyResult {
923
+ rejected: null;
924
+ matched: number;
925
+ applied: number;
926
+ failed: number;
927
+ semanticUnavailable: boolean;
928
+ }
929
+
930
+ export interface FilterBackApplyDeps {
931
+ client: RemitClient;
932
+ matchDeps: OrganizeMatchDeps;
933
+ moveService?: PlacementMoveService;
934
+ /** The embedder index time uses; read from env when absent. */
935
+ embedder?: MessageEmbedder;
936
+ }
937
+
938
+ /**
939
+ * Run one standing filter over the inbox as if each matching message had just
940
+ * arrived. Every message the matcher returns is a match: its label always
941
+ * applies, being additive. The index-time {@link FilterPipeline} only orders
942
+ * the exclusive move, so the move is skipped where it names another active
943
+ * filter. The pipeline re-scores a similarity filter against the stored
944
+ * preview rather than the full body, so its verdict on whether this filter
945
+ * matches is not asked; and an evaluate it could not finish names no other
946
+ * filter, so the move goes ahead rather than silently dropping out.
947
+ */
948
+ export const backApplyFilter = async (
949
+ deps: FilterBackApplyDeps,
950
+ accountConfigId: string,
951
+ filterId: string,
952
+ ): Promise<FilterBackApplyResult | OrganizeMatchRejected> => {
953
+ const { client, matchDeps, moveService } = deps;
954
+ const loaded = await loadFilter(client, accountConfigId, filterId);
955
+ if (loaded.rejected) return loaded;
956
+ const config = buildFilterConfig(
957
+ {
958
+ filterService: client.filter,
959
+ filterAnchorService: client.filterAnchor,
960
+ messageLabelService: client.messageLabel,
961
+ placementMoveService: moveService,
962
+ },
963
+ deps.embedder,
964
+ );
965
+ if (!config) {
966
+ throw new Error(
967
+ "Filter back-apply needs the message-management queue to move mail",
968
+ );
969
+ }
970
+ const pipeline = new FilterPipeline(config, logger);
971
+
972
+ const mailboxIds = await inboxMailboxIds(client, accountConfigId);
973
+ const nothing: FilterBackApplyResult = {
974
+ rejected: null,
975
+ matched: 0,
976
+ applied: 0,
977
+ failed: 0,
978
+ semanticUnavailable: false,
979
+ };
980
+ if (loaded.matchesNothing || mailboxIds.length === 0) return nothing;
981
+
982
+ const match = await matchOrganize(
983
+ matchDeps,
984
+ accountConfigId,
985
+ loaded.predicate,
986
+ ORGANIZE_MATCH_LIMIT,
987
+ { filterId, mailboxIds },
988
+ );
989
+ if (match.rejected) return match;
990
+
991
+ const { filter, predicate } = loaded;
992
+ const result: FilterBackApplyResult = {
993
+ ...nothing,
994
+ matched: match.messageIds.length,
995
+ semanticUnavailable: match.semanticUnavailable,
996
+ };
997
+ for (const messageId of match.messageIds) {
998
+ const message = await filterMessageOf(client, accountConfigId, messageId);
999
+ if (!message) continue;
1000
+ const decision = await pipeline.evaluate(
1001
+ accountConfigId,
1002
+ messageId,
1003
+ message,
1004
+ );
1005
+ const move = (decision.move?.filterId ?? filterId) === filterId;
1006
+ if (!move && filter.actionLabelId === NO_ACTION) continue;
1007
+ const outcome = await applyOrganize(
1008
+ { client, moveService },
1009
+ accountConfigId,
1010
+ [messageId],
1011
+ {
1012
+ ...predicate,
1013
+ actionMailboxId: move ? filter.actionMailboxId : NO_ACTION,
1014
+ },
1015
+ );
1016
+ result.applied += outcome.applied;
1017
+ result.failed += outcome.failed;
1018
+ }
1019
+ return result;
1020
+ };
1021
+
1022
+ /** Whether a job row runs a standing filter rather than its own snapshot. */
1023
+ export const isFilterJob = (
1024
+ job: Pick<OrganizeJobRequestItem, "filterId">,
1025
+ ): boolean => job.filterId !== NO_ACTION;