@remit/backend 0.0.138 → 0.0.139
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/handlers/filter.test.ts +127 -1
- package/src/handlers/filter.ts +26 -3
- package/src/handlers/mailbox-settled-target.test.ts +13 -1
- package/src/handlers/organize.test.ts +90 -1
- package/src/handlers/organize.ts +34 -47
- package/src/service/organize-queue.ts +98 -0
- package/src/service/organize.test.ts +7 -0
- package/src/service/organize.ts +300 -31
package/package.json
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import { describe, it, mock } from "node:test";
|
|
2
|
+
import { afterEach, describe, it, mock } from "node:test";
|
|
3
|
+
import type { SendMessageCommand } from "@aws-sdk/client-sqs";
|
|
3
4
|
import type {
|
|
4
5
|
CreateFilterInput,
|
|
5
6
|
UpdateFilterInput as UpdateFilterRequestBody,
|
|
6
7
|
} from "@remit/api-openapi-types";
|
|
7
8
|
import type {
|
|
8
9
|
CreateFilterAnchorInput,
|
|
10
|
+
CreateOrganizeJobRequestInput,
|
|
9
11
|
CreateFilterInput as FilterAnchorTxCreateInput,
|
|
10
12
|
FilterItem,
|
|
11
13
|
} from "@remit/data-ports";
|
|
@@ -16,9 +18,19 @@ import {
|
|
|
16
18
|
FilterState,
|
|
17
19
|
} from "@remit/domain-enums";
|
|
18
20
|
import type { AnchorPayload } from "@remit/search-service";
|
|
21
|
+
import type { APIGatewayProxyEvent } from "aws-lambda";
|
|
22
|
+
import type { Context } from "openapi-backend";
|
|
23
|
+
import { deriveAccountConfigId } from "../auth.js";
|
|
24
|
+
import {
|
|
25
|
+
_resetForTest,
|
|
26
|
+
type RemitClient,
|
|
27
|
+
setClient,
|
|
28
|
+
} from "../service/data-client.js";
|
|
29
|
+
import { sqsClient } from "../service/sqs.js";
|
|
19
30
|
import {
|
|
20
31
|
createFilterWithAnchor,
|
|
21
32
|
type FilterCrudDeps,
|
|
33
|
+
FilterOperations,
|
|
22
34
|
pickFilterUpdate,
|
|
23
35
|
rejectAnchorMutation,
|
|
24
36
|
resolveFilterScopeExpiry,
|
|
@@ -516,3 +528,117 @@ describe("createFilterWithAnchor (#351)", () => {
|
|
|
516
528
|
);
|
|
517
529
|
});
|
|
518
530
|
});
|
|
531
|
+
|
|
532
|
+
describe("createFilter queues the new filter's pass over the inbox (#1354)", () => {
|
|
533
|
+
const SUB = "cognito-sub-1354";
|
|
534
|
+
const ACCOUNT_CONFIG_ID = deriveAccountConfigId(SUB);
|
|
535
|
+
const ACCOUNT_ID = "acc-1354";
|
|
536
|
+
|
|
537
|
+
const created: FilterItem = {
|
|
538
|
+
filterId: "flt-new",
|
|
539
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
540
|
+
name: "Invoices",
|
|
541
|
+
scope: FilterScope.Standing,
|
|
542
|
+
state: FilterState.Active,
|
|
543
|
+
disabledReason: "None",
|
|
544
|
+
hasAnchor: false,
|
|
545
|
+
ruleChangedAt: 1,
|
|
546
|
+
actionChangedAt: 1,
|
|
547
|
+
matchOperator: FilterMatchOperator.And,
|
|
548
|
+
literalClauses: [{ field: "From", value: "billing@example.com" }],
|
|
549
|
+
actionLabelId: "None",
|
|
550
|
+
actionMailboxId: "mbx-invoices",
|
|
551
|
+
createdAt: 0,
|
|
552
|
+
updatedAt: 0,
|
|
553
|
+
};
|
|
554
|
+
|
|
555
|
+
afterEach(() => {
|
|
556
|
+
mock.restoreAll();
|
|
557
|
+
_resetForTest();
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
const world = (filter: FilterItem) => {
|
|
561
|
+
process.env.SQS_QUEUE_URL_ACCOUNT_FANOUT =
|
|
562
|
+
"http://localhost:9324/queue/account-fanout-test";
|
|
563
|
+
const jobs: CreateOrganizeJobRequestInput[] = [];
|
|
564
|
+
const enqueued: SendMessageCommand[] = [];
|
|
565
|
+
mock.method(sqsClient, "send", async (command: SendMessageCommand) => {
|
|
566
|
+
enqueued.push(command);
|
|
567
|
+
return {};
|
|
568
|
+
});
|
|
569
|
+
setClient({
|
|
570
|
+
account: {
|
|
571
|
+
get: async () => ({
|
|
572
|
+
accountId: ACCOUNT_ID,
|
|
573
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
574
|
+
}),
|
|
575
|
+
},
|
|
576
|
+
mailbox: {
|
|
577
|
+
get: async () => ({
|
|
578
|
+
mailboxId: "mbx-invoices",
|
|
579
|
+
accountId: ACCOUNT_ID,
|
|
580
|
+
fullPath: "Invoices",
|
|
581
|
+
syncStatus: "synced",
|
|
582
|
+
}),
|
|
583
|
+
},
|
|
584
|
+
filterAnchorTransaction: { createWithAnchor: async () => filter },
|
|
585
|
+
organizeJobRequest: {
|
|
586
|
+
create: async (input: CreateOrganizeJobRequestInput) => {
|
|
587
|
+
jobs.push(input);
|
|
588
|
+
return { ...input, organizeJobId: "job-new", state: "Pending" };
|
|
589
|
+
},
|
|
590
|
+
},
|
|
591
|
+
} as unknown as RemitClient);
|
|
592
|
+
return { jobs, enqueued };
|
|
593
|
+
};
|
|
594
|
+
|
|
595
|
+
const createFilter = (body: Partial<CreateFilterInput>) =>
|
|
596
|
+
(
|
|
597
|
+
FilterOperations.FilterOperations_createFilter as (
|
|
598
|
+
context: Context,
|
|
599
|
+
event: APIGatewayProxyEvent,
|
|
600
|
+
) => Promise<Record<string, unknown>>
|
|
601
|
+
)(
|
|
602
|
+
{
|
|
603
|
+
request: {
|
|
604
|
+
params: { accountId: ACCOUNT_ID },
|
|
605
|
+
requestBody: {
|
|
606
|
+
name: created.name,
|
|
607
|
+
scope: created.scope,
|
|
608
|
+
matchOperator: created.matchOperator,
|
|
609
|
+
literalClauses: created.literalClauses,
|
|
610
|
+
actionLabelId: created.actionLabelId,
|
|
611
|
+
actionMailboxId: created.actionMailboxId,
|
|
612
|
+
...body,
|
|
613
|
+
},
|
|
614
|
+
},
|
|
615
|
+
} as unknown as Context,
|
|
616
|
+
{
|
|
617
|
+
requestContext: { authorizer: { claims: { sub: SUB } } },
|
|
618
|
+
} as unknown as APIGatewayProxyEvent,
|
|
619
|
+
);
|
|
620
|
+
|
|
621
|
+
it("queues an organize job for the created filter and returns its id", async () => {
|
|
622
|
+
const { jobs, enqueued } = world(created);
|
|
623
|
+
|
|
624
|
+
const response = await createFilter({});
|
|
625
|
+
|
|
626
|
+
assert.equal(jobs.length, 1);
|
|
627
|
+
assert.equal(jobs[0]?.filterId, "flt-new");
|
|
628
|
+
assert.equal(enqueued.length, 1);
|
|
629
|
+
assert.match(String(enqueued[0]?.input.MessageBody), /job-new/);
|
|
630
|
+
assert.equal(response.organizeJobId, "job-new");
|
|
631
|
+
assert.equal(response.filterId, "flt-new");
|
|
632
|
+
});
|
|
633
|
+
|
|
634
|
+
it("names only the filter: the predicate columns stay at their sentinels", async () => {
|
|
635
|
+
const { jobs } = world(created);
|
|
636
|
+
|
|
637
|
+
await createFilter({});
|
|
638
|
+
|
|
639
|
+
assert.equal(jobs[0]?.anchorMessageId, "None");
|
|
640
|
+
assert.deepEqual(jobs[0]?.literalClauses, []);
|
|
641
|
+
assert.equal(jobs[0]?.actionLabelId, "None");
|
|
642
|
+
assert.equal(jobs[0]?.actionMailboxId, "None");
|
|
643
|
+
});
|
|
644
|
+
});
|
package/src/handlers/filter.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type {
|
|
2
2
|
CreateFilterInput,
|
|
3
|
+
CreateFilterResponse,
|
|
3
4
|
FilterResponse,
|
|
4
5
|
UpdateFilterInput as UpdateFilterRequestBody,
|
|
5
6
|
} from "@remit/api-openapi-types";
|
|
@@ -19,9 +20,10 @@ import {
|
|
|
19
20
|
} from "@remit/domain-enums";
|
|
20
21
|
import type { AnchorPayload } from "@remit/search-service";
|
|
21
22
|
import type { APIGatewayProxyEvent } from "aws-lambda";
|
|
22
|
-
import { getAccountConfigIdFromEvent } from "../auth.js";
|
|
23
|
+
import { getAccountConfigIdFromEvent, getSubFromEvent } from "../auth.js";
|
|
23
24
|
import { getClient } from "../service/data-client.js";
|
|
24
25
|
import { buildFilterAnchor } from "../service/filter.js";
|
|
26
|
+
import { queueOrganizeJob } from "../service/organize-queue.js";
|
|
25
27
|
import type {
|
|
26
28
|
FilterDetailOperationIds,
|
|
27
29
|
FilterOperationIds,
|
|
@@ -340,6 +342,16 @@ export const createFilterWithAnchor = async (
|
|
|
340
342
|
);
|
|
341
343
|
};
|
|
342
344
|
|
|
345
|
+
const requireUserId = (event: APIGatewayProxyEvent): string => {
|
|
346
|
+
const userId = getSubFromEvent(event);
|
|
347
|
+
if (!userId) {
|
|
348
|
+
throw new Error(
|
|
349
|
+
"Missing Cognito `sub`: cannot attribute an organize job to a user",
|
|
350
|
+
);
|
|
351
|
+
}
|
|
352
|
+
return userId;
|
|
353
|
+
};
|
|
354
|
+
|
|
343
355
|
/**
|
|
344
356
|
* A filter's `actionMailboxId` is a durable reference, so its target has to be
|
|
345
357
|
* a folder the mail server has settled (D12, first row) — 422 otherwise. The
|
|
@@ -393,11 +405,15 @@ export const FilterOperations: Record<
|
|
|
393
405
|
};
|
|
394
406
|
},
|
|
395
407
|
|
|
396
|
-
FilterOperations_createFilter: async (
|
|
408
|
+
FilterOperations_createFilter: async (
|
|
409
|
+
context,
|
|
410
|
+
...args: unknown[]
|
|
411
|
+
): Promise<CreateFilterResponse> => {
|
|
397
412
|
const event = args[0] as APIGatewayProxyEvent;
|
|
398
413
|
const accountConfigId = getAccountConfigIdFromEvent(event);
|
|
399
414
|
const { accountId } = context.request.params as { accountId: string };
|
|
400
415
|
const input = context.request.requestBody as CreateFilterInput;
|
|
416
|
+
const userId = requireUserId(event);
|
|
401
417
|
|
|
402
418
|
const client = await getClient();
|
|
403
419
|
const account = await client.account.get(accountId);
|
|
@@ -412,7 +428,14 @@ export const FilterOperations: Record<
|
|
|
412
428
|
accountConfigId,
|
|
413
429
|
input,
|
|
414
430
|
);
|
|
415
|
-
|
|
431
|
+
// The new filter's first pass over the inbox. The worker reads the filter
|
|
432
|
+
// when it runs and refuses one that has nothing to run (#1354).
|
|
433
|
+
const job = await queueOrganizeJob(client, {
|
|
434
|
+
accountConfigId,
|
|
435
|
+
userId,
|
|
436
|
+
filterId: filter.filterId,
|
|
437
|
+
});
|
|
438
|
+
return { ...toFilterResponse(filter), organizeJobId: job.organizeJobId };
|
|
416
439
|
},
|
|
417
440
|
};
|
|
418
441
|
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* only removed by the user, so a `mailboxId` that names nothing never exists.
|
|
10
10
|
*/
|
|
11
11
|
import assert from "node:assert/strict";
|
|
12
|
-
import { afterEach, describe, it } from "node:test";
|
|
12
|
+
import { afterEach, describe, it, mock } from "node:test";
|
|
13
13
|
import type { MailboxItem } from "@remit/data-ports";
|
|
14
14
|
import { CanonicalMailboxRole, MailboxSyncStatus } from "@remit/domain-enums";
|
|
15
15
|
import type { APIGatewayProxyEvent } from "aws-lambda";
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
type RemitClient,
|
|
21
21
|
setClient,
|
|
22
22
|
} from "../service/data-client.js";
|
|
23
|
+
import { sqsClient } from "../service/sqs.js";
|
|
23
24
|
import { FilterDetailOperations, FilterOperations } from "./filter.js";
|
|
24
25
|
import { FolderRoleOperations } from "./folder-role.js";
|
|
25
26
|
import { MessageBulkOperations } from "./message.js";
|
|
@@ -55,6 +56,9 @@ const world = (syncStatus: MailboxItem["syncStatus"]): Writes => {
|
|
|
55
56
|
copies: [],
|
|
56
57
|
};
|
|
57
58
|
const row = target(syncStatus);
|
|
59
|
+
process.env.SQS_QUEUE_URL_ACCOUNT_FANOUT =
|
|
60
|
+
"http://localhost:9324/queue/account-fanout-test";
|
|
61
|
+
mock.method(sqsClient, "send", async () => ({}));
|
|
58
62
|
|
|
59
63
|
setClient({
|
|
60
64
|
account: {
|
|
@@ -85,6 +89,13 @@ const world = (syncStatus: MailboxItem["syncStatus"]): Writes => {
|
|
|
85
89
|
return { filterId: "flt-1" };
|
|
86
90
|
},
|
|
87
91
|
},
|
|
92
|
+
organizeJobRequest: {
|
|
93
|
+
create: async (input: Record<string, unknown>) => ({
|
|
94
|
+
...input,
|
|
95
|
+
organizeJobId: "job-1",
|
|
96
|
+
state: "Pending",
|
|
97
|
+
}),
|
|
98
|
+
},
|
|
88
99
|
accountSetting: {
|
|
89
100
|
get: async () => null,
|
|
90
101
|
upsert: async (item: unknown) => {
|
|
@@ -168,6 +179,7 @@ const SITES = [
|
|
|
168
179
|
|
|
169
180
|
describe("a write that binds to a folder requires a settled folder", () => {
|
|
170
181
|
afterEach(() => {
|
|
182
|
+
mock.restoreAll();
|
|
171
183
|
_resetForTest();
|
|
172
184
|
});
|
|
173
185
|
|
|
@@ -4,11 +4,12 @@ import type { SendMessageCommand } from "@aws-sdk/client-sqs";
|
|
|
4
4
|
import type { OrganizeInput } from "@remit/api-openapi-types";
|
|
5
5
|
import type {
|
|
6
6
|
CreateOrganizeJobRequestInput,
|
|
7
|
+
FilterItem,
|
|
7
8
|
IAccountRepository,
|
|
8
9
|
IOrganizeJobRequestRepository,
|
|
9
10
|
OrganizeJobRequestItem,
|
|
10
11
|
} from "@remit/data-ports";
|
|
11
|
-
import { BadRequestError } from "@remit/data-ports/errors";
|
|
12
|
+
import { BadRequestError, NotFoundError } from "@remit/data-ports/errors";
|
|
12
13
|
import { FilterMatchOperator } from "@remit/domain-enums";
|
|
13
14
|
import type { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda";
|
|
14
15
|
import type { Context } from "openapi-backend";
|
|
@@ -186,3 +187,91 @@ describe("createOrganizeJob rejected-rule refusal (reader #995)", () => {
|
|
|
186
187
|
);
|
|
187
188
|
});
|
|
188
189
|
});
|
|
190
|
+
|
|
191
|
+
describe("createOrganizeJob by filterId — Run now (#1354)", () => {
|
|
192
|
+
afterEach(() => {
|
|
193
|
+
mock.restoreAll();
|
|
194
|
+
_resetForTest();
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
const standing = (state: FilterItem["state"]): FilterItem => ({
|
|
198
|
+
filterId: "flt-1",
|
|
199
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
200
|
+
name: "Invoices",
|
|
201
|
+
scope: "Standing",
|
|
202
|
+
state,
|
|
203
|
+
disabledReason: state === "Active" ? "None" : "UserDisabled",
|
|
204
|
+
hasAnchor: false,
|
|
205
|
+
ruleChangedAt: 1,
|
|
206
|
+
actionChangedAt: 1,
|
|
207
|
+
matchOperator: FilterMatchOperator.And,
|
|
208
|
+
literalClauses: [{ field: "From", value: "billing@example.com" }],
|
|
209
|
+
actionLabelId: "None",
|
|
210
|
+
actionMailboxId: "mbx-invoices",
|
|
211
|
+
createdAt: 0,
|
|
212
|
+
updatedAt: 0,
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
const withFilter = (filter: FilterItem): void => {
|
|
216
|
+
process.env.SQS_QUEUE_URL_ACCOUNT_FANOUT =
|
|
217
|
+
"http://localhost:9324/queue/account-fanout-test";
|
|
218
|
+
enqueued.length = 0;
|
|
219
|
+
createdRows.length = 0;
|
|
220
|
+
mock.method(sqsClient, "send", async (command: SendMessageCommand) => {
|
|
221
|
+
enqueued.push(command);
|
|
222
|
+
return {};
|
|
223
|
+
});
|
|
224
|
+
setClient({
|
|
225
|
+
account: {
|
|
226
|
+
get: async () => ({
|
|
227
|
+
accountId: ACCOUNT_ID,
|
|
228
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
229
|
+
}),
|
|
230
|
+
},
|
|
231
|
+
filter: {
|
|
232
|
+
get: async () => filter,
|
|
233
|
+
},
|
|
234
|
+
organizeJobRequest: {
|
|
235
|
+
create: async (row: CreateOrganizeJobRequestInput) => {
|
|
236
|
+
createdRows.push(row);
|
|
237
|
+
return { ...row, organizeJobId: "job-run", state: "Pending" };
|
|
238
|
+
},
|
|
239
|
+
},
|
|
240
|
+
} as unknown as RemitClient);
|
|
241
|
+
};
|
|
242
|
+
|
|
243
|
+
it("queues a job that names only the filter", async () => {
|
|
244
|
+
withFilter(standing("Active"));
|
|
245
|
+
|
|
246
|
+
const response = await postOrganize(input({ filterId: "flt-1" }));
|
|
247
|
+
|
|
248
|
+
assert.equal(response.statusCode, 202);
|
|
249
|
+
assert.equal(createdRows[0]?.filterId, "flt-1");
|
|
250
|
+
assert.equal(createdRows[0]?.actionMailboxId, "None");
|
|
251
|
+
assert.deepEqual(createdRows[0]?.literalClauses, []);
|
|
252
|
+
assert.equal(enqueued.length, 1);
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
it("answers an unknown filter with a 404 and queues nothing", async () => {
|
|
256
|
+
withFilter(standing("Active"));
|
|
257
|
+
setClient({
|
|
258
|
+
account: {
|
|
259
|
+
get: async () => ({
|
|
260
|
+
accountId: ACCOUNT_ID,
|
|
261
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
262
|
+
}),
|
|
263
|
+
},
|
|
264
|
+
filter: {
|
|
265
|
+
get: async () => {
|
|
266
|
+
throw new NotFoundError("Filter not found: flt-gone");
|
|
267
|
+
},
|
|
268
|
+
},
|
|
269
|
+
} as unknown as RemitClient);
|
|
270
|
+
|
|
271
|
+
const response = await postOrganize(input({ filterId: "flt-gone" }));
|
|
272
|
+
|
|
273
|
+
assert.equal(response.statusCode, 404);
|
|
274
|
+
assert.deepEqual(createdRows, []);
|
|
275
|
+
assert.deepEqual(enqueued, []);
|
|
276
|
+
});
|
|
277
|
+
});
|
package/src/handlers/organize.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { SendMessageCommand } from "@aws-sdk/client-sqs";
|
|
2
1
|
import type {
|
|
3
2
|
CreateOrganizeJobResponse,
|
|
4
3
|
OrganizeInput,
|
|
@@ -6,9 +5,7 @@ import type {
|
|
|
6
5
|
OrganizePreviewResponse,
|
|
7
6
|
} from "@remit/api-openapi-types";
|
|
8
7
|
import { BadRequestError, NotFoundError } from "@remit/data-ports/errors";
|
|
9
|
-
import { logger } from "@remit/logger-lambda";
|
|
10
8
|
import type { APIGatewayProxyEvent } from "aws-lambda";
|
|
11
|
-
import { env } from "expect-env";
|
|
12
9
|
import type { Context } from "openapi-backend";
|
|
13
10
|
import { getAccountConfigIdFromEvent, getSubFromEvent } from "../auth.js";
|
|
14
11
|
import { getClient, type RemitClient } from "../service/data-client.js";
|
|
@@ -19,7 +16,10 @@ import {
|
|
|
19
16
|
type OrganizePredicate,
|
|
20
17
|
organizePredicateRejection,
|
|
21
18
|
} from "../service/organize.js";
|
|
22
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
type OrganizeJobRequest,
|
|
21
|
+
queueOrganizeJob,
|
|
22
|
+
} from "../service/organize-queue.js";
|
|
23
23
|
import type {
|
|
24
24
|
OperationHandler,
|
|
25
25
|
OrganizeJobDetailOperationIds,
|
|
@@ -29,13 +29,6 @@ import { assertAccountOwnership } from "./account-ownership.js";
|
|
|
29
29
|
|
|
30
30
|
const NONE = "None";
|
|
31
31
|
|
|
32
|
-
/**
|
|
33
|
-
* How long a finished (or abandoned) job row lingers before the table-wide TTL
|
|
34
|
-
* reclaims it (RFC 034 Decision 1). A back-apply runs once and needs no standing
|
|
35
|
-
* lifetime; a week is ample for a client to poll the result.
|
|
36
|
-
*/
|
|
37
|
-
const ORGANIZE_JOB_TTL_SECONDS = 7 * 24 * 60 * 60;
|
|
38
|
-
|
|
39
32
|
/**
|
|
40
33
|
* Normalize the request body into the flattened predicate the job row and the
|
|
41
34
|
* matcher share. `anchorMessageId` collapses to the `"None"` sentinel when
|
|
@@ -70,6 +63,7 @@ const toOrganizeJobResponse = (
|
|
|
70
63
|
accountConfigId: job.accountConfigId,
|
|
71
64
|
userId: job.userId,
|
|
72
65
|
state: job.state,
|
|
66
|
+
filterId: job.filterId,
|
|
73
67
|
anchorMessageId: job.anchorMessageId,
|
|
74
68
|
matchOperator: job.matchOperator,
|
|
75
69
|
literalClauses: job.literalClauses,
|
|
@@ -84,6 +78,27 @@ const toOrganizeJobResponse = (
|
|
|
84
78
|
updatedAt: job.updatedAt,
|
|
85
79
|
});
|
|
86
80
|
|
|
81
|
+
/**
|
|
82
|
+
* What a create asks the worker to run. A saved filter is named and read when
|
|
83
|
+
* the job runs, so Run now and a filter's create share one path (#1354); an
|
|
84
|
+
* "all like these" predicate the matcher can never honour is answered here,
|
|
85
|
+
* not with a 202 the worker has to fail later (reader #463).
|
|
86
|
+
*/
|
|
87
|
+
const requestedJob = async (
|
|
88
|
+
client: RemitClient,
|
|
89
|
+
owner: { accountConfigId: string; userId: string },
|
|
90
|
+
input: OrganizeInput,
|
|
91
|
+
): Promise<OrganizeJobRequest> => {
|
|
92
|
+
if (input.filterId) {
|
|
93
|
+
await client.filter.get(owner.accountConfigId, input.filterId);
|
|
94
|
+
return { ...owner, filterId: input.filterId };
|
|
95
|
+
}
|
|
96
|
+
const predicate = predicateFromInput(input);
|
|
97
|
+
const rejection = organizePredicateRejection(predicate);
|
|
98
|
+
if (rejection) throw new BadRequestError(rejection.message);
|
|
99
|
+
return { ...owner, predicate };
|
|
100
|
+
};
|
|
101
|
+
|
|
87
102
|
export const OrganizeOperations: Record<
|
|
88
103
|
OrganizeOperationIds,
|
|
89
104
|
OperationHandler<OrganizeOperationIds>
|
|
@@ -107,42 +122,9 @@ export const OrganizeOperations: Record<
|
|
|
107
122
|
const client = await getClient();
|
|
108
123
|
await assertAccount(client, accountId, accountConfigId, "act");
|
|
109
124
|
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
// the worker has to fail later (reader #463).
|
|
114
|
-
const rejection = organizePredicateRejection(predicate);
|
|
115
|
-
if (rejection) throw new BadRequestError(rejection.message);
|
|
116
|
-
|
|
117
|
-
const ttl = Math.floor(Date.now() / 1000) + ORGANIZE_JOB_TTL_SECONDS;
|
|
118
|
-
|
|
119
|
-
const job = await client.organizeJobRequest.create({
|
|
120
|
-
accountConfigId,
|
|
121
|
-
userId,
|
|
122
|
-
anchorMessageId: predicate.anchorMessageId,
|
|
123
|
-
matchOperator: predicate.matchOperator,
|
|
124
|
-
literalClauses: predicate.literalClauses,
|
|
125
|
-
similarityThreshold: predicate.similarityThreshold,
|
|
126
|
-
actionLabelId: predicate.actionLabelId,
|
|
127
|
-
actionMailboxId: predicate.actionMailboxId,
|
|
128
|
-
ttl,
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
await sqsClient.send(
|
|
132
|
-
new SendMessageCommand({
|
|
133
|
-
QueueUrl: env.SQS_QUEUE_URL_ACCOUNT_FANOUT,
|
|
134
|
-
MessageBody: JSON.stringify({
|
|
135
|
-
type: "OrganizeJob",
|
|
136
|
-
accountConfigId,
|
|
137
|
-
organizeJobId: job.organizeJobId,
|
|
138
|
-
}),
|
|
139
|
-
}),
|
|
140
|
-
);
|
|
141
|
-
|
|
142
|
-
// biome-ignore lint/plugin/no-logger-info: a back-apply is an audit-grade signal
|
|
143
|
-
logger.info(
|
|
144
|
-
{ accountConfigId, organizeJobId: job.organizeJobId },
|
|
145
|
-
"Organize back-apply job initiated",
|
|
125
|
+
const job = await queueOrganizeJob(
|
|
126
|
+
client,
|
|
127
|
+
await requestedJob(client, { accountConfigId, userId }, input),
|
|
146
128
|
);
|
|
147
129
|
|
|
148
130
|
return {
|
|
@@ -164,6 +146,11 @@ export const OrganizeOperations: Record<
|
|
|
164
146
|
const client = await getClient();
|
|
165
147
|
await assertAccount(client, accountId, accountConfigId, "read");
|
|
166
148
|
|
|
149
|
+
if (input.filterId) {
|
|
150
|
+
throw new BadRequestError(
|
|
151
|
+
"Preview takes a rule, not a saved filter. Run the filter to apply it.",
|
|
152
|
+
);
|
|
153
|
+
}
|
|
167
154
|
const result = await matchOrganize(
|
|
168
155
|
buildOrganizeMatchDeps(client),
|
|
169
156
|
accountConfigId,
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { SendMessageCommand } from "@aws-sdk/client-sqs";
|
|
2
|
+
import type {
|
|
3
|
+
CreateOrganizeJobRequestInput,
|
|
4
|
+
OrganizeJobRequestItem,
|
|
5
|
+
} from "@remit/data-ports";
|
|
6
|
+
import { FilterMatchOperator } from "@remit/domain-enums";
|
|
7
|
+
import { logger } from "@remit/logger-lambda";
|
|
8
|
+
import {
|
|
9
|
+
DEFAULT_SEMANTIC_MATCH_THRESHOLD,
|
|
10
|
+
NO_ACTION,
|
|
11
|
+
} from "@remit/mailbox-service";
|
|
12
|
+
import { env } from "expect-env";
|
|
13
|
+
import type { RemitClient } from "./data-client.js";
|
|
14
|
+
import type { OrganizePredicate } from "./organize.js";
|
|
15
|
+
import { sqsClient } from "./sqs.js";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* How long a finished (or abandoned) job row lingers before the table-wide TTL
|
|
19
|
+
* reclaims it (RFC 034 Decision 1). A back-apply runs once and needs no standing
|
|
20
|
+
* lifetime; a week is ample for a client to poll the result.
|
|
21
|
+
*/
|
|
22
|
+
const ORGANIZE_JOB_TTL_SECONDS = 7 * 24 * 60 * 60;
|
|
23
|
+
|
|
24
|
+
interface JobOwner {
|
|
25
|
+
accountConfigId: string;
|
|
26
|
+
userId: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* An "all like these" pass carries its predicate; a filter job names only the
|
|
31
|
+
* filter, which the worker reads when it runs.
|
|
32
|
+
*/
|
|
33
|
+
export type OrganizeJobRequest = JobOwner &
|
|
34
|
+
({ predicate: OrganizePredicate } | { filterId: string });
|
|
35
|
+
|
|
36
|
+
/** A filter job's row: the filter id, every predicate column at its sentinel. */
|
|
37
|
+
const filterJobColumns = (
|
|
38
|
+
filterId: string,
|
|
39
|
+
): Omit<CreateOrganizeJobRequestInput, keyof JobOwner | "ttl"> => ({
|
|
40
|
+
filterId,
|
|
41
|
+
anchorMessageId: NO_ACTION,
|
|
42
|
+
matchOperator: FilterMatchOperator.And,
|
|
43
|
+
literalClauses: [],
|
|
44
|
+
similarityThreshold: DEFAULT_SEMANTIC_MATCH_THRESHOLD,
|
|
45
|
+
actionLabelId: NO_ACTION,
|
|
46
|
+
actionMailboxId: NO_ACTION,
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
const predicateColumns = (
|
|
50
|
+
predicate: OrganizePredicate,
|
|
51
|
+
): Omit<CreateOrganizeJobRequestInput, keyof JobOwner | "ttl"> => ({
|
|
52
|
+
filterId: NO_ACTION,
|
|
53
|
+
...predicate,
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Write a Pending job row and hand it to the account-fanout worker. Every
|
|
58
|
+
* back-apply goes through here: "all like these", a new filter's first pass,
|
|
59
|
+
* and Run now on a saved filter.
|
|
60
|
+
*/
|
|
61
|
+
export const queueOrganizeJob = async (
|
|
62
|
+
client: Pick<RemitClient, "organizeJobRequest">,
|
|
63
|
+
request: OrganizeJobRequest,
|
|
64
|
+
): Promise<OrganizeJobRequestItem> => {
|
|
65
|
+
const { accountConfigId, userId } = request;
|
|
66
|
+
const columns =
|
|
67
|
+
"filterId" in request
|
|
68
|
+
? filterJobColumns(request.filterId)
|
|
69
|
+
: predicateColumns(request.predicate);
|
|
70
|
+
const job = await client.organizeJobRequest.create({
|
|
71
|
+
accountConfigId,
|
|
72
|
+
userId,
|
|
73
|
+
...columns,
|
|
74
|
+
ttl: Math.floor(Date.now() / 1000) + ORGANIZE_JOB_TTL_SECONDS,
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
await sqsClient.send(
|
|
78
|
+
new SendMessageCommand({
|
|
79
|
+
QueueUrl: env.SQS_QUEUE_URL_ACCOUNT_FANOUT,
|
|
80
|
+
MessageBody: JSON.stringify({
|
|
81
|
+
type: "OrganizeJob",
|
|
82
|
+
accountConfigId,
|
|
83
|
+
organizeJobId: job.organizeJobId,
|
|
84
|
+
}),
|
|
85
|
+
}),
|
|
86
|
+
);
|
|
87
|
+
|
|
88
|
+
// biome-ignore lint/plugin/no-logger-info: a back-apply is an audit-grade signal
|
|
89
|
+
logger.info(
|
|
90
|
+
{
|
|
91
|
+
accountConfigId,
|
|
92
|
+
organizeJobId: job.organizeJobId,
|
|
93
|
+
filterId: job.filterId,
|
|
94
|
+
},
|
|
95
|
+
"Organize back-apply job initiated",
|
|
96
|
+
);
|
|
97
|
+
return job;
|
|
98
|
+
};
|
|
@@ -289,6 +289,7 @@ const matchDeps = (
|
|
|
289
289
|
},
|
|
290
290
|
listAccountFilterMessages: async () => ({ items: corpus }),
|
|
291
291
|
filterAnchors: {
|
|
292
|
+
get: async () => null,
|
|
292
293
|
listByAccountConfig: async () => filterAnchorRows,
|
|
293
294
|
put: async () => {
|
|
294
295
|
throw new Error("matchDeps must not repair an anchor");
|
|
@@ -330,6 +331,7 @@ const vectorlessDeps = (
|
|
|
330
331
|
},
|
|
331
332
|
listAccountFilterMessages: async () => ({ items: corpus }),
|
|
332
333
|
filterAnchors: {
|
|
334
|
+
get: async () => null,
|
|
333
335
|
listByAccountConfig: async () => [],
|
|
334
336
|
put: async () => {
|
|
335
337
|
throw moduleNotFound();
|
|
@@ -471,6 +473,7 @@ describe("matchOrganize honors the persisted FilterAnchor (reader #350)", () =>
|
|
|
471
473
|
}),
|
|
472
474
|
listAccountFilterMessages: async () => ({ items: [] }),
|
|
473
475
|
filterAnchors: {
|
|
476
|
+
get: async () => null,
|
|
474
477
|
listByAccountConfig: async () => [persistedAnchor],
|
|
475
478
|
put: async () => {
|
|
476
479
|
throw new Error("a current anchor must not be rewritten");
|
|
@@ -757,6 +760,7 @@ describe("matchOrganize on a deployment without the vector pipeline", () => {
|
|
|
757
760
|
}),
|
|
758
761
|
listAccountFilterMessages: async () => ({ items: [] }),
|
|
759
762
|
filterAnchors: {
|
|
763
|
+
get: async () => null,
|
|
760
764
|
listByAccountConfig: async () => [drifted],
|
|
761
765
|
put: async () => {
|
|
762
766
|
throw new Error("an unrepairable anchor must not be rewritten");
|
|
@@ -798,6 +802,7 @@ describe("matchOrganize on a deployment without the vector pipeline", () => {
|
|
|
798
802
|
}),
|
|
799
803
|
listAccountFilterMessages: async () => ({ items: [] }),
|
|
800
804
|
filterAnchors: {
|
|
805
|
+
get: async () => null,
|
|
801
806
|
listByAccountConfig: async () => [],
|
|
802
807
|
put: async () => {
|
|
803
808
|
throw new Error("unreachable");
|
|
@@ -1120,6 +1125,7 @@ describe("back-apply repairs a drifted anchor the way index-time matching does (
|
|
|
1120
1125
|
}),
|
|
1121
1126
|
listAccountFilterMessages: async () => ({ items: [] }),
|
|
1122
1127
|
filterAnchors: {
|
|
1128
|
+
get: async () => null,
|
|
1123
1129
|
listByAccountConfig: async () => anchors,
|
|
1124
1130
|
put: async (input: CreateFilterAnchorInput) => {
|
|
1125
1131
|
anchorPuts.push(input);
|
|
@@ -1319,6 +1325,7 @@ const storeBackedDeps = (
|
|
|
1319
1325
|
};
|
|
1320
1326
|
},
|
|
1321
1327
|
filterAnchors: {
|
|
1328
|
+
get: async () => null,
|
|
1322
1329
|
listByAccountConfig: async () => [],
|
|
1323
1330
|
put: async () => {
|
|
1324
1331
|
throw new Error("unreachable");
|
package/src/service/organize.ts
CHANGED
|
@@ -5,18 +5,22 @@ import type {
|
|
|
5
5
|
IFilterAnchorRepository,
|
|
6
6
|
OrganizeJobRequestItem,
|
|
7
7
|
} from "@remit/data-ports";
|
|
8
|
-
import {
|
|
8
|
+
import { NotFoundError } from "@remit/data-ports/errors";
|
|
9
|
+
import { FilterClauseField, FilterState } from "@remit/domain-enums";
|
|
9
10
|
import { logger } from "@remit/logger-lambda";
|
|
10
11
|
import {
|
|
11
12
|
DEFAULT_SEMANTIC_MATCH_THRESHOLD,
|
|
12
13
|
type FilterMessage,
|
|
14
|
+
FilterPipeline,
|
|
13
15
|
type LiteralClauseNarrowing,
|
|
14
16
|
literalClausesMatch,
|
|
15
17
|
literalClauseTerms,
|
|
18
|
+
type MessageEmbedder,
|
|
16
19
|
NO_ACTION,
|
|
17
20
|
PlacementMoveService,
|
|
18
21
|
refreshAnchorForEmbedder,
|
|
19
22
|
} from "@remit/mailbox-service";
|
|
23
|
+
import { buildFilterConfig } from "@remit/mailbox-service/filter-config";
|
|
20
24
|
import {
|
|
21
25
|
type AnchorPayload,
|
|
22
26
|
buildMessageAnchor,
|
|
@@ -135,6 +139,18 @@ export interface OrganizeCandidatePage {
|
|
|
135
139
|
export interface OrganizeCandidateQuery extends LiteralClauseNarrowing {
|
|
136
140
|
limit: number;
|
|
137
141
|
continuationToken?: string;
|
|
142
|
+
mailboxIds?: readonly string[];
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Where a pass looks and which standing filter it runs for. A filter
|
|
147
|
+
* back-apply reads that filter's own FilterAnchor row by id, narrows both the
|
|
148
|
+
* literal corpus and the vector query to the inbox, and matches a body-content
|
|
149
|
+
* clause on the stored preview; an "all like these" pass sets neither.
|
|
150
|
+
*/
|
|
151
|
+
export interface OrganizeMatchScope {
|
|
152
|
+
filterId?: string;
|
|
153
|
+
mailboxIds?: readonly string[];
|
|
138
154
|
}
|
|
139
155
|
|
|
140
156
|
export interface OrganizeMatchDeps {
|
|
@@ -159,16 +175,19 @@ export interface OrganizeMatchDeps {
|
|
|
159
175
|
/**
|
|
160
176
|
* Every persisted FilterAnchor for the account — read-only, always
|
|
161
177
|
* available (never gated behind {@link semantic}, since it never touches
|
|
162
|
-
* the vector store). A back-apply
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
*
|
|
178
|
+
* the vector store). A filter back-apply reads its own row by `filterId`
|
|
179
|
+
* (#1354); an "all like these" predicate carries only a bare
|
|
180
|
+
* `anchorMessageId`, so this is how {@link matchSemantic} finds "the
|
|
181
|
+
* standing filter this anchor came from," if one still exists, to read its
|
|
182
|
+
* fixed-at-save-time vector instead of re-deriving one from the anchor
|
|
183
|
+
* message's current chunks (reader #350 / RFC 039 Decision 1).
|
|
168
184
|
* `put` is the write half of the lazy anchor-drift repair
|
|
169
185
|
* ({@link refreshAnchorForEmbedder}), never a new standing rule.
|
|
170
186
|
*/
|
|
171
|
-
filterAnchors: Pick<
|
|
187
|
+
filterAnchors: Pick<
|
|
188
|
+
IFilterAnchorRepository,
|
|
189
|
+
"get" | "listByAccountConfig" | "put"
|
|
190
|
+
>;
|
|
172
191
|
}
|
|
173
192
|
|
|
174
193
|
/**
|
|
@@ -179,7 +198,10 @@ export interface OrganizeMatchDeps {
|
|
|
179
198
|
* make it (reader #463).
|
|
180
199
|
*/
|
|
181
200
|
export interface OrganizeRejection {
|
|
182
|
-
reason:
|
|
201
|
+
reason:
|
|
202
|
+
| "BodyContentWithoutVectorPipeline"
|
|
203
|
+
| "FilterNotActive"
|
|
204
|
+
| "FilterMissing";
|
|
183
205
|
message: string;
|
|
184
206
|
}
|
|
185
207
|
|
|
@@ -246,9 +268,9 @@ const filterMessageFromChunks = (
|
|
|
246
268
|
/**
|
|
247
269
|
* The persisted FilterAnchor whose `anchorMessageId` matches this predicate's
|
|
248
270
|
* anchor, if the account has a standing filter built from that message (RFC
|
|
249
|
-
* 034 Decision 2 / RFC 039 Decision 1, reader #350).
|
|
250
|
-
*
|
|
251
|
-
* only way to recover "the filter this anchor came from" is by the value it
|
|
271
|
+
* 034 Decision 2 / RFC 039 Decision 1, reader #350). An "all like these"
|
|
272
|
+
* predicate carries no `filterId` — that job structurally is not a Filter — so
|
|
273
|
+
* the only way to recover "the filter this anchor came from" is by the value it
|
|
252
274
|
* was anchored on. The scan is over one account's (small, bounded) anchor
|
|
253
275
|
* rows, read-only, and never touches the vector store. Returns `undefined`
|
|
254
276
|
* when no standing filter was ever anchored on this message (or it has since
|
|
@@ -331,24 +353,38 @@ const matchSemantic = async (
|
|
|
331
353
|
accountConfigId: string,
|
|
332
354
|
predicate: OrganizePredicate,
|
|
333
355
|
limit: number,
|
|
356
|
+
scope: OrganizeMatchScope,
|
|
334
357
|
): Promise<{ messageIds: string[]; indexEmpty: boolean }> => {
|
|
335
358
|
const semantic = deps.semantic();
|
|
336
|
-
const persisted =
|
|
337
|
-
deps.filterAnchors,
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
359
|
+
const persisted = scope.filterId
|
|
360
|
+
? ((await deps.filterAnchors.get(accountConfigId, scope.filterId)) ??
|
|
361
|
+
undefined)
|
|
362
|
+
: await findPersistedAnchor(
|
|
363
|
+
deps.filterAnchors,
|
|
364
|
+
accountConfigId,
|
|
365
|
+
predicate.anchorMessageId,
|
|
366
|
+
);
|
|
341
367
|
const anchor: AnchorPayload | null = persisted
|
|
342
368
|
? await anchorForWiden(deps, semantic, persisted)
|
|
343
369
|
: await semantic.buildAnchor(accountConfigId, predicate.anchorMessageId);
|
|
344
370
|
if (!anchor) return { messageIds: [], indexEmpty: true };
|
|
345
371
|
const threshold =
|
|
346
372
|
predicate.similarityThreshold ?? DEFAULT_SEMANTIC_MATCH_THRESHOLD;
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
373
|
+
const folders: readonly (string | undefined)[] = scope.mailboxIds ?? [
|
|
374
|
+
undefined,
|
|
375
|
+
];
|
|
376
|
+
const pages = await Promise.all(
|
|
377
|
+
folders.map((mailboxId) =>
|
|
378
|
+
semantic.vectorStore.query({
|
|
379
|
+
vector: anchor.anchorEmbedding,
|
|
380
|
+
topK: limit * VECTOR_CHUNK_FACTOR,
|
|
381
|
+
filter: mailboxId
|
|
382
|
+
? { accountConfigId, mailboxId }
|
|
383
|
+
: { accountConfigId },
|
|
384
|
+
}),
|
|
385
|
+
),
|
|
386
|
+
);
|
|
387
|
+
const matches = pages.flat();
|
|
352
388
|
if (matches.length === 0) return { messageIds: [], indexEmpty: true };
|
|
353
389
|
const bestScore = new Map<string, number>();
|
|
354
390
|
for (const match of matches) {
|
|
@@ -436,6 +472,7 @@ const matchLiteral = async (
|
|
|
436
472
|
accountConfigId: string,
|
|
437
473
|
predicate: OrganizePredicate,
|
|
438
474
|
limit: number,
|
|
475
|
+
scope: OrganizeMatchScope,
|
|
439
476
|
): Promise<string[]> => {
|
|
440
477
|
const clauses = predicate.literalClauses;
|
|
441
478
|
const narrowing = literalClauseTerms(clauses, predicate.matchOperator);
|
|
@@ -449,6 +486,7 @@ const matchLiteral = async (
|
|
|
449
486
|
...narrowing,
|
|
450
487
|
limit,
|
|
451
488
|
continuationToken,
|
|
489
|
+
mailboxIds: scope.mailboxIds,
|
|
452
490
|
});
|
|
453
491
|
for (const { messageId, message } of page.items) {
|
|
454
492
|
if (seen.has(messageId)) continue;
|
|
@@ -488,6 +526,7 @@ export const matchOrganize = async (
|
|
|
488
526
|
accountConfigId: string,
|
|
489
527
|
predicate: OrganizePredicate,
|
|
490
528
|
limit: number = ORGANIZE_MATCH_LIMIT,
|
|
529
|
+
scope: OrganizeMatchScope = {},
|
|
491
530
|
): Promise<OrganizeMatchResult> => {
|
|
492
531
|
const anchored = hasAnchor(predicate);
|
|
493
532
|
const clauses = predicate.literalClauses;
|
|
@@ -501,13 +540,16 @@ export const matchOrganize = async (
|
|
|
501
540
|
}
|
|
502
541
|
|
|
503
542
|
if (!anchored) {
|
|
504
|
-
const rejection =
|
|
543
|
+
const rejection = scope.filterId
|
|
544
|
+
? null
|
|
545
|
+
: organizePredicateRejection(predicate);
|
|
505
546
|
if (rejection) return { rejected: rejection };
|
|
506
547
|
const messageIds = await matchLiteral(
|
|
507
548
|
deps,
|
|
508
549
|
accountConfigId,
|
|
509
550
|
predicate,
|
|
510
551
|
limit,
|
|
552
|
+
scope,
|
|
511
553
|
);
|
|
512
554
|
return {
|
|
513
555
|
rejected: null,
|
|
@@ -523,6 +565,7 @@ export const matchOrganize = async (
|
|
|
523
565
|
accountConfigId,
|
|
524
566
|
predicate,
|
|
525
567
|
limit,
|
|
568
|
+
scope,
|
|
526
569
|
);
|
|
527
570
|
return {
|
|
528
571
|
rejected: null,
|
|
@@ -543,13 +586,14 @@ export const matchOrganize = async (
|
|
|
543
586
|
semanticIndexEmpty: false,
|
|
544
587
|
};
|
|
545
588
|
}
|
|
546
|
-
const rejection = bodyContentRejection(clauses);
|
|
589
|
+
const rejection = scope.filterId ? null : bodyContentRejection(clauses);
|
|
547
590
|
if (rejection) return { rejected: rejection };
|
|
548
591
|
const messageIds = await matchLiteral(
|
|
549
592
|
deps,
|
|
550
593
|
accountConfigId,
|
|
551
594
|
predicate,
|
|
552
595
|
limit,
|
|
596
|
+
scope,
|
|
553
597
|
);
|
|
554
598
|
return {
|
|
555
599
|
rejected: null,
|
|
@@ -591,12 +635,11 @@ const buildSemanticFromEnv = (): OrganizeSemanticDeps => {
|
|
|
591
635
|
*
|
|
592
636
|
* `From`/`Subject`/`listId` come from the row verbatim (full fidelity), so a
|
|
593
637
|
* `From`, `Subject`, `FromDomain` or `ListId` clause matches here exactly as it
|
|
594
|
-
* does at index time. `text`
|
|
595
|
-
*
|
|
596
|
-
* a body-content clause
|
|
597
|
-
*
|
|
598
|
-
*
|
|
599
|
-
* empty `text` is never matched against.
|
|
638
|
+
* does at index time. `text` is the row's stored body preview (`snippet`),
|
|
639
|
+
* the same text search matches free words against. An "all like these" pass
|
|
640
|
+
* refuses a body-content (`HasWords`) clause before reading it (see
|
|
641
|
+
* {@link bodyContentRejection}); a filter back-apply matches it on the
|
|
642
|
+
* preview, the mail search showed the user when the rule was made (#1354).
|
|
600
643
|
*/
|
|
601
644
|
const listAccountFilterMessagesFromClient =
|
|
602
645
|
(client: RemitClient): OrganizeMatchDeps["listAccountFilterMessages"] =>
|
|
@@ -609,6 +652,7 @@ const listAccountFilterMessagesFromClient =
|
|
|
609
652
|
limit: query.limit,
|
|
610
653
|
continuationToken: query.continuationToken,
|
|
611
654
|
excludeDeleted: true,
|
|
655
|
+
mailboxIds: query.mailboxIds,
|
|
612
656
|
},
|
|
613
657
|
);
|
|
614
658
|
return {
|
|
@@ -618,7 +662,7 @@ const listAccountFilterMessagesFromClient =
|
|
|
618
662
|
from: row.fromEmail ?? "",
|
|
619
663
|
fromName: row.fromName ?? "",
|
|
620
664
|
subject: row.subject ?? "",
|
|
621
|
-
text: "",
|
|
665
|
+
text: row.snippet ?? "",
|
|
622
666
|
listId: row.listId ?? "",
|
|
623
667
|
},
|
|
624
668
|
})),
|
|
@@ -754,3 +798,228 @@ export const applyOrganize = async (
|
|
|
754
798
|
}
|
|
755
799
|
return { applied, failed };
|
|
756
800
|
};
|
|
801
|
+
|
|
802
|
+
const FILTER_NOT_ACTIVE_MESSAGE =
|
|
803
|
+
"This filter is turned off or has expired. Turn it on to run it over the inbox.";
|
|
804
|
+
|
|
805
|
+
const FILTER_MISSING_MESSAGE =
|
|
806
|
+
"This filter was deleted before it could run over the inbox.";
|
|
807
|
+
|
|
808
|
+
const rejectFilter = (
|
|
809
|
+
reason: OrganizeRejection["reason"],
|
|
810
|
+
message: string,
|
|
811
|
+
): OrganizeMatchRejected => ({ rejected: { reason, message } });
|
|
812
|
+
|
|
813
|
+
/**
|
|
814
|
+
* A standing filter's predicate and action, read the way index time reads
|
|
815
|
+
* them. `matchSemantic` reads the anchor vector by `filterId`; the anchor
|
|
816
|
+
* message id here only says the filter widens.
|
|
817
|
+
*/
|
|
818
|
+
const predicateFromFilter = (
|
|
819
|
+
filter: FilterItem,
|
|
820
|
+
anchorMessageId: string,
|
|
821
|
+
): OrganizePredicate => ({
|
|
822
|
+
anchorMessageId,
|
|
823
|
+
matchOperator: filter.matchOperator,
|
|
824
|
+
literalClauses: filter.literalClauses,
|
|
825
|
+
similarityThreshold: DEFAULT_SEMANTIC_MATCH_THRESHOLD,
|
|
826
|
+
actionLabelId: filter.actionLabelId,
|
|
827
|
+
actionMailboxId: filter.actionMailboxId,
|
|
828
|
+
});
|
|
829
|
+
|
|
830
|
+
interface LoadedFilter {
|
|
831
|
+
rejected: null;
|
|
832
|
+
filter: FilterItem;
|
|
833
|
+
predicate: OrganizePredicate;
|
|
834
|
+
/**
|
|
835
|
+
* Marked `hasAnchor` with its FilterAnchor row gone. Index time matches
|
|
836
|
+
* such a filter against nothing, so the back-apply does too.
|
|
837
|
+
*/
|
|
838
|
+
matchesNothing: boolean;
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
/**
|
|
842
|
+
* Read the filter a job names: refused when it was deleted or is not Active,
|
|
843
|
+
* since index time evaluates neither.
|
|
844
|
+
*/
|
|
845
|
+
const loadFilter = async (
|
|
846
|
+
client: Pick<RemitClient, "filter" | "filterAnchor">,
|
|
847
|
+
accountConfigId: string,
|
|
848
|
+
filterId: string,
|
|
849
|
+
): Promise<LoadedFilter | OrganizeMatchRejected> => {
|
|
850
|
+
const stored = await client.filter
|
|
851
|
+
.get(accountConfigId, filterId)
|
|
852
|
+
.catch((error: unknown) => {
|
|
853
|
+
if (error instanceof NotFoundError) return null;
|
|
854
|
+
throw error;
|
|
855
|
+
});
|
|
856
|
+
if (!stored) return rejectFilter("FilterMissing", FILTER_MISSING_MESSAGE);
|
|
857
|
+
const filter = await client.filter.refreshExpiry(stored);
|
|
858
|
+
if (filter.state !== FilterState.Active) {
|
|
859
|
+
return rejectFilter("FilterNotActive", FILTER_NOT_ACTIVE_MESSAGE);
|
|
860
|
+
}
|
|
861
|
+
if (!filter.hasAnchor) {
|
|
862
|
+
return {
|
|
863
|
+
rejected: null,
|
|
864
|
+
filter,
|
|
865
|
+
predicate: predicateFromFilter(filter, NO_ACTION),
|
|
866
|
+
matchesNothing: false,
|
|
867
|
+
};
|
|
868
|
+
}
|
|
869
|
+
const anchor = await client.filterAnchor.get(accountConfigId, filterId);
|
|
870
|
+
return {
|
|
871
|
+
rejected: null,
|
|
872
|
+
filter,
|
|
873
|
+
predicate: predicateFromFilter(
|
|
874
|
+
filter,
|
|
875
|
+
anchor?.anchorMessageId ?? NO_ACTION,
|
|
876
|
+
),
|
|
877
|
+
matchesNothing: anchor === null,
|
|
878
|
+
};
|
|
879
|
+
};
|
|
880
|
+
|
|
881
|
+
/** Every inbox under the config — the folders a filter back-apply runs over. */
|
|
882
|
+
const inboxMailboxIds = async (
|
|
883
|
+
client: Pick<RemitClient, "account" | "mailboxSpecialUse">,
|
|
884
|
+
accountConfigId: string,
|
|
885
|
+
): Promise<string[]> => {
|
|
886
|
+
const accounts =
|
|
887
|
+
await client.account.listMailSourcesByAccountConfig(accountConfigId);
|
|
888
|
+
const inboxes = await Promise.all(
|
|
889
|
+
accounts.map((account) =>
|
|
890
|
+
client.mailboxSpecialUse.findInboxMailbox(account.accountId),
|
|
891
|
+
),
|
|
892
|
+
);
|
|
893
|
+
return inboxes.flatMap((inbox) => (inbox ? [inbox.mailboxId] : []));
|
|
894
|
+
};
|
|
895
|
+
|
|
896
|
+
/**
|
|
897
|
+
* The message as the index-time pipeline reads it, from its thread row. The
|
|
898
|
+
* body is the stored preview, so a `HasWords` clause matches what search
|
|
899
|
+
* matched. A message deleted since its vector was written has no row.
|
|
900
|
+
*/
|
|
901
|
+
const filterMessageOf = async (
|
|
902
|
+
client: Pick<RemitClient, "threadMessage">,
|
|
903
|
+
accountConfigId: string,
|
|
904
|
+
messageId: string,
|
|
905
|
+
): Promise<FilterMessage | null> => {
|
|
906
|
+
const row = await client.threadMessage
|
|
907
|
+
.getByMessageId(accountConfigId, messageId)
|
|
908
|
+
.catch((error: unknown) => {
|
|
909
|
+
if (error instanceof NotFoundError) return null;
|
|
910
|
+
throw error;
|
|
911
|
+
});
|
|
912
|
+
if (!row) return null;
|
|
913
|
+
return {
|
|
914
|
+
from: row.fromEmail ?? "",
|
|
915
|
+
fromName: row.fromName ?? "",
|
|
916
|
+
subject: row.subject ?? "",
|
|
917
|
+
text: row.snippet ?? "",
|
|
918
|
+
listId: row.listId ?? "",
|
|
919
|
+
};
|
|
920
|
+
};
|
|
921
|
+
|
|
922
|
+
export interface FilterBackApplyResult {
|
|
923
|
+
rejected: null;
|
|
924
|
+
matched: number;
|
|
925
|
+
applied: number;
|
|
926
|
+
failed: number;
|
|
927
|
+
semanticUnavailable: boolean;
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
export interface FilterBackApplyDeps {
|
|
931
|
+
client: RemitClient;
|
|
932
|
+
matchDeps: OrganizeMatchDeps;
|
|
933
|
+
moveService?: PlacementMoveService;
|
|
934
|
+
/** The embedder index time uses; read from env when absent. */
|
|
935
|
+
embedder?: MessageEmbedder;
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
/**
|
|
939
|
+
* Run one standing filter over the inbox as if each matching message had just
|
|
940
|
+
* arrived. Every message the matcher returns is a match: its label always
|
|
941
|
+
* applies, being additive. The index-time {@link FilterPipeline} only orders
|
|
942
|
+
* the exclusive move, so the move is skipped where it names another active
|
|
943
|
+
* filter. The pipeline re-scores a similarity filter against the stored
|
|
944
|
+
* preview rather than the full body, so its verdict on whether this filter
|
|
945
|
+
* matches is not asked; and an evaluate it could not finish names no other
|
|
946
|
+
* filter, so the move goes ahead rather than silently dropping out.
|
|
947
|
+
*/
|
|
948
|
+
export const backApplyFilter = async (
|
|
949
|
+
deps: FilterBackApplyDeps,
|
|
950
|
+
accountConfigId: string,
|
|
951
|
+
filterId: string,
|
|
952
|
+
): Promise<FilterBackApplyResult | OrganizeMatchRejected> => {
|
|
953
|
+
const { client, matchDeps, moveService } = deps;
|
|
954
|
+
const loaded = await loadFilter(client, accountConfigId, filterId);
|
|
955
|
+
if (loaded.rejected) return loaded;
|
|
956
|
+
const config = buildFilterConfig(
|
|
957
|
+
{
|
|
958
|
+
filterService: client.filter,
|
|
959
|
+
filterAnchorService: client.filterAnchor,
|
|
960
|
+
messageLabelService: client.messageLabel,
|
|
961
|
+
placementMoveService: moveService,
|
|
962
|
+
},
|
|
963
|
+
deps.embedder,
|
|
964
|
+
);
|
|
965
|
+
if (!config) {
|
|
966
|
+
throw new Error(
|
|
967
|
+
"Filter back-apply needs the message-management queue to move mail",
|
|
968
|
+
);
|
|
969
|
+
}
|
|
970
|
+
const pipeline = new FilterPipeline(config, logger);
|
|
971
|
+
|
|
972
|
+
const mailboxIds = await inboxMailboxIds(client, accountConfigId);
|
|
973
|
+
const nothing: FilterBackApplyResult = {
|
|
974
|
+
rejected: null,
|
|
975
|
+
matched: 0,
|
|
976
|
+
applied: 0,
|
|
977
|
+
failed: 0,
|
|
978
|
+
semanticUnavailable: false,
|
|
979
|
+
};
|
|
980
|
+
if (loaded.matchesNothing || mailboxIds.length === 0) return nothing;
|
|
981
|
+
|
|
982
|
+
const match = await matchOrganize(
|
|
983
|
+
matchDeps,
|
|
984
|
+
accountConfigId,
|
|
985
|
+
loaded.predicate,
|
|
986
|
+
ORGANIZE_MATCH_LIMIT,
|
|
987
|
+
{ filterId, mailboxIds },
|
|
988
|
+
);
|
|
989
|
+
if (match.rejected) return match;
|
|
990
|
+
|
|
991
|
+
const { filter, predicate } = loaded;
|
|
992
|
+
const result: FilterBackApplyResult = {
|
|
993
|
+
...nothing,
|
|
994
|
+
matched: match.messageIds.length,
|
|
995
|
+
semanticUnavailable: match.semanticUnavailable,
|
|
996
|
+
};
|
|
997
|
+
for (const messageId of match.messageIds) {
|
|
998
|
+
const message = await filterMessageOf(client, accountConfigId, messageId);
|
|
999
|
+
if (!message) continue;
|
|
1000
|
+
const decision = await pipeline.evaluate(
|
|
1001
|
+
accountConfigId,
|
|
1002
|
+
messageId,
|
|
1003
|
+
message,
|
|
1004
|
+
);
|
|
1005
|
+
const move = (decision.move?.filterId ?? filterId) === filterId;
|
|
1006
|
+
if (!move && filter.actionLabelId === NO_ACTION) continue;
|
|
1007
|
+
const outcome = await applyOrganize(
|
|
1008
|
+
{ client, moveService },
|
|
1009
|
+
accountConfigId,
|
|
1010
|
+
[messageId],
|
|
1011
|
+
{
|
|
1012
|
+
...predicate,
|
|
1013
|
+
actionMailboxId: move ? filter.actionMailboxId : NO_ACTION,
|
|
1014
|
+
},
|
|
1015
|
+
);
|
|
1016
|
+
result.applied += outcome.applied;
|
|
1017
|
+
result.failed += outcome.failed;
|
|
1018
|
+
}
|
|
1019
|
+
return result;
|
|
1020
|
+
};
|
|
1021
|
+
|
|
1022
|
+
/** Whether a job row runs a standing filter rather than its own snapshot. */
|
|
1023
|
+
export const isFilterJob = (
|
|
1024
|
+
job: Pick<OrganizeJobRequestItem, "filterId">,
|
|
1025
|
+
): boolean => job.filterId !== NO_ACTION;
|