@remit/backend 0.0.137 → 0.0.139
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/handlers/calendar-event.ts +5 -1
- package/src/handlers/calendar-suggestion.test.ts +397 -1
- package/src/handlers/calendar-suggestion.ts +67 -13
- package/src/handlers/filter.test.ts +127 -1
- package/src/handlers/filter.ts +26 -3
- package/src/handlers/mailbox-settled-target.test.ts +13 -1
- package/src/handlers/organize.test.ts +90 -1
- package/src/handlers/organize.ts +34 -47
- package/src/service/organize-queue.ts +98 -0
- package/src/service/organize.test.ts +7 -0
- package/src/service/organize.ts +300 -31
- package/src/types.ts +2 -1
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import { describe, it, mock } from "node:test";
|
|
2
|
+
import { afterEach, describe, it, mock } from "node:test";
|
|
3
|
+
import type { SendMessageCommand } from "@aws-sdk/client-sqs";
|
|
3
4
|
import type {
|
|
4
5
|
CreateFilterInput,
|
|
5
6
|
UpdateFilterInput as UpdateFilterRequestBody,
|
|
6
7
|
} from "@remit/api-openapi-types";
|
|
7
8
|
import type {
|
|
8
9
|
CreateFilterAnchorInput,
|
|
10
|
+
CreateOrganizeJobRequestInput,
|
|
9
11
|
CreateFilterInput as FilterAnchorTxCreateInput,
|
|
10
12
|
FilterItem,
|
|
11
13
|
} from "@remit/data-ports";
|
|
@@ -16,9 +18,19 @@ import {
|
|
|
16
18
|
FilterState,
|
|
17
19
|
} from "@remit/domain-enums";
|
|
18
20
|
import type { AnchorPayload } from "@remit/search-service";
|
|
21
|
+
import type { APIGatewayProxyEvent } from "aws-lambda";
|
|
22
|
+
import type { Context } from "openapi-backend";
|
|
23
|
+
import { deriveAccountConfigId } from "../auth.js";
|
|
24
|
+
import {
|
|
25
|
+
_resetForTest,
|
|
26
|
+
type RemitClient,
|
|
27
|
+
setClient,
|
|
28
|
+
} from "../service/data-client.js";
|
|
29
|
+
import { sqsClient } from "../service/sqs.js";
|
|
19
30
|
import {
|
|
20
31
|
createFilterWithAnchor,
|
|
21
32
|
type FilterCrudDeps,
|
|
33
|
+
FilterOperations,
|
|
22
34
|
pickFilterUpdate,
|
|
23
35
|
rejectAnchorMutation,
|
|
24
36
|
resolveFilterScopeExpiry,
|
|
@@ -516,3 +528,117 @@ describe("createFilterWithAnchor (#351)", () => {
|
|
|
516
528
|
);
|
|
517
529
|
});
|
|
518
530
|
});
|
|
531
|
+
|
|
532
|
+
describe("createFilter queues the new filter's pass over the inbox (#1354)", () => {
|
|
533
|
+
const SUB = "cognito-sub-1354";
|
|
534
|
+
const ACCOUNT_CONFIG_ID = deriveAccountConfigId(SUB);
|
|
535
|
+
const ACCOUNT_ID = "acc-1354";
|
|
536
|
+
|
|
537
|
+
const created: FilterItem = {
|
|
538
|
+
filterId: "flt-new",
|
|
539
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
540
|
+
name: "Invoices",
|
|
541
|
+
scope: FilterScope.Standing,
|
|
542
|
+
state: FilterState.Active,
|
|
543
|
+
disabledReason: "None",
|
|
544
|
+
hasAnchor: false,
|
|
545
|
+
ruleChangedAt: 1,
|
|
546
|
+
actionChangedAt: 1,
|
|
547
|
+
matchOperator: FilterMatchOperator.And,
|
|
548
|
+
literalClauses: [{ field: "From", value: "billing@example.com" }],
|
|
549
|
+
actionLabelId: "None",
|
|
550
|
+
actionMailboxId: "mbx-invoices",
|
|
551
|
+
createdAt: 0,
|
|
552
|
+
updatedAt: 0,
|
|
553
|
+
};
|
|
554
|
+
|
|
555
|
+
afterEach(() => {
|
|
556
|
+
mock.restoreAll();
|
|
557
|
+
_resetForTest();
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
const world = (filter: FilterItem) => {
|
|
561
|
+
process.env.SQS_QUEUE_URL_ACCOUNT_FANOUT =
|
|
562
|
+
"http://localhost:9324/queue/account-fanout-test";
|
|
563
|
+
const jobs: CreateOrganizeJobRequestInput[] = [];
|
|
564
|
+
const enqueued: SendMessageCommand[] = [];
|
|
565
|
+
mock.method(sqsClient, "send", async (command: SendMessageCommand) => {
|
|
566
|
+
enqueued.push(command);
|
|
567
|
+
return {};
|
|
568
|
+
});
|
|
569
|
+
setClient({
|
|
570
|
+
account: {
|
|
571
|
+
get: async () => ({
|
|
572
|
+
accountId: ACCOUNT_ID,
|
|
573
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
574
|
+
}),
|
|
575
|
+
},
|
|
576
|
+
mailbox: {
|
|
577
|
+
get: async () => ({
|
|
578
|
+
mailboxId: "mbx-invoices",
|
|
579
|
+
accountId: ACCOUNT_ID,
|
|
580
|
+
fullPath: "Invoices",
|
|
581
|
+
syncStatus: "synced",
|
|
582
|
+
}),
|
|
583
|
+
},
|
|
584
|
+
filterAnchorTransaction: { createWithAnchor: async () => filter },
|
|
585
|
+
organizeJobRequest: {
|
|
586
|
+
create: async (input: CreateOrganizeJobRequestInput) => {
|
|
587
|
+
jobs.push(input);
|
|
588
|
+
return { ...input, organizeJobId: "job-new", state: "Pending" };
|
|
589
|
+
},
|
|
590
|
+
},
|
|
591
|
+
} as unknown as RemitClient);
|
|
592
|
+
return { jobs, enqueued };
|
|
593
|
+
};
|
|
594
|
+
|
|
595
|
+
const createFilter = (body: Partial<CreateFilterInput>) =>
|
|
596
|
+
(
|
|
597
|
+
FilterOperations.FilterOperations_createFilter as (
|
|
598
|
+
context: Context,
|
|
599
|
+
event: APIGatewayProxyEvent,
|
|
600
|
+
) => Promise<Record<string, unknown>>
|
|
601
|
+
)(
|
|
602
|
+
{
|
|
603
|
+
request: {
|
|
604
|
+
params: { accountId: ACCOUNT_ID },
|
|
605
|
+
requestBody: {
|
|
606
|
+
name: created.name,
|
|
607
|
+
scope: created.scope,
|
|
608
|
+
matchOperator: created.matchOperator,
|
|
609
|
+
literalClauses: created.literalClauses,
|
|
610
|
+
actionLabelId: created.actionLabelId,
|
|
611
|
+
actionMailboxId: created.actionMailboxId,
|
|
612
|
+
...body,
|
|
613
|
+
},
|
|
614
|
+
},
|
|
615
|
+
} as unknown as Context,
|
|
616
|
+
{
|
|
617
|
+
requestContext: { authorizer: { claims: { sub: SUB } } },
|
|
618
|
+
} as unknown as APIGatewayProxyEvent,
|
|
619
|
+
);
|
|
620
|
+
|
|
621
|
+
it("queues an organize job for the created filter and returns its id", async () => {
|
|
622
|
+
const { jobs, enqueued } = world(created);
|
|
623
|
+
|
|
624
|
+
const response = await createFilter({});
|
|
625
|
+
|
|
626
|
+
assert.equal(jobs.length, 1);
|
|
627
|
+
assert.equal(jobs[0]?.filterId, "flt-new");
|
|
628
|
+
assert.equal(enqueued.length, 1);
|
|
629
|
+
assert.match(String(enqueued[0]?.input.MessageBody), /job-new/);
|
|
630
|
+
assert.equal(response.organizeJobId, "job-new");
|
|
631
|
+
assert.equal(response.filterId, "flt-new");
|
|
632
|
+
});
|
|
633
|
+
|
|
634
|
+
it("names only the filter: the predicate columns stay at their sentinels", async () => {
|
|
635
|
+
const { jobs } = world(created);
|
|
636
|
+
|
|
637
|
+
await createFilter({});
|
|
638
|
+
|
|
639
|
+
assert.equal(jobs[0]?.anchorMessageId, "None");
|
|
640
|
+
assert.deepEqual(jobs[0]?.literalClauses, []);
|
|
641
|
+
assert.equal(jobs[0]?.actionLabelId, "None");
|
|
642
|
+
assert.equal(jobs[0]?.actionMailboxId, "None");
|
|
643
|
+
});
|
|
644
|
+
});
|
package/src/handlers/filter.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type {
|
|
2
2
|
CreateFilterInput,
|
|
3
|
+
CreateFilterResponse,
|
|
3
4
|
FilterResponse,
|
|
4
5
|
UpdateFilterInput as UpdateFilterRequestBody,
|
|
5
6
|
} from "@remit/api-openapi-types";
|
|
@@ -19,9 +20,10 @@ import {
|
|
|
19
20
|
} from "@remit/domain-enums";
|
|
20
21
|
import type { AnchorPayload } from "@remit/search-service";
|
|
21
22
|
import type { APIGatewayProxyEvent } from "aws-lambda";
|
|
22
|
-
import { getAccountConfigIdFromEvent } from "../auth.js";
|
|
23
|
+
import { getAccountConfigIdFromEvent, getSubFromEvent } from "../auth.js";
|
|
23
24
|
import { getClient } from "../service/data-client.js";
|
|
24
25
|
import { buildFilterAnchor } from "../service/filter.js";
|
|
26
|
+
import { queueOrganizeJob } from "../service/organize-queue.js";
|
|
25
27
|
import type {
|
|
26
28
|
FilterDetailOperationIds,
|
|
27
29
|
FilterOperationIds,
|
|
@@ -340,6 +342,16 @@ export const createFilterWithAnchor = async (
|
|
|
340
342
|
);
|
|
341
343
|
};
|
|
342
344
|
|
|
345
|
+
const requireUserId = (event: APIGatewayProxyEvent): string => {
|
|
346
|
+
const userId = getSubFromEvent(event);
|
|
347
|
+
if (!userId) {
|
|
348
|
+
throw new Error(
|
|
349
|
+
"Missing Cognito `sub`: cannot attribute an organize job to a user",
|
|
350
|
+
);
|
|
351
|
+
}
|
|
352
|
+
return userId;
|
|
353
|
+
};
|
|
354
|
+
|
|
343
355
|
/**
|
|
344
356
|
* A filter's `actionMailboxId` is a durable reference, so its target has to be
|
|
345
357
|
* a folder the mail server has settled (D12, first row) — 422 otherwise. The
|
|
@@ -393,11 +405,15 @@ export const FilterOperations: Record<
|
|
|
393
405
|
};
|
|
394
406
|
},
|
|
395
407
|
|
|
396
|
-
FilterOperations_createFilter: async (
|
|
408
|
+
FilterOperations_createFilter: async (
|
|
409
|
+
context,
|
|
410
|
+
...args: unknown[]
|
|
411
|
+
): Promise<CreateFilterResponse> => {
|
|
397
412
|
const event = args[0] as APIGatewayProxyEvent;
|
|
398
413
|
const accountConfigId = getAccountConfigIdFromEvent(event);
|
|
399
414
|
const { accountId } = context.request.params as { accountId: string };
|
|
400
415
|
const input = context.request.requestBody as CreateFilterInput;
|
|
416
|
+
const userId = requireUserId(event);
|
|
401
417
|
|
|
402
418
|
const client = await getClient();
|
|
403
419
|
const account = await client.account.get(accountId);
|
|
@@ -412,7 +428,14 @@ export const FilterOperations: Record<
|
|
|
412
428
|
accountConfigId,
|
|
413
429
|
input,
|
|
414
430
|
);
|
|
415
|
-
|
|
431
|
+
// The new filter's first pass over the inbox. The worker reads the filter
|
|
432
|
+
// when it runs and refuses one that has nothing to run (#1354).
|
|
433
|
+
const job = await queueOrganizeJob(client, {
|
|
434
|
+
accountConfigId,
|
|
435
|
+
userId,
|
|
436
|
+
filterId: filter.filterId,
|
|
437
|
+
});
|
|
438
|
+
return { ...toFilterResponse(filter), organizeJobId: job.organizeJobId };
|
|
416
439
|
},
|
|
417
440
|
};
|
|
418
441
|
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* only removed by the user, so a `mailboxId` that names nothing never exists.
|
|
10
10
|
*/
|
|
11
11
|
import assert from "node:assert/strict";
|
|
12
|
-
import { afterEach, describe, it } from "node:test";
|
|
12
|
+
import { afterEach, describe, it, mock } from "node:test";
|
|
13
13
|
import type { MailboxItem } from "@remit/data-ports";
|
|
14
14
|
import { CanonicalMailboxRole, MailboxSyncStatus } from "@remit/domain-enums";
|
|
15
15
|
import type { APIGatewayProxyEvent } from "aws-lambda";
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
type RemitClient,
|
|
21
21
|
setClient,
|
|
22
22
|
} from "../service/data-client.js";
|
|
23
|
+
import { sqsClient } from "../service/sqs.js";
|
|
23
24
|
import { FilterDetailOperations, FilterOperations } from "./filter.js";
|
|
24
25
|
import { FolderRoleOperations } from "./folder-role.js";
|
|
25
26
|
import { MessageBulkOperations } from "./message.js";
|
|
@@ -55,6 +56,9 @@ const world = (syncStatus: MailboxItem["syncStatus"]): Writes => {
|
|
|
55
56
|
copies: [],
|
|
56
57
|
};
|
|
57
58
|
const row = target(syncStatus);
|
|
59
|
+
process.env.SQS_QUEUE_URL_ACCOUNT_FANOUT =
|
|
60
|
+
"http://localhost:9324/queue/account-fanout-test";
|
|
61
|
+
mock.method(sqsClient, "send", async () => ({}));
|
|
58
62
|
|
|
59
63
|
setClient({
|
|
60
64
|
account: {
|
|
@@ -85,6 +89,13 @@ const world = (syncStatus: MailboxItem["syncStatus"]): Writes => {
|
|
|
85
89
|
return { filterId: "flt-1" };
|
|
86
90
|
},
|
|
87
91
|
},
|
|
92
|
+
organizeJobRequest: {
|
|
93
|
+
create: async (input: Record<string, unknown>) => ({
|
|
94
|
+
...input,
|
|
95
|
+
organizeJobId: "job-1",
|
|
96
|
+
state: "Pending",
|
|
97
|
+
}),
|
|
98
|
+
},
|
|
88
99
|
accountSetting: {
|
|
89
100
|
get: async () => null,
|
|
90
101
|
upsert: async (item: unknown) => {
|
|
@@ -168,6 +179,7 @@ const SITES = [
|
|
|
168
179
|
|
|
169
180
|
describe("a write that binds to a folder requires a settled folder", () => {
|
|
170
181
|
afterEach(() => {
|
|
182
|
+
mock.restoreAll();
|
|
171
183
|
_resetForTest();
|
|
172
184
|
});
|
|
173
185
|
|
|
@@ -4,11 +4,12 @@ import type { SendMessageCommand } from "@aws-sdk/client-sqs";
|
|
|
4
4
|
import type { OrganizeInput } from "@remit/api-openapi-types";
|
|
5
5
|
import type {
|
|
6
6
|
CreateOrganizeJobRequestInput,
|
|
7
|
+
FilterItem,
|
|
7
8
|
IAccountRepository,
|
|
8
9
|
IOrganizeJobRequestRepository,
|
|
9
10
|
OrganizeJobRequestItem,
|
|
10
11
|
} from "@remit/data-ports";
|
|
11
|
-
import { BadRequestError } from "@remit/data-ports/errors";
|
|
12
|
+
import { BadRequestError, NotFoundError } from "@remit/data-ports/errors";
|
|
12
13
|
import { FilterMatchOperator } from "@remit/domain-enums";
|
|
13
14
|
import type { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda";
|
|
14
15
|
import type { Context } from "openapi-backend";
|
|
@@ -186,3 +187,91 @@ describe("createOrganizeJob rejected-rule refusal (reader #995)", () => {
|
|
|
186
187
|
);
|
|
187
188
|
});
|
|
188
189
|
});
|
|
190
|
+
|
|
191
|
+
describe("createOrganizeJob by filterId — Run now (#1354)", () => {
|
|
192
|
+
afterEach(() => {
|
|
193
|
+
mock.restoreAll();
|
|
194
|
+
_resetForTest();
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
const standing = (state: FilterItem["state"]): FilterItem => ({
|
|
198
|
+
filterId: "flt-1",
|
|
199
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
200
|
+
name: "Invoices",
|
|
201
|
+
scope: "Standing",
|
|
202
|
+
state,
|
|
203
|
+
disabledReason: state === "Active" ? "None" : "UserDisabled",
|
|
204
|
+
hasAnchor: false,
|
|
205
|
+
ruleChangedAt: 1,
|
|
206
|
+
actionChangedAt: 1,
|
|
207
|
+
matchOperator: FilterMatchOperator.And,
|
|
208
|
+
literalClauses: [{ field: "From", value: "billing@example.com" }],
|
|
209
|
+
actionLabelId: "None",
|
|
210
|
+
actionMailboxId: "mbx-invoices",
|
|
211
|
+
createdAt: 0,
|
|
212
|
+
updatedAt: 0,
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
const withFilter = (filter: FilterItem): void => {
|
|
216
|
+
process.env.SQS_QUEUE_URL_ACCOUNT_FANOUT =
|
|
217
|
+
"http://localhost:9324/queue/account-fanout-test";
|
|
218
|
+
enqueued.length = 0;
|
|
219
|
+
createdRows.length = 0;
|
|
220
|
+
mock.method(sqsClient, "send", async (command: SendMessageCommand) => {
|
|
221
|
+
enqueued.push(command);
|
|
222
|
+
return {};
|
|
223
|
+
});
|
|
224
|
+
setClient({
|
|
225
|
+
account: {
|
|
226
|
+
get: async () => ({
|
|
227
|
+
accountId: ACCOUNT_ID,
|
|
228
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
229
|
+
}),
|
|
230
|
+
},
|
|
231
|
+
filter: {
|
|
232
|
+
get: async () => filter,
|
|
233
|
+
},
|
|
234
|
+
organizeJobRequest: {
|
|
235
|
+
create: async (row: CreateOrganizeJobRequestInput) => {
|
|
236
|
+
createdRows.push(row);
|
|
237
|
+
return { ...row, organizeJobId: "job-run", state: "Pending" };
|
|
238
|
+
},
|
|
239
|
+
},
|
|
240
|
+
} as unknown as RemitClient);
|
|
241
|
+
};
|
|
242
|
+
|
|
243
|
+
it("queues a job that names only the filter", async () => {
|
|
244
|
+
withFilter(standing("Active"));
|
|
245
|
+
|
|
246
|
+
const response = await postOrganize(input({ filterId: "flt-1" }));
|
|
247
|
+
|
|
248
|
+
assert.equal(response.statusCode, 202);
|
|
249
|
+
assert.equal(createdRows[0]?.filterId, "flt-1");
|
|
250
|
+
assert.equal(createdRows[0]?.actionMailboxId, "None");
|
|
251
|
+
assert.deepEqual(createdRows[0]?.literalClauses, []);
|
|
252
|
+
assert.equal(enqueued.length, 1);
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
it("answers an unknown filter with a 404 and queues nothing", async () => {
|
|
256
|
+
withFilter(standing("Active"));
|
|
257
|
+
setClient({
|
|
258
|
+
account: {
|
|
259
|
+
get: async () => ({
|
|
260
|
+
accountId: ACCOUNT_ID,
|
|
261
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
262
|
+
}),
|
|
263
|
+
},
|
|
264
|
+
filter: {
|
|
265
|
+
get: async () => {
|
|
266
|
+
throw new NotFoundError("Filter not found: flt-gone");
|
|
267
|
+
},
|
|
268
|
+
},
|
|
269
|
+
} as unknown as RemitClient);
|
|
270
|
+
|
|
271
|
+
const response = await postOrganize(input({ filterId: "flt-gone" }));
|
|
272
|
+
|
|
273
|
+
assert.equal(response.statusCode, 404);
|
|
274
|
+
assert.deepEqual(createdRows, []);
|
|
275
|
+
assert.deepEqual(enqueued, []);
|
|
276
|
+
});
|
|
277
|
+
});
|
package/src/handlers/organize.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { SendMessageCommand } from "@aws-sdk/client-sqs";
|
|
2
1
|
import type {
|
|
3
2
|
CreateOrganizeJobResponse,
|
|
4
3
|
OrganizeInput,
|
|
@@ -6,9 +5,7 @@ import type {
|
|
|
6
5
|
OrganizePreviewResponse,
|
|
7
6
|
} from "@remit/api-openapi-types";
|
|
8
7
|
import { BadRequestError, NotFoundError } from "@remit/data-ports/errors";
|
|
9
|
-
import { logger } from "@remit/logger-lambda";
|
|
10
8
|
import type { APIGatewayProxyEvent } from "aws-lambda";
|
|
11
|
-
import { env } from "expect-env";
|
|
12
9
|
import type { Context } from "openapi-backend";
|
|
13
10
|
import { getAccountConfigIdFromEvent, getSubFromEvent } from "../auth.js";
|
|
14
11
|
import { getClient, type RemitClient } from "../service/data-client.js";
|
|
@@ -19,7 +16,10 @@ import {
|
|
|
19
16
|
type OrganizePredicate,
|
|
20
17
|
organizePredicateRejection,
|
|
21
18
|
} from "../service/organize.js";
|
|
22
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
type OrganizeJobRequest,
|
|
21
|
+
queueOrganizeJob,
|
|
22
|
+
} from "../service/organize-queue.js";
|
|
23
23
|
import type {
|
|
24
24
|
OperationHandler,
|
|
25
25
|
OrganizeJobDetailOperationIds,
|
|
@@ -29,13 +29,6 @@ import { assertAccountOwnership } from "./account-ownership.js";
|
|
|
29
29
|
|
|
30
30
|
const NONE = "None";
|
|
31
31
|
|
|
32
|
-
/**
|
|
33
|
-
* How long a finished (or abandoned) job row lingers before the table-wide TTL
|
|
34
|
-
* reclaims it (RFC 034 Decision 1). A back-apply runs once and needs no standing
|
|
35
|
-
* lifetime; a week is ample for a client to poll the result.
|
|
36
|
-
*/
|
|
37
|
-
const ORGANIZE_JOB_TTL_SECONDS = 7 * 24 * 60 * 60;
|
|
38
|
-
|
|
39
32
|
/**
|
|
40
33
|
* Normalize the request body into the flattened predicate the job row and the
|
|
41
34
|
* matcher share. `anchorMessageId` collapses to the `"None"` sentinel when
|
|
@@ -70,6 +63,7 @@ const toOrganizeJobResponse = (
|
|
|
70
63
|
accountConfigId: job.accountConfigId,
|
|
71
64
|
userId: job.userId,
|
|
72
65
|
state: job.state,
|
|
66
|
+
filterId: job.filterId,
|
|
73
67
|
anchorMessageId: job.anchorMessageId,
|
|
74
68
|
matchOperator: job.matchOperator,
|
|
75
69
|
literalClauses: job.literalClauses,
|
|
@@ -84,6 +78,27 @@ const toOrganizeJobResponse = (
|
|
|
84
78
|
updatedAt: job.updatedAt,
|
|
85
79
|
});
|
|
86
80
|
|
|
81
|
+
/**
|
|
82
|
+
* What a create asks the worker to run. A saved filter is named and read when
|
|
83
|
+
* the job runs, so Run now and a filter's create share one path (#1354); an
|
|
84
|
+
* "all like these" predicate the matcher can never honour is answered here,
|
|
85
|
+
* not with a 202 the worker has to fail later (reader #463).
|
|
86
|
+
*/
|
|
87
|
+
const requestedJob = async (
|
|
88
|
+
client: RemitClient,
|
|
89
|
+
owner: { accountConfigId: string; userId: string },
|
|
90
|
+
input: OrganizeInput,
|
|
91
|
+
): Promise<OrganizeJobRequest> => {
|
|
92
|
+
if (input.filterId) {
|
|
93
|
+
await client.filter.get(owner.accountConfigId, input.filterId);
|
|
94
|
+
return { ...owner, filterId: input.filterId };
|
|
95
|
+
}
|
|
96
|
+
const predicate = predicateFromInput(input);
|
|
97
|
+
const rejection = organizePredicateRejection(predicate);
|
|
98
|
+
if (rejection) throw new BadRequestError(rejection.message);
|
|
99
|
+
return { ...owner, predicate };
|
|
100
|
+
};
|
|
101
|
+
|
|
87
102
|
export const OrganizeOperations: Record<
|
|
88
103
|
OrganizeOperationIds,
|
|
89
104
|
OperationHandler<OrganizeOperationIds>
|
|
@@ -107,42 +122,9 @@ export const OrganizeOperations: Record<
|
|
|
107
122
|
const client = await getClient();
|
|
108
123
|
await assertAccount(client, accountId, accountConfigId, "act");
|
|
109
124
|
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
// the worker has to fail later (reader #463).
|
|
114
|
-
const rejection = organizePredicateRejection(predicate);
|
|
115
|
-
if (rejection) throw new BadRequestError(rejection.message);
|
|
116
|
-
|
|
117
|
-
const ttl = Math.floor(Date.now() / 1000) + ORGANIZE_JOB_TTL_SECONDS;
|
|
118
|
-
|
|
119
|
-
const job = await client.organizeJobRequest.create({
|
|
120
|
-
accountConfigId,
|
|
121
|
-
userId,
|
|
122
|
-
anchorMessageId: predicate.anchorMessageId,
|
|
123
|
-
matchOperator: predicate.matchOperator,
|
|
124
|
-
literalClauses: predicate.literalClauses,
|
|
125
|
-
similarityThreshold: predicate.similarityThreshold,
|
|
126
|
-
actionLabelId: predicate.actionLabelId,
|
|
127
|
-
actionMailboxId: predicate.actionMailboxId,
|
|
128
|
-
ttl,
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
await sqsClient.send(
|
|
132
|
-
new SendMessageCommand({
|
|
133
|
-
QueueUrl: env.SQS_QUEUE_URL_ACCOUNT_FANOUT,
|
|
134
|
-
MessageBody: JSON.stringify({
|
|
135
|
-
type: "OrganizeJob",
|
|
136
|
-
accountConfigId,
|
|
137
|
-
organizeJobId: job.organizeJobId,
|
|
138
|
-
}),
|
|
139
|
-
}),
|
|
140
|
-
);
|
|
141
|
-
|
|
142
|
-
// biome-ignore lint/plugin/no-logger-info: a back-apply is an audit-grade signal
|
|
143
|
-
logger.info(
|
|
144
|
-
{ accountConfigId, organizeJobId: job.organizeJobId },
|
|
145
|
-
"Organize back-apply job initiated",
|
|
125
|
+
const job = await queueOrganizeJob(
|
|
126
|
+
client,
|
|
127
|
+
await requestedJob(client, { accountConfigId, userId }, input),
|
|
146
128
|
);
|
|
147
129
|
|
|
148
130
|
return {
|
|
@@ -164,6 +146,11 @@ export const OrganizeOperations: Record<
|
|
|
164
146
|
const client = await getClient();
|
|
165
147
|
await assertAccount(client, accountId, accountConfigId, "read");
|
|
166
148
|
|
|
149
|
+
if (input.filterId) {
|
|
150
|
+
throw new BadRequestError(
|
|
151
|
+
"Preview takes a rule, not a saved filter. Run the filter to apply it.",
|
|
152
|
+
);
|
|
153
|
+
}
|
|
167
154
|
const result = await matchOrganize(
|
|
168
155
|
buildOrganizeMatchDeps(client),
|
|
169
156
|
accountConfigId,
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { SendMessageCommand } from "@aws-sdk/client-sqs";
|
|
2
|
+
import type {
|
|
3
|
+
CreateOrganizeJobRequestInput,
|
|
4
|
+
OrganizeJobRequestItem,
|
|
5
|
+
} from "@remit/data-ports";
|
|
6
|
+
import { FilterMatchOperator } from "@remit/domain-enums";
|
|
7
|
+
import { logger } from "@remit/logger-lambda";
|
|
8
|
+
import {
|
|
9
|
+
DEFAULT_SEMANTIC_MATCH_THRESHOLD,
|
|
10
|
+
NO_ACTION,
|
|
11
|
+
} from "@remit/mailbox-service";
|
|
12
|
+
import { env } from "expect-env";
|
|
13
|
+
import type { RemitClient } from "./data-client.js";
|
|
14
|
+
import type { OrganizePredicate } from "./organize.js";
|
|
15
|
+
import { sqsClient } from "./sqs.js";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* How long a finished (or abandoned) job row lingers before the table-wide TTL
|
|
19
|
+
* reclaims it (RFC 034 Decision 1). A back-apply runs once and needs no standing
|
|
20
|
+
* lifetime; a week is ample for a client to poll the result.
|
|
21
|
+
*/
|
|
22
|
+
const ORGANIZE_JOB_TTL_SECONDS = 7 * 24 * 60 * 60;
|
|
23
|
+
|
|
24
|
+
interface JobOwner {
|
|
25
|
+
accountConfigId: string;
|
|
26
|
+
userId: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* An "all like these" pass carries its predicate; a filter job names only the
|
|
31
|
+
* filter, which the worker reads when it runs.
|
|
32
|
+
*/
|
|
33
|
+
export type OrganizeJobRequest = JobOwner &
|
|
34
|
+
({ predicate: OrganizePredicate } | { filterId: string });
|
|
35
|
+
|
|
36
|
+
/** A filter job's row: the filter id, every predicate column at its sentinel. */
|
|
37
|
+
const filterJobColumns = (
|
|
38
|
+
filterId: string,
|
|
39
|
+
): Omit<CreateOrganizeJobRequestInput, keyof JobOwner | "ttl"> => ({
|
|
40
|
+
filterId,
|
|
41
|
+
anchorMessageId: NO_ACTION,
|
|
42
|
+
matchOperator: FilterMatchOperator.And,
|
|
43
|
+
literalClauses: [],
|
|
44
|
+
similarityThreshold: DEFAULT_SEMANTIC_MATCH_THRESHOLD,
|
|
45
|
+
actionLabelId: NO_ACTION,
|
|
46
|
+
actionMailboxId: NO_ACTION,
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
const predicateColumns = (
|
|
50
|
+
predicate: OrganizePredicate,
|
|
51
|
+
): Omit<CreateOrganizeJobRequestInput, keyof JobOwner | "ttl"> => ({
|
|
52
|
+
filterId: NO_ACTION,
|
|
53
|
+
...predicate,
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Write a Pending job row and hand it to the account-fanout worker. Every
|
|
58
|
+
* back-apply goes through here: "all like these", a new filter's first pass,
|
|
59
|
+
* and Run now on a saved filter.
|
|
60
|
+
*/
|
|
61
|
+
export const queueOrganizeJob = async (
|
|
62
|
+
client: Pick<RemitClient, "organizeJobRequest">,
|
|
63
|
+
request: OrganizeJobRequest,
|
|
64
|
+
): Promise<OrganizeJobRequestItem> => {
|
|
65
|
+
const { accountConfigId, userId } = request;
|
|
66
|
+
const columns =
|
|
67
|
+
"filterId" in request
|
|
68
|
+
? filterJobColumns(request.filterId)
|
|
69
|
+
: predicateColumns(request.predicate);
|
|
70
|
+
const job = await client.organizeJobRequest.create({
|
|
71
|
+
accountConfigId,
|
|
72
|
+
userId,
|
|
73
|
+
...columns,
|
|
74
|
+
ttl: Math.floor(Date.now() / 1000) + ORGANIZE_JOB_TTL_SECONDS,
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
await sqsClient.send(
|
|
78
|
+
new SendMessageCommand({
|
|
79
|
+
QueueUrl: env.SQS_QUEUE_URL_ACCOUNT_FANOUT,
|
|
80
|
+
MessageBody: JSON.stringify({
|
|
81
|
+
type: "OrganizeJob",
|
|
82
|
+
accountConfigId,
|
|
83
|
+
organizeJobId: job.organizeJobId,
|
|
84
|
+
}),
|
|
85
|
+
}),
|
|
86
|
+
);
|
|
87
|
+
|
|
88
|
+
// biome-ignore lint/plugin/no-logger-info: a back-apply is an audit-grade signal
|
|
89
|
+
logger.info(
|
|
90
|
+
{
|
|
91
|
+
accountConfigId,
|
|
92
|
+
organizeJobId: job.organizeJobId,
|
|
93
|
+
filterId: job.filterId,
|
|
94
|
+
},
|
|
95
|
+
"Organize back-apply job initiated",
|
|
96
|
+
);
|
|
97
|
+
return job;
|
|
98
|
+
};
|
|
@@ -289,6 +289,7 @@ const matchDeps = (
|
|
|
289
289
|
},
|
|
290
290
|
listAccountFilterMessages: async () => ({ items: corpus }),
|
|
291
291
|
filterAnchors: {
|
|
292
|
+
get: async () => null,
|
|
292
293
|
listByAccountConfig: async () => filterAnchorRows,
|
|
293
294
|
put: async () => {
|
|
294
295
|
throw new Error("matchDeps must not repair an anchor");
|
|
@@ -330,6 +331,7 @@ const vectorlessDeps = (
|
|
|
330
331
|
},
|
|
331
332
|
listAccountFilterMessages: async () => ({ items: corpus }),
|
|
332
333
|
filterAnchors: {
|
|
334
|
+
get: async () => null,
|
|
333
335
|
listByAccountConfig: async () => [],
|
|
334
336
|
put: async () => {
|
|
335
337
|
throw moduleNotFound();
|
|
@@ -471,6 +473,7 @@ describe("matchOrganize honors the persisted FilterAnchor (reader #350)", () =>
|
|
|
471
473
|
}),
|
|
472
474
|
listAccountFilterMessages: async () => ({ items: [] }),
|
|
473
475
|
filterAnchors: {
|
|
476
|
+
get: async () => null,
|
|
474
477
|
listByAccountConfig: async () => [persistedAnchor],
|
|
475
478
|
put: async () => {
|
|
476
479
|
throw new Error("a current anchor must not be rewritten");
|
|
@@ -757,6 +760,7 @@ describe("matchOrganize on a deployment without the vector pipeline", () => {
|
|
|
757
760
|
}),
|
|
758
761
|
listAccountFilterMessages: async () => ({ items: [] }),
|
|
759
762
|
filterAnchors: {
|
|
763
|
+
get: async () => null,
|
|
760
764
|
listByAccountConfig: async () => [drifted],
|
|
761
765
|
put: async () => {
|
|
762
766
|
throw new Error("an unrepairable anchor must not be rewritten");
|
|
@@ -798,6 +802,7 @@ describe("matchOrganize on a deployment without the vector pipeline", () => {
|
|
|
798
802
|
}),
|
|
799
803
|
listAccountFilterMessages: async () => ({ items: [] }),
|
|
800
804
|
filterAnchors: {
|
|
805
|
+
get: async () => null,
|
|
801
806
|
listByAccountConfig: async () => [],
|
|
802
807
|
put: async () => {
|
|
803
808
|
throw new Error("unreachable");
|
|
@@ -1120,6 +1125,7 @@ describe("back-apply repairs a drifted anchor the way index-time matching does (
|
|
|
1120
1125
|
}),
|
|
1121
1126
|
listAccountFilterMessages: async () => ({ items: [] }),
|
|
1122
1127
|
filterAnchors: {
|
|
1128
|
+
get: async () => null,
|
|
1123
1129
|
listByAccountConfig: async () => anchors,
|
|
1124
1130
|
put: async (input: CreateFilterAnchorInput) => {
|
|
1125
1131
|
anchorPuts.push(input);
|
|
@@ -1319,6 +1325,7 @@ const storeBackedDeps = (
|
|
|
1319
1325
|
};
|
|
1320
1326
|
},
|
|
1321
1327
|
filterAnchors: {
|
|
1328
|
+
get: async () => null,
|
|
1322
1329
|
listByAccountConfig: async () => [],
|
|
1323
1330
|
put: async () => {
|
|
1324
1331
|
throw new Error("unreachable");
|