@remit/account-worker 0.0.25 → 0.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/cascade-enumerate.test.ts +3 -53
- package/src/cascade.ts +2 -55
- package/src/compose-relational.test.ts +1 -14
- package/src/compose-relational.ts +5 -6
- package/src/config.test.ts +0 -5
- package/src/config.ts +1 -6
- package/src/deletion-capabilities.ts +2 -2
- package/src/events.ts +1 -1
- package/src/handlers/account-export.test.ts +1 -1
- package/src/handlers/account-export.ts +1 -1
- package/src/handlers/account-fanout.test.ts +1 -2
- package/src/handlers/account-fanout.ts +7 -6
- package/src/handlers/account-finalize.ts +6 -8
- package/src/handlers/account-purge.test.ts +4 -50
- package/src/handlers/account-purge.ts +7 -99
- package/src/handlers/organize-job.test.ts +314 -16
- package/src/handlers/organize-job.ts +78 -26
- package/src/handlers/sender-category-backapply.test.ts +1 -1
- package/src/handlers/sender-category-backapply.ts +1 -1
- package/src/poller.ts +5 -1
- package/src/cascade.test.ts +0 -77
|
@@ -1,19 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
SendMessageBatchCommand,
|
|
4
|
-
SendMessageCommand,
|
|
5
|
-
type SQSClient,
|
|
6
|
-
} from "@aws-sdk/client-sqs";
|
|
7
|
-
import { Message } from "@remit/electrodb-entities";
|
|
8
|
-
import type { Logger } from "@remit/logger-lambda";
|
|
9
|
-
import type { SearchIndexMessage } from "@remit/search-index-worker";
|
|
10
|
-
import { Entity } from "electrodb";
|
|
1
|
+
import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
|
|
2
|
+
import type { Logger } from "@remit/logger";
|
|
11
3
|
import type { CascadeServices } from "../cascade.js";
|
|
12
4
|
import {
|
|
13
|
-
dataBackend as defaultDataBackend,
|
|
14
5
|
getAccountPurgeDeleteQueueUrl,
|
|
15
6
|
getCascadeServices,
|
|
16
|
-
getSearchIndexQueueUrl,
|
|
17
7
|
getSqsClient,
|
|
18
8
|
} from "../config.js";
|
|
19
9
|
import type {
|
|
@@ -26,12 +16,8 @@ export interface ProcessPurgeFanoutDeps {
|
|
|
26
16
|
services?: CascadeServices;
|
|
27
17
|
sqs?: SQSClient;
|
|
28
18
|
accountPurgeDeleteQueueUrl?: string;
|
|
29
|
-
searchIndexQueueUrl?: string;
|
|
30
|
-
dataBackend?: string;
|
|
31
19
|
}
|
|
32
20
|
|
|
33
|
-
const SQS_BATCH_SIZE = 10;
|
|
34
|
-
|
|
35
21
|
/**
|
|
36
22
|
* Message subtrees carried per FIFO finalize message. Each item is two ids
|
|
37
23
|
* (~70 bytes of JSON), so 100 stays far under the 256 KB SQS limit, and the
|
|
@@ -40,59 +26,6 @@ const SQS_BATCH_SIZE = 10;
|
|
|
40
26
|
*/
|
|
41
27
|
const SUBTREE_BATCH_SIZE = 100;
|
|
42
28
|
|
|
43
|
-
const messageEntity = new Entity(Message, { table: "purge-keys" });
|
|
44
|
-
|
|
45
|
-
const messageKeys = (messageId: string): { pk: string; sk: string } => {
|
|
46
|
-
const { Key } = messageEntity.delete({ messageId }).params<{
|
|
47
|
-
Key: { pk: string; sk: string };
|
|
48
|
-
}>();
|
|
49
|
-
return Key;
|
|
50
|
-
};
|
|
51
|
-
|
|
52
|
-
const buildVectorDeleteEvent = (
|
|
53
|
-
accountId: string,
|
|
54
|
-
messageId: string,
|
|
55
|
-
): SearchIndexMessage => ({
|
|
56
|
-
eventName: "REMOVE",
|
|
57
|
-
entity: "Message",
|
|
58
|
-
eventID: randomUUID(),
|
|
59
|
-
eventTimestamp: Date.now(),
|
|
60
|
-
accountId,
|
|
61
|
-
keys: messageKeys(messageId),
|
|
62
|
-
messageId,
|
|
63
|
-
});
|
|
64
|
-
|
|
65
|
-
const enqueueVectorDeletes = async (
|
|
66
|
-
sqs: SQSClient,
|
|
67
|
-
queueUrl: string,
|
|
68
|
-
accountId: string,
|
|
69
|
-
messageIds: string[],
|
|
70
|
-
): Promise<void> => {
|
|
71
|
-
for (let i = 0; i < messageIds.length; i += SQS_BATCH_SIZE) {
|
|
72
|
-
const batch = messageIds.slice(i, i + SQS_BATCH_SIZE);
|
|
73
|
-
const result = await sqs.send(
|
|
74
|
-
new SendMessageBatchCommand({
|
|
75
|
-
QueueUrl: queueUrl,
|
|
76
|
-
Entries: batch.map((messageId, idx) => ({
|
|
77
|
-
Id: `${i + idx}`,
|
|
78
|
-
MessageBody: JSON.stringify(
|
|
79
|
-
buildVectorDeleteEvent(accountId, messageId),
|
|
80
|
-
),
|
|
81
|
-
})),
|
|
82
|
-
}),
|
|
83
|
-
);
|
|
84
|
-
// SendMessageBatch returns HTTP 200 with per-entry failures in `Failed`;
|
|
85
|
-
// throw so the failed deletes surface to the DLQ and the whole fanout
|
|
86
|
-
// replays idempotently rather than silently skipping vector deletes.
|
|
87
|
-
if (result.Failed?.length) {
|
|
88
|
-
const failedIds = result.Failed.map((f) => f.Id).join(", ");
|
|
89
|
-
throw new Error(
|
|
90
|
-
`Search-index vector-delete batch had ${result.Failed.length} failed entries: ${failedIds}`,
|
|
91
|
-
);
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
};
|
|
95
|
-
|
|
96
29
|
/**
|
|
97
30
|
* Enqueue the destructive purge work onto the FIFO finalize queue, single
|
|
98
31
|
* message group per account so the worker processes everything strictly in
|
|
@@ -141,9 +74,8 @@ const enqueuePurgeFinalize = async (
|
|
|
141
74
|
* manifest (`pk = accountConfigId`, scoped to this account's mailbox set) —
|
|
142
75
|
* since the threading-visibility fix every persisted Message has exactly one
|
|
143
76
|
* manifest row, so the partition is a complete, describe-free index of the
|
|
144
|
-
* account's messages.
|
|
145
|
-
*
|
|
146
|
-
* worker as a stream of FIFO messages (subtree batches + one container
|
|
77
|
+
* account's messages. Hands the destructive DDB+S3+CloudFront work to the
|
|
78
|
+
* finalize worker as a stream of FIFO messages (subtree batches + one container
|
|
147
79
|
* leftover, single message group). No self-loop, no recursion: the producer
|
|
148
80
|
* enqueues all the work and the worker only ever consumes (#1069).
|
|
149
81
|
*
|
|
@@ -151,8 +83,8 @@ const enqueuePurgeFinalize = async (
|
|
|
151
83
|
* its sibling accounts, the tenant-shared Address rows, and the soft-deleted
|
|
152
84
|
* account row are untouched.
|
|
153
85
|
*
|
|
154
|
-
* Replay-safe: the manifest read is read-only and the
|
|
155
|
-
*
|
|
86
|
+
* Replay-safe: the manifest read is read-only and the finalize enqueues are
|
|
87
|
+
* idempotent. A redelivery after the account's rows are gone reads
|
|
156
88
|
* an empty manifest and enqueues only a container leftover that no-ops.
|
|
157
89
|
*/
|
|
158
90
|
export const processAccountDataPurge = async (
|
|
@@ -162,13 +94,8 @@ export const processAccountDataPurge = async (
|
|
|
162
94
|
): Promise<void> => {
|
|
163
95
|
const { accountId, accountConfigId } = event;
|
|
164
96
|
const services = deps.services ?? (await getCascadeServices());
|
|
165
|
-
const sqsFor = (queueUrl: string): SQSClient =>
|
|
166
|
-
deps.sqs ?? getSqsClient(queueUrl);
|
|
167
97
|
const accountPurgeDeleteQueueUrl =
|
|
168
98
|
deps.accountPurgeDeleteQueueUrl ?? getAccountPurgeDeleteQueueUrl();
|
|
169
|
-
const searchIndexQueueUrl =
|
|
170
|
-
deps.searchIndexQueueUrl ?? getSearchIndexQueueUrl();
|
|
171
|
-
const dataBackend = deps.dataBackend ?? defaultDataBackend;
|
|
172
99
|
|
|
173
100
|
let mailboxIds: Set<string>;
|
|
174
101
|
try {
|
|
@@ -194,27 +121,8 @@ export const processAccountDataPurge = async (
|
|
|
194
121
|
messageId: tm.messageId,
|
|
195
122
|
}));
|
|
196
123
|
|
|
197
|
-
const messageIds = [...new Set(items.map((i) => i.messageId))];
|
|
198
|
-
|
|
199
|
-
// On the relational backend the destructive delete emits a `message.removed`
|
|
200
|
-
// outbox row per message, which the search-index worker relays as a
|
|
201
|
-
// search-index REMOVE; enqueuing here too would double the removal. On
|
|
202
|
-
// DynamoDB this fanout is the sole producer of the vector deletes (#457).
|
|
203
|
-
if (dataBackend !== "sqlite") {
|
|
204
|
-
await enqueueVectorDeletes(
|
|
205
|
-
sqsFor(searchIndexQueueUrl),
|
|
206
|
-
searchIndexQueueUrl,
|
|
207
|
-
accountId,
|
|
208
|
-
messageIds,
|
|
209
|
-
);
|
|
210
|
-
log.info(
|
|
211
|
-
{ accountConfigId, accountId, count: messageIds.length },
|
|
212
|
-
"Enqueued search-index deletes for purged account",
|
|
213
|
-
);
|
|
214
|
-
}
|
|
215
|
-
|
|
216
124
|
await enqueuePurgeFinalize(
|
|
217
|
-
|
|
125
|
+
deps.sqs ?? getSqsClient(accountPurgeDeleteQueueUrl),
|
|
218
126
|
accountPurgeDeleteQueueUrl,
|
|
219
127
|
accountId,
|
|
220
128
|
accountConfigId,
|
|
@@ -1,10 +1,24 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import {
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { afterEach, describe, it } from "node:test";
|
|
3
4
|
import type { RemitClient } from "@remit/backend/client";
|
|
4
|
-
import
|
|
5
|
-
|
|
5
|
+
import {
|
|
6
|
+
buildOrganizeMatchDeps,
|
|
7
|
+
type OrganizeMatchDeps,
|
|
8
|
+
} from "@remit/backend/organize";
|
|
9
|
+
import type { FilterItem } from "@remit/data-ports";
|
|
10
|
+
import { NotFoundError } from "@remit/data-ports/errors";
|
|
11
|
+
import {
|
|
12
|
+
DrizzleThreadMessageRepository,
|
|
13
|
+
SearchIndex,
|
|
14
|
+
} from "@remit/drizzle-service";
|
|
15
|
+
import { createShippedSqliteDb } from "@remit/drizzle-service/test-sqlite";
|
|
16
|
+
import { noopLogger } from "@remit/logger";
|
|
6
17
|
import type { OrganizeJobEvent } from "../events.js";
|
|
7
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
type ProcessOrganizeJobDeps,
|
|
20
|
+
processOrganizeJob,
|
|
21
|
+
} from "./organize-job.js";
|
|
8
22
|
|
|
9
23
|
interface Update {
|
|
10
24
|
state: string;
|
|
@@ -29,10 +43,9 @@ const jobClient = (
|
|
|
29
43
|
get: async () => ({
|
|
30
44
|
organizeJobId: "job-1",
|
|
31
45
|
accountConfigId: "cfg-1",
|
|
32
|
-
|
|
46
|
+
filterId: "None",
|
|
33
47
|
matchOperator: "And",
|
|
34
48
|
literalClauses,
|
|
35
|
-
similarityThreshold: 0.75,
|
|
36
49
|
actionLabelId: "lbl-1",
|
|
37
50
|
actionMailboxId: "None",
|
|
38
51
|
}),
|
|
@@ -49,19 +62,10 @@ const jobClient = (
|
|
|
49
62
|
*/
|
|
50
63
|
const explodingMatchDeps = (): OrganizeMatchDeps =>
|
|
51
64
|
({
|
|
52
|
-
semantic: () => {
|
|
53
|
-
throw new Error("the vector pipeline must not be reached");
|
|
54
|
-
},
|
|
55
65
|
listAccountFilterMessages: async () => {
|
|
56
66
|
throw new Error("SQLITE_BUSY");
|
|
57
67
|
},
|
|
58
|
-
|
|
59
|
-
listByAccountConfig: async () => [],
|
|
60
|
-
put: async () => {
|
|
61
|
-
throw new Error("unreachable");
|
|
62
|
-
},
|
|
63
|
-
},
|
|
64
|
-
}) as unknown as OrganizeMatchDeps;
|
|
68
|
+
}) as OrganizeMatchDeps;
|
|
65
69
|
|
|
66
70
|
describe("processOrganizeJob (reader #463)", () => {
|
|
67
71
|
it("fails the job on a refused rule and acknowledges the record", async () => {
|
|
@@ -104,3 +108,297 @@ describe("processOrganizeJob (reader #463)", () => {
|
|
|
104
108
|
assert.equal(updates[1]?.errorMessage, "SQLITE_BUSY");
|
|
105
109
|
});
|
|
106
110
|
});
|
|
111
|
+
|
|
112
|
+
interface Move {
|
|
113
|
+
messageId: string;
|
|
114
|
+
destinationMailboxId: string;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const CONFIG = "cfg-1";
|
|
118
|
+
const INBOX = "mbx-inbox";
|
|
119
|
+
const ARCHIVE = "mbx-archive";
|
|
120
|
+
|
|
121
|
+
const standingFilter = (over: Partial<FilterItem> = {}): FilterItem => ({
|
|
122
|
+
filterId: "flt-1",
|
|
123
|
+
accountConfigId: CONFIG,
|
|
124
|
+
name: "Invoices",
|
|
125
|
+
scope: "Standing",
|
|
126
|
+
state: "Active",
|
|
127
|
+
disabledReason: "None",
|
|
128
|
+
ruleChangedAt: 1,
|
|
129
|
+
actionChangedAt: 1,
|
|
130
|
+
matchOperator: "And",
|
|
131
|
+
literalClauses: [{ field: "From", value: "billing@example.com" }],
|
|
132
|
+
actionLabelId: "None",
|
|
133
|
+
actionMailboxId: "mbx-invoices",
|
|
134
|
+
createdAt: 0,
|
|
135
|
+
updatedAt: 0,
|
|
136
|
+
...over,
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The thread rows live in the shipped SQLite schema, read through the real
|
|
141
|
+
* repository, so the inbox narrowing is the store's own query. One invoice
|
|
142
|
+
* sits in the inbox and one in the archive, both from the filter's sender
|
|
143
|
+
* and both saying "overdue" in their preview.
|
|
144
|
+
*/
|
|
145
|
+
const seedThreads = async (
|
|
146
|
+
threadMessage: DrizzleThreadMessageRepository,
|
|
147
|
+
): Promise<{ inbox: string; archive: string }> => {
|
|
148
|
+
const seed = async (mailboxId: string, uid: number): Promise<string> => {
|
|
149
|
+
const messageId = randomUUID();
|
|
150
|
+
await threadMessage.create({
|
|
151
|
+
accountConfigId: CONFIG,
|
|
152
|
+
threadId: randomUUID(),
|
|
153
|
+
messageId,
|
|
154
|
+
mailboxId,
|
|
155
|
+
uid,
|
|
156
|
+
referenceOrder: 0,
|
|
157
|
+
internalDate: uid,
|
|
158
|
+
sentDate: uid,
|
|
159
|
+
subject: "Your invoice",
|
|
160
|
+
fromEmail: "billing@example.com",
|
|
161
|
+
fromName: "Billing",
|
|
162
|
+
snippet: "This invoice is overdue",
|
|
163
|
+
isRead: false,
|
|
164
|
+
isDeleted: false,
|
|
165
|
+
hasAttachment: false,
|
|
166
|
+
hasStars: false,
|
|
167
|
+
});
|
|
168
|
+
return messageId;
|
|
169
|
+
};
|
|
170
|
+
return { inbox: await seed(INBOX, 1), archive: await seed(ARCHIVE, 2) };
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
interface World {
|
|
174
|
+
client: RemitClient;
|
|
175
|
+
inbox: string;
|
|
176
|
+
archive: string;
|
|
177
|
+
updates: Update[];
|
|
178
|
+
filterUpdates: Array<{ filterId: string; patch: Record<string, unknown> }>;
|
|
179
|
+
moves: Move[];
|
|
180
|
+
labeled: string[];
|
|
181
|
+
close: () => void;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const world = async (filters: FilterItem[]): Promise<World> => {
|
|
185
|
+
const store = createShippedSqliteDb();
|
|
186
|
+
const threadMessage = new DrizzleThreadMessageRepository(
|
|
187
|
+
store.db as never,
|
|
188
|
+
new SearchIndex(store.db as never, store.db as never),
|
|
189
|
+
);
|
|
190
|
+
const { inbox, archive } = await seedThreads(threadMessage);
|
|
191
|
+
const updates: Update[] = [];
|
|
192
|
+
const filterUpdates: World["filterUpdates"] = [];
|
|
193
|
+
const labeled: string[] = [];
|
|
194
|
+
const mailboxOf: Record<string, string> = {
|
|
195
|
+
[inbox]: INBOX,
|
|
196
|
+
[archive]: ARCHIVE,
|
|
197
|
+
};
|
|
198
|
+
const client = {
|
|
199
|
+
threadMessage,
|
|
200
|
+
organizeJobRequest: {
|
|
201
|
+
get: async () => ({
|
|
202
|
+
organizeJobId: "job-1",
|
|
203
|
+
accountConfigId: CONFIG,
|
|
204
|
+
filterId: "flt-1",
|
|
205
|
+
matchOperator: "And",
|
|
206
|
+
literalClauses: [],
|
|
207
|
+
actionLabelId: "None",
|
|
208
|
+
actionMailboxId: "None",
|
|
209
|
+
}),
|
|
210
|
+
update: async (_id: string, patch: Update) => {
|
|
211
|
+
updates.push(patch);
|
|
212
|
+
},
|
|
213
|
+
},
|
|
214
|
+
filter: {
|
|
215
|
+
get: async (_cfg: string, filterId: string) => {
|
|
216
|
+
const found = filters.find((f) => f.filterId === filterId);
|
|
217
|
+
if (!found) throw new NotFoundError(`Filter not found: ${filterId}`);
|
|
218
|
+
return found;
|
|
219
|
+
},
|
|
220
|
+
refreshExpiry: async (filter: FilterItem) => filter,
|
|
221
|
+
listByAccountAndState: async (_cfg: string, state: string) =>
|
|
222
|
+
filters.filter((f) => f.state === state),
|
|
223
|
+
update: async (
|
|
224
|
+
_cfg: string,
|
|
225
|
+
filterId: string,
|
|
226
|
+
patch: Record<string, unknown>,
|
|
227
|
+
) => {
|
|
228
|
+
filterUpdates.push({ filterId, patch });
|
|
229
|
+
return {};
|
|
230
|
+
},
|
|
231
|
+
},
|
|
232
|
+
account: {
|
|
233
|
+
listMailSourcesByAccountConfig: async () => [{ accountId: "acct-1" }],
|
|
234
|
+
get: async (accountId: string) => ({
|
|
235
|
+
accountId,
|
|
236
|
+
email: accountId === "acct-1" ? "me@example.com" : "me-too@example.com",
|
|
237
|
+
}),
|
|
238
|
+
},
|
|
239
|
+
mailboxSpecialUse: {
|
|
240
|
+
findInboxMailbox: async () => ({ mailboxId: INBOX, fullPath: "INBOX" }),
|
|
241
|
+
},
|
|
242
|
+
message: {
|
|
243
|
+
get: async (messageId: string) => ({
|
|
244
|
+
messageId,
|
|
245
|
+
mailboxId: mailboxOf[messageId],
|
|
246
|
+
}),
|
|
247
|
+
},
|
|
248
|
+
mailbox: {
|
|
249
|
+
resolveAccountId: async (mailboxId: string) =>
|
|
250
|
+
mailboxId === "mbx-gone"
|
|
251
|
+
? null
|
|
252
|
+
: mailboxId === "mbx-sibling-travel"
|
|
253
|
+
? "acct-2"
|
|
254
|
+
: "acct-1",
|
|
255
|
+
get: async (_accountId: string, mailboxId: string) => ({
|
|
256
|
+
mailboxId,
|
|
257
|
+
fullPath: mailboxId === "mbx-sibling-travel" ? "Travel" : "Invoices",
|
|
258
|
+
}),
|
|
259
|
+
},
|
|
260
|
+
messageLabel: {
|
|
261
|
+
apply: async (input: { messageId: string }) => {
|
|
262
|
+
labeled.push(input.messageId);
|
|
263
|
+
return {};
|
|
264
|
+
},
|
|
265
|
+
},
|
|
266
|
+
} as unknown as RemitClient;
|
|
267
|
+
return {
|
|
268
|
+
client,
|
|
269
|
+
inbox,
|
|
270
|
+
archive,
|
|
271
|
+
updates,
|
|
272
|
+
filterUpdates,
|
|
273
|
+
moves: [],
|
|
274
|
+
labeled,
|
|
275
|
+
close: store.close,
|
|
276
|
+
};
|
|
277
|
+
};
|
|
278
|
+
|
|
279
|
+
const recordingMoves = (moves: Move[]) =>
|
|
280
|
+
({
|
|
281
|
+
moveMessage: async (
|
|
282
|
+
_cfg: string,
|
|
283
|
+
messageId: string,
|
|
284
|
+
destinationMailboxId: string,
|
|
285
|
+
) => {
|
|
286
|
+
moves.push({ messageId, destinationMailboxId });
|
|
287
|
+
},
|
|
288
|
+
}) as unknown as NonNullable<ProcessOrganizeJobDeps["moveService"]>;
|
|
289
|
+
|
|
290
|
+
const runFilterJob = (w: World) =>
|
|
291
|
+
processOrganizeJob(event, noopLogger, {
|
|
292
|
+
client: w.client,
|
|
293
|
+
matchDeps: buildOrganizeMatchDeps(w.client),
|
|
294
|
+
moveService: recordingMoves(w.moves),
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
describe("processOrganizeJob for a filter job (#1354)", () => {
|
|
298
|
+
let current: World | undefined;
|
|
299
|
+
afterEach(() => current?.close());
|
|
300
|
+
|
|
301
|
+
it("moves the pre-existing matching inbox message and leaves the archive alone", async () => {
|
|
302
|
+
current = await world([standingFilter()]);
|
|
303
|
+
|
|
304
|
+
await runFilterJob(current);
|
|
305
|
+
|
|
306
|
+
assert.deepEqual(current.moves, [
|
|
307
|
+
{ messageId: current.inbox, destinationMailboxId: "mbx-invoices" },
|
|
308
|
+
]);
|
|
309
|
+
const done = current.updates.at(-1);
|
|
310
|
+
assert.equal(done?.state, "Complete");
|
|
311
|
+
assert.equal(done?.matchedCount, 1);
|
|
312
|
+
assert.equal(done?.appliedCount, 1);
|
|
313
|
+
});
|
|
314
|
+
|
|
315
|
+
it("matches a free-text rule from search on the stored preview", async () => {
|
|
316
|
+
current = await world([
|
|
317
|
+
standingFilter({
|
|
318
|
+
literalClauses: [{ field: "HasWords", value: "overdue" }],
|
|
319
|
+
}),
|
|
320
|
+
]);
|
|
321
|
+
|
|
322
|
+
await runFilterJob(current);
|
|
323
|
+
|
|
324
|
+
assert.deepEqual(current.moves, [
|
|
325
|
+
{ messageId: current.inbox, destinationMailboxId: "mbx-invoices" },
|
|
326
|
+
]);
|
|
327
|
+
assert.equal(current.updates.at(-1)?.state, "Complete");
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
it("leaves the move to a newer filter that also matches, and still labels", async () => {
|
|
331
|
+
current = await world([
|
|
332
|
+
standingFilter({ actionLabelId: "lbl-bills" }),
|
|
333
|
+
standingFilter({
|
|
334
|
+
filterId: "flt-2",
|
|
335
|
+
name: "Receipts",
|
|
336
|
+
actionChangedAt: 2,
|
|
337
|
+
actionMailboxId: "mbx-receipts",
|
|
338
|
+
}),
|
|
339
|
+
]);
|
|
340
|
+
|
|
341
|
+
await runFilterJob(current);
|
|
342
|
+
|
|
343
|
+
assert.deepEqual(current.moves, []);
|
|
344
|
+
assert.deepEqual(current.labeled, [current.inbox]);
|
|
345
|
+
assert.equal(current.updates.at(-1)?.state, "Complete");
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
it("skips another account's mail, as if the filter had not matched", async () => {
|
|
349
|
+
current = await world([
|
|
350
|
+
standingFilter({
|
|
351
|
+
actionLabelId: "lbl-trips",
|
|
352
|
+
actionMailboxId: "mbx-sibling-travel",
|
|
353
|
+
}),
|
|
354
|
+
]);
|
|
355
|
+
|
|
356
|
+
await runFilterJob(current);
|
|
357
|
+
|
|
358
|
+
assert.deepEqual(current.moves, []);
|
|
359
|
+
assert.deepEqual(current.labeled, []);
|
|
360
|
+
assert.deepEqual(current.filterUpdates, []);
|
|
361
|
+
const done = current.updates.at(-1);
|
|
362
|
+
assert.equal(done?.state, "Complete");
|
|
363
|
+
assert.equal(done?.appliedCount, 0);
|
|
364
|
+
assert.equal(done?.failedCount, 0);
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
it("turns the filter off and fails the job when no account holds its folder", async () => {
|
|
368
|
+
current = await world([standingFilter({ actionMailboxId: "mbx-gone" })]);
|
|
369
|
+
|
|
370
|
+
await runFilterJob(current);
|
|
371
|
+
|
|
372
|
+
assert.deepEqual(current.moves, []);
|
|
373
|
+
assert.deepEqual(current.filterUpdates, [
|
|
374
|
+
{
|
|
375
|
+
filterId: "flt-1",
|
|
376
|
+
patch: { state: "Disabled", disabledReason: "FolderMissing" },
|
|
377
|
+
},
|
|
378
|
+
]);
|
|
379
|
+
const done = current.updates.at(-1);
|
|
380
|
+
assert.equal(done?.state, "Failed");
|
|
381
|
+
assert.match(String(done?.errorMessage), /no longer exists/);
|
|
382
|
+
});
|
|
383
|
+
|
|
384
|
+
it("fails the job without a retry when the filter is turned off", async () => {
|
|
385
|
+
current = await world([
|
|
386
|
+
standingFilter({ state: "Disabled", disabledReason: "UserDisabled" }),
|
|
387
|
+
]);
|
|
388
|
+
|
|
389
|
+
await runFilterJob(current);
|
|
390
|
+
|
|
391
|
+
assert.deepEqual(current.moves, []);
|
|
392
|
+
assert.equal(current.updates.at(-1)?.state, "Failed");
|
|
393
|
+
assert.match(String(current.updates.at(-1)?.errorMessage), /turned off/);
|
|
394
|
+
});
|
|
395
|
+
|
|
396
|
+
it("fails the job without a retry when the filter was deleted", async () => {
|
|
397
|
+
current = await world([]);
|
|
398
|
+
|
|
399
|
+
await runFilterJob(current);
|
|
400
|
+
|
|
401
|
+
assert.equal(current.updates.at(-1)?.state, "Failed");
|
|
402
|
+
assert.match(String(current.updates.at(-1)?.errorMessage), /deleted/);
|
|
403
|
+
});
|
|
404
|
+
});
|
|
@@ -1,30 +1,85 @@
|
|
|
1
1
|
import { getClient, type RemitClient } from "@remit/backend/client";
|
|
2
2
|
import {
|
|
3
|
+
type ApplyOrganizeDeps,
|
|
3
4
|
applyOrganize,
|
|
5
|
+
backApplyFilter,
|
|
4
6
|
buildOrganizeMatchDeps,
|
|
5
7
|
buildOrganizeMoveService,
|
|
8
|
+
isFilterJob,
|
|
6
9
|
matchOrganize,
|
|
7
10
|
ORGANIZE_MATCH_LIMIT,
|
|
8
11
|
type OrganizeMatchDeps,
|
|
12
|
+
type OrganizeRejection,
|
|
9
13
|
predicateFromJob,
|
|
10
14
|
} from "@remit/backend/organize";
|
|
11
|
-
import type { Logger } from "@remit/logger
|
|
15
|
+
import type { Logger } from "@remit/logger";
|
|
12
16
|
import type { OrganizeJobEvent } from "../events.js";
|
|
13
17
|
|
|
14
18
|
export interface ProcessOrganizeJobDeps {
|
|
15
19
|
client?: RemitClient;
|
|
16
20
|
matchDeps?: OrganizeMatchDeps;
|
|
21
|
+
moveService?: ApplyOrganizeDeps["moveService"];
|
|
17
22
|
}
|
|
18
23
|
|
|
24
|
+
interface OrganizeOutcome {
|
|
25
|
+
rejected: OrganizeRejection | null;
|
|
26
|
+
matched: number;
|
|
27
|
+
applied: number;
|
|
28
|
+
failed: number;
|
|
29
|
+
failures: string[];
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const rejectedOutcome = (rejected: OrganizeRejection): OrganizeOutcome => ({
|
|
33
|
+
rejected,
|
|
34
|
+
matched: 0,
|
|
35
|
+
applied: 0,
|
|
36
|
+
failed: 0,
|
|
37
|
+
failures: [],
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The "all like these" pass: the predicate snapshotted on the row, over every
|
|
42
|
+
* folder, every requested move applied (reader #497).
|
|
43
|
+
*/
|
|
44
|
+
const runSnapshot = async (
|
|
45
|
+
client: RemitClient,
|
|
46
|
+
matchDeps: OrganizeMatchDeps,
|
|
47
|
+
moveService: ApplyOrganizeDeps["moveService"],
|
|
48
|
+
job: Awaited<ReturnType<RemitClient["organizeJobRequest"]["get"]>>,
|
|
49
|
+
): Promise<OrganizeOutcome> => {
|
|
50
|
+
const predicate = predicateFromJob(job);
|
|
51
|
+
const match = await matchOrganize(
|
|
52
|
+
matchDeps,
|
|
53
|
+
job.accountConfigId,
|
|
54
|
+
predicate,
|
|
55
|
+
ORGANIZE_MATCH_LIMIT,
|
|
56
|
+
);
|
|
57
|
+
if (match.rejected) return rejectedOutcome(match.rejected);
|
|
58
|
+
const { applied, failed, failures } = await applyOrganize(
|
|
59
|
+
{ client, moveService },
|
|
60
|
+
job.accountConfigId,
|
|
61
|
+
match.messageIds,
|
|
62
|
+
predicate,
|
|
63
|
+
);
|
|
64
|
+
return {
|
|
65
|
+
rejected: null,
|
|
66
|
+
matched: match.messageIds.length,
|
|
67
|
+
applied,
|
|
68
|
+
failed,
|
|
69
|
+
failures,
|
|
70
|
+
};
|
|
71
|
+
};
|
|
72
|
+
|
|
19
73
|
/**
|
|
20
|
-
* Run a
|
|
21
|
-
* against the job's snapshotted predicate
|
|
22
|
-
*
|
|
74
|
+
* Run a back-apply job (RFC 034, #1278): an "all like these" pass matches the
|
|
75
|
+
* corpus against the job's snapshotted predicate; a filter job (#1354) runs the
|
|
76
|
+
* named standing filter over the inbox with index-time move precedence. Either
|
|
77
|
+
* applies the action to every match in one pass, then records the counts. Mirrors the export job's lifecycle —
|
|
23
78
|
* Running, then Complete/Failed — on the same fanout seam.
|
|
24
79
|
*
|
|
25
80
|
* Reuses the shared matcher (so the applied set equals what preview returned)
|
|
26
81
|
* and the idempotent apply plumbing; `appliedByFilterId` is never written and no
|
|
27
|
-
* Filter
|
|
82
|
+
* Filter row is ever created. Both back-apply actions run here: the
|
|
28
83
|
* additive label upsert and the exclusive folder move, the latter through the
|
|
29
84
|
* same local-first placement mover body sync uses (`buildOrganizeMoveService`),
|
|
30
85
|
* so a redelivered job re-applies both idempotently.
|
|
@@ -53,52 +108,49 @@ export const processOrganizeJob = async (
|
|
|
53
108
|
);
|
|
54
109
|
|
|
55
110
|
try {
|
|
56
|
-
const predicate = predicateFromJob(job);
|
|
57
111
|
const matchDeps = deps.matchDeps ?? buildOrganizeMatchDeps(client);
|
|
58
|
-
const
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
112
|
+
const moveService = deps.moveService ?? buildOrganizeMoveService(client);
|
|
113
|
+
const outcome = isFilterJob(job)
|
|
114
|
+
? await backApplyFilter(
|
|
115
|
+
{ client, matchDeps, moveService },
|
|
116
|
+
accountConfigId,
|
|
117
|
+
job.filterId,
|
|
118
|
+
).then((result) =>
|
|
119
|
+
result.rejected ? rejectedOutcome(result.rejected) : result,
|
|
120
|
+
)
|
|
121
|
+
: await runSnapshot(client, matchDeps, moveService, job);
|
|
64
122
|
|
|
65
|
-
if (
|
|
123
|
+
if (outcome.rejected) {
|
|
66
124
|
await client.organizeJobRequest.update(organizeJobId, {
|
|
67
125
|
state: "Failed",
|
|
68
126
|
matchedCount: 0,
|
|
69
127
|
appliedCount: 0,
|
|
70
128
|
failedCount: 0,
|
|
71
|
-
errorMessage:
|
|
129
|
+
errorMessage: outcome.rejected.message,
|
|
72
130
|
});
|
|
73
131
|
log.warn(
|
|
74
|
-
{ accountConfigId, organizeJobId, reason:
|
|
132
|
+
{ accountConfigId, organizeJobId, reason: outcome.rejected.reason },
|
|
75
133
|
"Organize back-apply refused the job's rule; failing the job without a retry",
|
|
76
134
|
);
|
|
77
135
|
return;
|
|
78
136
|
}
|
|
79
137
|
|
|
80
|
-
const {
|
|
81
|
-
const { applied, failed } = await applyOrganize(
|
|
82
|
-
{ client, moveService: buildOrganizeMoveService(client) },
|
|
83
|
-
accountConfigId,
|
|
84
|
-
messageIds,
|
|
85
|
-
predicate,
|
|
86
|
-
);
|
|
87
|
-
|
|
138
|
+
const { matched, applied, failed, failures } = outcome;
|
|
88
139
|
await client.organizeJobRequest.update(organizeJobId, {
|
|
89
140
|
state: "Complete",
|
|
90
|
-
matchedCount:
|
|
141
|
+
matchedCount: matched,
|
|
91
142
|
appliedCount: applied,
|
|
92
143
|
failedCount: failed,
|
|
144
|
+
errorMessage: failures[0] ?? "",
|
|
93
145
|
});
|
|
94
146
|
log.info(
|
|
95
147
|
{
|
|
96
148
|
accountConfigId,
|
|
97
149
|
organizeJobId,
|
|
98
|
-
|
|
150
|
+
filterId: job.filterId,
|
|
151
|
+
matched,
|
|
99
152
|
applied,
|
|
100
153
|
failed,
|
|
101
|
-
semanticUnavailable,
|
|
102
154
|
},
|
|
103
155
|
"Organize back-apply complete",
|
|
104
156
|
);
|
|
@@ -2,7 +2,7 @@ import assert from "node:assert/strict";
|
|
|
2
2
|
import { describe, it } from "node:test";
|
|
3
3
|
import type { RemitClient } from "@remit/backend/client";
|
|
4
4
|
import { NotFoundError } from "@remit/data-ports/errors";
|
|
5
|
-
import { noopLogger } from "@remit/logger
|
|
5
|
+
import { noopLogger } from "@remit/logger";
|
|
6
6
|
import type { SenderCategoryBackApplyEvent } from "../events.js";
|
|
7
7
|
import { processSenderCategoryBackApply } from "./sender-category-backapply.js";
|
|
8
8
|
|
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
SENDER_CATEGORY_BACKAPPLY_LIMIT,
|
|
5
5
|
} from "@remit/backend/sender-category";
|
|
6
6
|
import { NotFoundError } from "@remit/data-ports/errors";
|
|
7
|
-
import type { Logger } from "@remit/logger
|
|
7
|
+
import type { Logger } from "@remit/logger";
|
|
8
8
|
import type { SenderCategoryBackApplyEvent } from "../events.js";
|
|
9
9
|
|
|
10
10
|
export interface ProcessSenderCategoryBackApplyDeps {
|