@remit/backend 0.0.41 → 0.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/handlers/filter.test.ts +154 -3
- package/src/handlers/filter.ts +43 -71
- package/src/service/compose-postgres.ts +2 -0
- package/src/service/compose-sqlite.ts +2 -0
- package/src/service/create-remit-client.ts +8 -0
- package/src/service/organize.test.ts +320 -11
- package/src/service/organize.ts +267 -15
package/package.json
CHANGED
|
@@ -1,8 +1,23 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import { describe, it } from "node:test";
|
|
3
|
-
import type {
|
|
4
|
-
|
|
2
|
+
import { describe, it, mock } from "node:test";
|
|
3
|
+
import type {
|
|
4
|
+
CreateFilterInput,
|
|
5
|
+
UpdateFilterInput as UpdateFilterRequestBody,
|
|
6
|
+
} from "@remit/api-openapi-types";
|
|
7
|
+
import type {
|
|
8
|
+
CreateFilterAnchorInput,
|
|
9
|
+
CreateFilterInput as FilterAnchorTxCreateInput,
|
|
10
|
+
FilterItem,
|
|
11
|
+
} from "@remit/data-ports";
|
|
5
12
|
import {
|
|
13
|
+
FilterMatchOperator,
|
|
14
|
+
FilterScope,
|
|
15
|
+
FilterState,
|
|
16
|
+
} from "@remit/domain-enums";
|
|
17
|
+
import type { AnchorPayload } from "@remit/search-service";
|
|
18
|
+
import {
|
|
19
|
+
createFilterWithAnchor,
|
|
20
|
+
type FilterCrudDeps,
|
|
6
21
|
pickFilterUpdate,
|
|
7
22
|
rejectAnchorMutation,
|
|
8
23
|
resolveFilterScopeExpiry,
|
|
@@ -163,3 +178,139 @@ describe("resolveFilterScopeExpiry (reader #266)", () => {
|
|
|
163
178
|
);
|
|
164
179
|
});
|
|
165
180
|
});
|
|
181
|
+
|
|
182
|
+
describe("createFilterWithAnchor (#351)", () => {
|
|
183
|
+
const ACCOUNT_CONFIG_ID = "acct-1";
|
|
184
|
+
|
|
185
|
+
const baseInput: CreateFilterInput = {
|
|
186
|
+
name: "Booking confirmations",
|
|
187
|
+
scope: FilterScope.Standing,
|
|
188
|
+
matchOperator: FilterMatchOperator.And,
|
|
189
|
+
literalClauses: [],
|
|
190
|
+
actionLabelId: "None",
|
|
191
|
+
actionMailboxId: "None",
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
const baseFilter: FilterItem = {
|
|
195
|
+
filterId: "filter-1",
|
|
196
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
197
|
+
name: baseInput.name,
|
|
198
|
+
scope: FilterScope.Standing,
|
|
199
|
+
state: FilterState.Active,
|
|
200
|
+
hasAnchor: false,
|
|
201
|
+
ruleChangedAt: 1_700_000_000,
|
|
202
|
+
matchOperator: FilterMatchOperator.And,
|
|
203
|
+
literalClauses: [],
|
|
204
|
+
actionLabelId: "None",
|
|
205
|
+
actionMailboxId: "None",
|
|
206
|
+
createdAt: 1_700_000_000_000,
|
|
207
|
+
updatedAt: 1_700_000_000_000,
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
const anchorPayload: AnchorPayload = {
|
|
211
|
+
anchorEmbedding: [0.1, 0.2, 0.3],
|
|
212
|
+
anchorEmbeddingId: "amazon.titan-embed-text-v2:0@1024",
|
|
213
|
+
anchorSourceText: "Your booking is confirmed",
|
|
214
|
+
};
|
|
215
|
+
|
|
216
|
+
type AnchorArg = Omit<CreateFilterAnchorInput, "filterId"> | null;
|
|
217
|
+
const createWithAnchorMock = (
|
|
218
|
+
impl: (
|
|
219
|
+
filter: FilterAnchorTxCreateInput,
|
|
220
|
+
anchor: AnchorArg,
|
|
221
|
+
) => Promise<FilterItem>,
|
|
222
|
+
) => mock.fn(impl);
|
|
223
|
+
|
|
224
|
+
it("creates a purely-literal filter through one atomic call when there is no anchor message", async () => {
|
|
225
|
+
const createWithAnchor = createWithAnchorMock(async () => baseFilter);
|
|
226
|
+
const deps: FilterCrudDeps = {
|
|
227
|
+
filterAnchorTransaction: { createWithAnchor },
|
|
228
|
+
buildAnchor: mock.fn(async () => null),
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
const filter = await createFilterWithAnchor(
|
|
232
|
+
deps,
|
|
233
|
+
ACCOUNT_CONFIG_ID,
|
|
234
|
+
baseInput,
|
|
235
|
+
);
|
|
236
|
+
|
|
237
|
+
assert.equal(filter, baseFilter);
|
|
238
|
+
assert.equal(createWithAnchor.mock.calls.length, 1);
|
|
239
|
+
const [filterArg, anchorArg] = createWithAnchor.mock.calls[0].arguments;
|
|
240
|
+
assert.equal(filterArg.hasAnchor, false);
|
|
241
|
+
assert.equal(anchorArg, null);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
it("passes the built anchor and the new hasAnchor flag to the same atomic call", async () => {
|
|
245
|
+
const createWithAnchor = createWithAnchorMock(async () => ({
|
|
246
|
+
...baseFilter,
|
|
247
|
+
hasAnchor: true,
|
|
248
|
+
}));
|
|
249
|
+
const buildAnchor = mock.fn(async () => anchorPayload);
|
|
250
|
+
const deps: FilterCrudDeps = {
|
|
251
|
+
filterAnchorTransaction: { createWithAnchor },
|
|
252
|
+
buildAnchor,
|
|
253
|
+
};
|
|
254
|
+
const input: CreateFilterInput = {
|
|
255
|
+
...baseInput,
|
|
256
|
+
anchorMessageId: "msg-1",
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
const filter = await createFilterWithAnchor(deps, ACCOUNT_CONFIG_ID, input);
|
|
260
|
+
|
|
261
|
+
assert.equal(filter.hasAnchor, true);
|
|
262
|
+
assert.equal(buildAnchor.mock.calls.length, 1);
|
|
263
|
+
assert.deepEqual(buildAnchor.mock.calls[0].arguments, [
|
|
264
|
+
ACCOUNT_CONFIG_ID,
|
|
265
|
+
"msg-1",
|
|
266
|
+
]);
|
|
267
|
+
|
|
268
|
+
assert.equal(createWithAnchor.mock.calls.length, 1);
|
|
269
|
+
const [filterArg, anchorArg] = createWithAnchor.mock.calls[0].arguments;
|
|
270
|
+
assert.equal(filterArg.hasAnchor, true);
|
|
271
|
+
assert.deepEqual(anchorArg, {
|
|
272
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
273
|
+
anchorMessageId: "msg-1",
|
|
274
|
+
anchorEmbedding: anchorPayload.anchorEmbedding,
|
|
275
|
+
anchorEmbeddingId: anchorPayload.anchorEmbeddingId,
|
|
276
|
+
anchorSourceText: anchorPayload.anchorSourceText,
|
|
277
|
+
});
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
it("creates a purely-literal filter when the anchor message has no indexed chunks", async () => {
|
|
281
|
+
const createWithAnchor = createWithAnchorMock(async () => baseFilter);
|
|
282
|
+
const deps: FilterCrudDeps = {
|
|
283
|
+
filterAnchorTransaction: { createWithAnchor },
|
|
284
|
+
buildAnchor: mock.fn(async () => null),
|
|
285
|
+
};
|
|
286
|
+
const input: CreateFilterInput = {
|
|
287
|
+
...baseInput,
|
|
288
|
+
anchorMessageId: "msg-with-no-chunks",
|
|
289
|
+
};
|
|
290
|
+
|
|
291
|
+
await createFilterWithAnchor(deps, ACCOUNT_CONFIG_ID, input);
|
|
292
|
+
|
|
293
|
+
const [filterArg, anchorArg] = createWithAnchor.mock.calls[0].arguments;
|
|
294
|
+
assert.equal(filterArg.hasAnchor, false);
|
|
295
|
+
assert.equal(anchorArg, null);
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
it("propagates a failed atomic write as a failed create request, not a silent partial success", async () => {
|
|
299
|
+
const createWithAnchor = createWithAnchorMock(async () => {
|
|
300
|
+
throw new Error("anchor write failed");
|
|
301
|
+
});
|
|
302
|
+
const deps: FilterCrudDeps = {
|
|
303
|
+
filterAnchorTransaction: { createWithAnchor },
|
|
304
|
+
buildAnchor: mock.fn(async () => anchorPayload),
|
|
305
|
+
};
|
|
306
|
+
const input: CreateFilterInput = {
|
|
307
|
+
...baseInput,
|
|
308
|
+
anchorMessageId: "msg-1",
|
|
309
|
+
};
|
|
310
|
+
|
|
311
|
+
await assert.rejects(
|
|
312
|
+
createFilterWithAnchor(deps, ACCOUNT_CONFIG_ID, input),
|
|
313
|
+
/anchor write failed/,
|
|
314
|
+
);
|
|
315
|
+
});
|
|
316
|
+
});
|
package/src/handlers/filter.ts
CHANGED
|
@@ -3,7 +3,11 @@ import type {
|
|
|
3
3
|
FilterResponse,
|
|
4
4
|
UpdateFilterInput as UpdateFilterRequestBody,
|
|
5
5
|
} from "@remit/api-openapi-types";
|
|
6
|
-
import type {
|
|
6
|
+
import type {
|
|
7
|
+
FilterItem,
|
|
8
|
+
IFilterAnchorTransaction,
|
|
9
|
+
UpdateFilterInput,
|
|
10
|
+
} from "@remit/data-ports";
|
|
7
11
|
import { BadRequestError } from "@remit/data-ports/errors";
|
|
8
12
|
import { FilterScope, FilterState } from "@remit/domain-enums";
|
|
9
13
|
import type { AnchorPayload } from "@remit/search-service";
|
|
@@ -19,46 +23,18 @@ import type {
|
|
|
19
23
|
import { assertAccountOwnership } from "./account-ownership.js";
|
|
20
24
|
|
|
21
25
|
/**
|
|
22
|
-
* Minimal filter-service surface the
|
|
23
|
-
* so tests can stub it without a live table.
|
|
26
|
+
* Minimal filter-service surface the create handler needs — declared as a
|
|
27
|
+
* `Pick` so tests can stub it without a live table.
|
|
24
28
|
*/
|
|
25
29
|
export interface FilterCrudDeps {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
literalClauses: FilterItem["literalClauses"];
|
|
35
|
-
actionLabelId: string;
|
|
36
|
-
actionMailboxId: string;
|
|
37
|
-
hasAnchor: boolean;
|
|
38
|
-
}): Promise<FilterItem>;
|
|
39
|
-
get(accountConfigId: string, filterId: string): Promise<FilterItem>;
|
|
40
|
-
update(
|
|
41
|
-
accountConfigId: string,
|
|
42
|
-
filterId: string,
|
|
43
|
-
input: UpdateFilterInput,
|
|
44
|
-
): Promise<FilterItem>;
|
|
45
|
-
delete(accountConfigId: string, filterId: string): Promise<void>;
|
|
46
|
-
refreshExpiry(item: FilterItem): Promise<FilterItem>;
|
|
47
|
-
listPageByAccountConfig(
|
|
48
|
-
accountConfigId: string,
|
|
49
|
-
options?: { limit?: number; continuationToken?: string },
|
|
50
|
-
): Promise<{ items: FilterItem[]; continuationToken: string | undefined }>;
|
|
51
|
-
};
|
|
52
|
-
filterAnchor: {
|
|
53
|
-
put(input: {
|
|
54
|
-
accountConfigId: string;
|
|
55
|
-
filterId: string;
|
|
56
|
-
anchorMessageId: string;
|
|
57
|
-
anchorEmbedding: number[];
|
|
58
|
-
anchorEmbeddingId: string;
|
|
59
|
-
anchorSourceText: string;
|
|
60
|
-
}): Promise<unknown>;
|
|
61
|
-
};
|
|
30
|
+
/**
|
|
31
|
+
* Creates the `Filter` row and its optional sibling `FilterAnchor` row in
|
|
32
|
+
* one transaction (#351) — a failure on the anchor write can never leave a
|
|
33
|
+
* `Filter` durably marked `hasAnchor: true` with no matching anchor row,
|
|
34
|
+
* and it surfaces as a failed create request rather than a silently
|
|
35
|
+
* broken filter.
|
|
36
|
+
*/
|
|
37
|
+
filterAnchorTransaction: IFilterAnchorTransaction;
|
|
62
38
|
buildAnchor(
|
|
63
39
|
accountConfigId: string,
|
|
64
40
|
anchorMessageId: string,
|
|
@@ -221,12 +197,11 @@ const toFilterResponse = (item: FilterItem): FilterResponse => ({
|
|
|
221
197
|
|
|
222
198
|
/**
|
|
223
199
|
* Create a filter, wiring the semantic anchor when `anchorMessageId` is set
|
|
224
|
-
* (RFC 034 Decision 2). The anchor is built first,
|
|
225
|
-
*
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
229
|
-
* an empty anchor.
|
|
200
|
+
* (RFC 034 Decision 2). The anchor is built first, then the `Filter` row and
|
|
201
|
+
* its sibling `FilterAnchor` row are written together in one transaction
|
|
202
|
+
* (#351) — never as two independent writes a partial failure could split. A
|
|
203
|
+
* message with no indexed chunks yields no anchor: the filter is created as
|
|
204
|
+
* purely literal (`hasAnchor: false`) rather than with an empty anchor.
|
|
230
205
|
*/
|
|
231
206
|
export const createFilterWithAnchor = async (
|
|
232
207
|
deps: FilterCrudDeps,
|
|
@@ -238,31 +213,29 @@ export const createFilterWithAnchor = async (
|
|
|
238
213
|
? await deps.buildAnchor(accountConfigId, anchorMessageId)
|
|
239
214
|
: null;
|
|
240
215
|
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
name: input.name,
|
|
244
|
-
scope: input.scope,
|
|
245
|
-
expiresAt: input.expiresAt,
|
|
246
|
-
ttl: deriveFilterTtl(input.scope, input.expiresAt),
|
|
247
|
-
matchOperator: input.matchOperator,
|
|
248
|
-
literalClauses: input.literalClauses,
|
|
249
|
-
actionLabelId: input.actionLabelId,
|
|
250
|
-
actionMailboxId: input.actionMailboxId,
|
|
251
|
-
hasAnchor: anchor !== null,
|
|
252
|
-
});
|
|
253
|
-
|
|
254
|
-
if (anchor && anchorMessageId) {
|
|
255
|
-
await deps.filterAnchor.put({
|
|
216
|
+
return deps.filterAnchorTransaction.createWithAnchor(
|
|
217
|
+
{
|
|
256
218
|
accountConfigId,
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
219
|
+
name: input.name,
|
|
220
|
+
scope: input.scope,
|
|
221
|
+
expiresAt: input.expiresAt,
|
|
222
|
+
ttl: deriveFilterTtl(input.scope, input.expiresAt),
|
|
223
|
+
matchOperator: input.matchOperator,
|
|
224
|
+
literalClauses: input.literalClauses,
|
|
225
|
+
actionLabelId: input.actionLabelId,
|
|
226
|
+
actionMailboxId: input.actionMailboxId,
|
|
227
|
+
hasAnchor: anchor !== null,
|
|
228
|
+
},
|
|
229
|
+
anchor && anchorMessageId
|
|
230
|
+
? {
|
|
231
|
+
accountConfigId,
|
|
232
|
+
anchorMessageId,
|
|
233
|
+
anchorEmbedding: anchor.anchorEmbedding,
|
|
234
|
+
anchorEmbeddingId: anchor.anchorEmbeddingId,
|
|
235
|
+
anchorSourceText: anchor.anchorSourceText,
|
|
236
|
+
}
|
|
237
|
+
: null,
|
|
238
|
+
);
|
|
266
239
|
};
|
|
267
240
|
|
|
268
241
|
export const FilterOperations: Record<
|
|
@@ -311,8 +284,7 @@ export const FilterOperations: Record<
|
|
|
311
284
|
|
|
312
285
|
const filter = await createFilterWithAnchor(
|
|
313
286
|
{
|
|
314
|
-
|
|
315
|
-
filterAnchor: client.filterAnchor,
|
|
287
|
+
filterAnchorTransaction: client.filterAnchorTransaction,
|
|
316
288
|
buildAnchor: buildFilterAnchor,
|
|
317
289
|
},
|
|
318
290
|
accountConfigId,
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
AccountSettingRepo,
|
|
6
6
|
AddressRepo,
|
|
7
7
|
DrizzleEnvelopeRepository,
|
|
8
|
+
DrizzleFilterAnchorTransaction,
|
|
8
9
|
DrizzleMessageFlagRepository,
|
|
9
10
|
DrizzleMessageRepository,
|
|
10
11
|
DrizzleThreadMessageRepository,
|
|
@@ -65,6 +66,7 @@ export const buildPostgresClient = (): RemitClient => {
|
|
|
65
66
|
flagPush: new MessageFlagPushRepo(genericDb),
|
|
66
67
|
filter: new FilterRepo(genericDb),
|
|
67
68
|
filterAnchor: new FilterAnchorRepo(genericDb),
|
|
69
|
+
filterAnchorTransaction: new DrizzleFilterAnchorTransaction(genericDb),
|
|
68
70
|
label: new LabelRepo(genericDb),
|
|
69
71
|
messageLabel: new MessageLabelRepo(genericDb),
|
|
70
72
|
unitOfWork: new DrizzleUnitOfWork(messageDataDb),
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
AddressRepo,
|
|
7
7
|
createSqliteDatabase,
|
|
8
8
|
DrizzleEnvelopeRepository,
|
|
9
|
+
DrizzleFilterAnchorTransaction,
|
|
9
10
|
DrizzleMessageFlagRepository,
|
|
10
11
|
DrizzleMessageRepository,
|
|
11
12
|
DrizzleThreadMessageRepository,
|
|
@@ -73,6 +74,7 @@ export const buildSqliteClient = async (): Promise<RemitClient> => {
|
|
|
73
74
|
flagPush: new MessageFlagPushRepo(genericDb),
|
|
74
75
|
filter: new FilterRepo(genericDb),
|
|
75
76
|
filterAnchor: new FilterAnchorRepo(genericDb),
|
|
77
|
+
filterAnchorTransaction: new DrizzleFilterAnchorTransaction(genericDb),
|
|
76
78
|
label: new LabelRepo(genericDb),
|
|
77
79
|
messageLabel: new MessageLabelRepo(genericDb),
|
|
78
80
|
unitOfWork: new DrizzleUnitOfWork(messageDataDb),
|
|
@@ -6,6 +6,7 @@ import type {
|
|
|
6
6
|
IAddressRepository,
|
|
7
7
|
IEnvelopeRepository,
|
|
8
8
|
IFilterAnchorRepository,
|
|
9
|
+
IFilterAnchorTransaction,
|
|
9
10
|
IFilterRepository,
|
|
10
11
|
ILabelRepository,
|
|
11
12
|
IMailboxLockRepository,
|
|
@@ -90,6 +91,11 @@ export interface RemitClient {
|
|
|
90
91
|
// on the row still existing, so the missing reaper is housekeeping only.
|
|
91
92
|
filter: IFilterRepository;
|
|
92
93
|
filterAnchor: IFilterAnchorRepository;
|
|
94
|
+
|
|
95
|
+
// Atomically creates a Filter and its optional sibling FilterAnchor row in
|
|
96
|
+
// one transaction (#351) — a failed anchor write can never leave a Filter
|
|
97
|
+
// durably marked hasAnchor: true with no matching FilterAnchor row.
|
|
98
|
+
filterAnchorTransaction: IFilterAnchorTransaction;
|
|
93
99
|
label: ILabelRepository;
|
|
94
100
|
messageLabel: IMessageLabelRepository;
|
|
95
101
|
|
|
@@ -163,6 +169,7 @@ export interface RemitClientRepositories {
|
|
|
163
169
|
flagPush: IMessageFlagPushRepository;
|
|
164
170
|
filter: IFilterRepository;
|
|
165
171
|
filterAnchor: IFilterAnchorRepository;
|
|
172
|
+
filterAnchorTransaction: IFilterAnchorTransaction;
|
|
166
173
|
label: ILabelRepository;
|
|
167
174
|
messageLabel: IMessageLabelRepository;
|
|
168
175
|
unitOfWork?: IUnitOfWork;
|
|
@@ -365,6 +372,7 @@ export const createRemitClient = (deps: RemitClientDeps): RemitClient => {
|
|
|
365
372
|
organizeJobRequest: repositories.organizeJobRequest,
|
|
366
373
|
filter: repositories.filter,
|
|
367
374
|
filterAnchor: repositories.filterAnchor,
|
|
375
|
+
filterAnchorTransaction: repositories.filterAnchorTransaction,
|
|
368
376
|
label: repositories.label,
|
|
369
377
|
messageLabel: repositories.messageLabel,
|
|
370
378
|
unitOfWork: repositories.unitOfWork,
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { afterEach, beforeEach, describe, it } from "node:test";
|
|
3
|
-
import {
|
|
3
|
+
import type { FilterAnchorItem, FilterItem } from "@remit/data-ports";
|
|
4
|
+
import { NotFoundError } from "@remit/data-ports/errors";
|
|
5
|
+
import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
|
|
4
6
|
import type {
|
|
5
7
|
AnchorPayload,
|
|
6
8
|
ChunkMetadata,
|
|
@@ -68,15 +70,51 @@ const predicate = (
|
|
|
68
70
|
...over,
|
|
69
71
|
});
|
|
70
72
|
|
|
73
|
+
/** A standing filter fixture — the "other" filters the precedence check reads. */
|
|
74
|
+
const filterItem = (over: Partial<FilterItem> = {}): FilterItem => ({
|
|
75
|
+
filterId: "filter-other",
|
|
76
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
77
|
+
name: "other filter",
|
|
78
|
+
scope: "Standing",
|
|
79
|
+
state: FilterState.Active,
|
|
80
|
+
hasAnchor: false,
|
|
81
|
+
ruleChangedAt: 0,
|
|
82
|
+
matchOperator: FilterMatchOperator.And,
|
|
83
|
+
literalClauses: [],
|
|
84
|
+
actionLabelId: "None",
|
|
85
|
+
actionMailboxId: "None",
|
|
86
|
+
createdAt: 0,
|
|
87
|
+
updatedAt: 0,
|
|
88
|
+
...over,
|
|
89
|
+
});
|
|
90
|
+
|
|
71
91
|
/**
|
|
72
92
|
* A client that records MessageLabel writes and blows up if the back-apply path
|
|
73
93
|
* ever touches Filter/FilterAnchor — the RFC 034 guardrail: this scope never
|
|
74
94
|
* persists a standing rule.
|
|
95
|
+
*
|
|
96
|
+
* `activeFilters`/`filterAnchorRows`/`threadMessages` model the account's
|
|
97
|
+
* *other*, already-existing standing filters and message rows the exclusive-
|
|
98
|
+
* move precedence check (reader #350) reads — empty by default, so every
|
|
99
|
+
* existing test (which never seeded a competing filter) is unaffected and the
|
|
100
|
+
* precedence check is a same-length no-op.
|
|
75
101
|
*/
|
|
76
|
-
const trackingClient = (
|
|
102
|
+
const trackingClient = (
|
|
103
|
+
seed: {
|
|
104
|
+
activeFilters?: FilterItem[];
|
|
105
|
+
filterAnchorRows?: FilterAnchorItem[];
|
|
106
|
+
threadMessages?: Record<
|
|
107
|
+
string,
|
|
108
|
+
{ fromEmail?: string; fromName?: string; subject?: string }
|
|
109
|
+
>;
|
|
110
|
+
} = {},
|
|
111
|
+
) => {
|
|
77
112
|
const labeled: Array<{ messageId: string; labelId: string }> = [];
|
|
78
113
|
let filterWrites = 0;
|
|
79
114
|
let filterAnchorWrites = 0;
|
|
115
|
+
const activeFilters = seed.activeFilters ?? [];
|
|
116
|
+
const filterAnchorRows = seed.filterAnchorRows ?? [];
|
|
117
|
+
const threadMessages = seed.threadMessages ?? {};
|
|
80
118
|
const client = {
|
|
81
119
|
messageLabel: {
|
|
82
120
|
apply: async (input: {
|
|
@@ -100,17 +138,36 @@ const trackingClient = () => {
|
|
|
100
138
|
mailbox: {
|
|
101
139
|
resolveAccountId: async () => "acct-1",
|
|
102
140
|
},
|
|
141
|
+
threadMessage: {
|
|
142
|
+
get: async (_accountConfigId: string, messageId: string) => {
|
|
143
|
+
const row = threadMessages[messageId];
|
|
144
|
+
if (!row) {
|
|
145
|
+
throw new NotFoundError("ThreadMessage not found");
|
|
146
|
+
}
|
|
147
|
+
return {
|
|
148
|
+
threadMessageId: messageId,
|
|
149
|
+
fromEmail: row.fromEmail,
|
|
150
|
+
fromName: row.fromName,
|
|
151
|
+
subject: row.subject,
|
|
152
|
+
} as never;
|
|
153
|
+
},
|
|
154
|
+
},
|
|
103
155
|
filter: {
|
|
104
156
|
create: async () => {
|
|
105
157
|
filterWrites += 1;
|
|
106
158
|
return {} as never;
|
|
107
159
|
},
|
|
160
|
+
listByAccountAndState: async () => activeFilters,
|
|
161
|
+
refreshExpiry: async (filter: FilterItem) => filter,
|
|
108
162
|
},
|
|
109
163
|
filterAnchor: {
|
|
110
164
|
put: async () => {
|
|
111
165
|
filterAnchorWrites += 1;
|
|
112
166
|
return {} as never;
|
|
113
167
|
},
|
|
168
|
+
get: async (_accountConfigId: string, filterId: string) =>
|
|
169
|
+
filterAnchorRows.find((row) => row.filterId === filterId) ?? null,
|
|
170
|
+
listByAccountConfig: async () => filterAnchorRows,
|
|
114
171
|
},
|
|
115
172
|
} as unknown as RemitClient;
|
|
116
173
|
return {
|
|
@@ -160,19 +217,30 @@ const trackingMoveService = () => {
|
|
|
160
217
|
* Deps whose semantic side is the in-memory vector store — the vector-backed
|
|
161
218
|
* deployment. `listAccountFilterMessages` returns the given corpus (empty by
|
|
162
219
|
* default), and constructing the semantic side is tracked so a literal-only
|
|
163
|
-
* predicate can be shown never to build it.
|
|
220
|
+
* predicate can be shown never to build it. `filterAnchorRows` seeds the
|
|
221
|
+
* account's *persisted* FilterAnchor rows (empty by default) — the reverse
|
|
222
|
+
* lookup back-apply's anchor consultation reads (reader #350) — and `embed`
|
|
223
|
+
* is a deterministic stand-in for the configured embedding model, used only
|
|
224
|
+
* by the exclusive-move precedence check.
|
|
164
225
|
*/
|
|
165
226
|
const matchDeps = (
|
|
166
227
|
store: ReturnType<typeof createMemoryVectorStore>,
|
|
167
228
|
corpus: OrganizeCandidate[] = [],
|
|
229
|
+
filterAnchorRows: FilterAnchorItem[] = [],
|
|
168
230
|
): OrganizeMatchDeps & { semanticBuilds: () => number } => {
|
|
169
231
|
let semanticBuilds = 0;
|
|
170
232
|
return {
|
|
171
233
|
semantic: () => {
|
|
172
234
|
semanticBuilds += 1;
|
|
173
|
-
return {
|
|
235
|
+
return {
|
|
236
|
+
buildAnchor: async () => anchorPayload,
|
|
237
|
+
vectorStore: store,
|
|
238
|
+
embed: async (text: string) =>
|
|
239
|
+
text.includes("reservation") ? ANCHOR_VECTOR : ORTHOGONAL_VECTOR,
|
|
240
|
+
};
|
|
174
241
|
},
|
|
175
242
|
listAccountFilterMessages: async () => corpus,
|
|
243
|
+
filterAnchors: { listByAccountConfig: async () => filterAnchorRows },
|
|
176
244
|
semanticBuilds: () => semanticBuilds,
|
|
177
245
|
};
|
|
178
246
|
};
|
|
@@ -201,9 +269,13 @@ const vectorlessDeps = (
|
|
|
201
269
|
throw moduleNotFound();
|
|
202
270
|
},
|
|
203
271
|
},
|
|
272
|
+
embed: async () => {
|
|
273
|
+
throw moduleNotFound();
|
|
274
|
+
},
|
|
204
275
|
};
|
|
205
276
|
},
|
|
206
277
|
listAccountFilterMessages: async () => corpus,
|
|
278
|
+
filterAnchors: { listByAccountConfig: async () => [] },
|
|
207
279
|
semanticUsed: () => semanticUsed,
|
|
208
280
|
};
|
|
209
281
|
};
|
|
@@ -305,6 +377,77 @@ describe("matchOrganize", () => {
|
|
|
305
377
|
});
|
|
306
378
|
});
|
|
307
379
|
|
|
380
|
+
describe("matchOrganize honors the persisted FilterAnchor (reader #350)", () => {
|
|
381
|
+
it("still matches by reading the persisted anchor after the anchor message is purged", async () => {
|
|
382
|
+
const store = createMemoryVectorStore();
|
|
383
|
+
const matching = ["msg-1", "msg-2"];
|
|
384
|
+
await store.upsert([
|
|
385
|
+
...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
|
|
386
|
+
bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
|
|
387
|
+
]);
|
|
388
|
+
|
|
389
|
+
const persistedAnchor: FilterAnchorItem = {
|
|
390
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
391
|
+
filterId: "filter-a",
|
|
392
|
+
anchorEmbedding: ANCHOR_VECTOR,
|
|
393
|
+
anchorEmbeddingId: "test-model@4",
|
|
394
|
+
anchorSourceText: "book me a table",
|
|
395
|
+
anchorMessageId: "msg-anchor",
|
|
396
|
+
createdAt: 0,
|
|
397
|
+
updatedAt: 0,
|
|
398
|
+
};
|
|
399
|
+
|
|
400
|
+
let buildAnchorCalls = 0;
|
|
401
|
+
const deps: OrganizeMatchDeps = {
|
|
402
|
+
semantic: () => ({
|
|
403
|
+
buildAnchor: async () => {
|
|
404
|
+
// The live path: the anchor message's chunks are gone (purged),
|
|
405
|
+
// exactly what buildMessageAnchor returns in that case today.
|
|
406
|
+
buildAnchorCalls += 1;
|
|
407
|
+
return null;
|
|
408
|
+
},
|
|
409
|
+
vectorStore: store,
|
|
410
|
+
embed: async () => ANCHOR_VECTOR,
|
|
411
|
+
}),
|
|
412
|
+
listAccountFilterMessages: async () => [],
|
|
413
|
+
filterAnchors: { listByAccountConfig: async () => [persistedAnchor] },
|
|
414
|
+
};
|
|
415
|
+
|
|
416
|
+
const { messageIds } = await matchOrganize(
|
|
417
|
+
deps,
|
|
418
|
+
ACCOUNT_CONFIG_ID,
|
|
419
|
+
predicate(),
|
|
420
|
+
);
|
|
421
|
+
|
|
422
|
+
assert.deepEqual(
|
|
423
|
+
[...messageIds].sort(),
|
|
424
|
+
matching,
|
|
425
|
+
"the persisted anchor vector still finds the same matches the live message would have",
|
|
426
|
+
);
|
|
427
|
+
assert.equal(
|
|
428
|
+
buildAnchorCalls,
|
|
429
|
+
0,
|
|
430
|
+
"a persisted anchor must be read instead of re-deriving one from the (now-gone) live message",
|
|
431
|
+
);
|
|
432
|
+
});
|
|
433
|
+
|
|
434
|
+
it("falls back to deriving the anchor from the live message when no filter was ever anchored on it", async () => {
|
|
435
|
+
const store = createMemoryVectorStore();
|
|
436
|
+
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
437
|
+
|
|
438
|
+
// No persisted FilterAnchor names this anchorMessageId — an ad hoc "all
|
|
439
|
+
// like these" widen over a bare message selection, never tied to a
|
|
440
|
+
// standing filter.
|
|
441
|
+
const { messageIds } = await matchOrganize(
|
|
442
|
+
matchDeps(store),
|
|
443
|
+
ACCOUNT_CONFIG_ID,
|
|
444
|
+
predicate(),
|
|
445
|
+
);
|
|
446
|
+
|
|
447
|
+
assert.deepEqual(messageIds, ["msg-1"]);
|
|
448
|
+
});
|
|
449
|
+
});
|
|
450
|
+
|
|
308
451
|
describe("matchOrganize on a deployment without the vector pipeline", () => {
|
|
309
452
|
const ORIGINAL = process.env.DATA_BACKEND;
|
|
310
453
|
beforeEach(() => {
|
|
@@ -403,8 +546,10 @@ describe("matchOrganize on a deployment without the vector pipeline", () => {
|
|
|
403
546
|
query: async () => [],
|
|
404
547
|
getByMessage: async () => [],
|
|
405
548
|
},
|
|
549
|
+
embed: async () => [],
|
|
406
550
|
}),
|
|
407
551
|
listAccountFilterMessages: async () => [],
|
|
552
|
+
filterAnchors: { listByAccountConfig: async () => [] },
|
|
408
553
|
};
|
|
409
554
|
|
|
410
555
|
await assert.rejects(
|
|
@@ -437,7 +582,7 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
437
582
|
|
|
438
583
|
const tracked = trackingClient();
|
|
439
584
|
const result = await applyOrganize(
|
|
440
|
-
{ client: tracked.client },
|
|
585
|
+
{ client: tracked.client, match: matchDeps(store) },
|
|
441
586
|
ACCOUNT_CONFIG_ID,
|
|
442
587
|
applied.messageIds,
|
|
443
588
|
p,
|
|
@@ -473,7 +618,7 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
473
618
|
);
|
|
474
619
|
const tracked = trackingClient();
|
|
475
620
|
const result = await applyOrganize(
|
|
476
|
-
{ client: tracked.client },
|
|
621
|
+
{ client: tracked.client, match: matchDeps(store) },
|
|
477
622
|
ACCOUNT_CONFIG_ID,
|
|
478
623
|
matched,
|
|
479
624
|
p,
|
|
@@ -500,7 +645,11 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
500
645
|
const tracked = trackingClient();
|
|
501
646
|
const mover = trackingMoveService();
|
|
502
647
|
const result = await applyOrganize(
|
|
503
|
-
{
|
|
648
|
+
{
|
|
649
|
+
client: tracked.client,
|
|
650
|
+
moveService: mover.moveService,
|
|
651
|
+
match: matchDeps(store),
|
|
652
|
+
},
|
|
504
653
|
ACCOUNT_CONFIG_ID,
|
|
505
654
|
matched,
|
|
506
655
|
p,
|
|
@@ -546,7 +695,11 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
546
695
|
const tracked = trackingClient();
|
|
547
696
|
const mover = trackingMoveService();
|
|
548
697
|
const result = await applyOrganize(
|
|
549
|
-
{
|
|
698
|
+
{
|
|
699
|
+
client: tracked.client,
|
|
700
|
+
moveService: mover.moveService,
|
|
701
|
+
match: matchDeps(store),
|
|
702
|
+
},
|
|
550
703
|
ACCOUNT_CONFIG_ID,
|
|
551
704
|
matched,
|
|
552
705
|
p,
|
|
@@ -578,13 +731,21 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
578
731
|
const mover = trackingMoveService();
|
|
579
732
|
|
|
580
733
|
const first = await applyOrganize(
|
|
581
|
-
{
|
|
734
|
+
{
|
|
735
|
+
client: tracked.client,
|
|
736
|
+
moveService: mover.moveService,
|
|
737
|
+
match: matchDeps(store),
|
|
738
|
+
},
|
|
582
739
|
ACCOUNT_CONFIG_ID,
|
|
583
740
|
matched,
|
|
584
741
|
p,
|
|
585
742
|
);
|
|
586
743
|
const second = await applyOrganize(
|
|
587
|
-
{
|
|
744
|
+
{
|
|
745
|
+
client: tracked.client,
|
|
746
|
+
moveService: mover.moveService,
|
|
747
|
+
match: matchDeps(store),
|
|
748
|
+
},
|
|
588
749
|
ACCOUNT_CONFIG_ID,
|
|
589
750
|
matched,
|
|
590
751
|
p,
|
|
@@ -605,6 +766,154 @@ describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
|
605
766
|
});
|
|
606
767
|
});
|
|
607
768
|
|
|
769
|
+
describe("applyOrganize resolves move precedence against current Active filters (reader #350)", () => {
|
|
770
|
+
it("suppresses an out-ranked move but still applies the label", async () => {
|
|
771
|
+
const store = createMemoryVectorStore();
|
|
772
|
+
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
773
|
+
// The back-applied filter — "move to mbox-old" — is out-ranked by a
|
|
774
|
+
// more-recently-changed standing filter that currently claims msg-1 for a
|
|
775
|
+
// different destination.
|
|
776
|
+
const p = predicate({
|
|
777
|
+
actionLabelId: "lbl-1",
|
|
778
|
+
actionMailboxId: "mbox-old",
|
|
779
|
+
});
|
|
780
|
+
const newerFilter = filterItem({
|
|
781
|
+
filterId: "filter-newer",
|
|
782
|
+
ruleChangedAt: 1_000,
|
|
783
|
+
actionMailboxId: "mbox-new",
|
|
784
|
+
literalClauses: [{ field: "Subject", value: "reservation" }],
|
|
785
|
+
});
|
|
786
|
+
|
|
787
|
+
const { messageIds: matched } = await matchOrganize(
|
|
788
|
+
matchDeps(store),
|
|
789
|
+
ACCOUNT_CONFIG_ID,
|
|
790
|
+
p,
|
|
791
|
+
);
|
|
792
|
+
const tracked = trackingClient({
|
|
793
|
+
activeFilters: [newerFilter],
|
|
794
|
+
threadMessages: { "msg-1": { subject: "Dinner reservation" } },
|
|
795
|
+
});
|
|
796
|
+
const mover = trackingMoveService();
|
|
797
|
+
const result = await applyOrganize(
|
|
798
|
+
{
|
|
799
|
+
client: tracked.client,
|
|
800
|
+
moveService: mover.moveService,
|
|
801
|
+
match: matchDeps(store),
|
|
802
|
+
},
|
|
803
|
+
ACCOUNT_CONFIG_ID,
|
|
804
|
+
matched,
|
|
805
|
+
p,
|
|
806
|
+
);
|
|
807
|
+
|
|
808
|
+
assert.equal(result.applied, 1, "a suppressed move is not a failure");
|
|
809
|
+
assert.equal(result.failed, 0);
|
|
810
|
+
assert.deepEqual(
|
|
811
|
+
tracked.labeled,
|
|
812
|
+
[{ messageId: "msg-1", labelId: "lbl-1" }],
|
|
813
|
+
"the additive label still applies even though the move is suppressed",
|
|
814
|
+
);
|
|
815
|
+
assert.deepEqual(
|
|
816
|
+
mover.moves,
|
|
817
|
+
[],
|
|
818
|
+
"the exclusive move is skipped in favor of the newer filter's own move",
|
|
819
|
+
);
|
|
820
|
+
});
|
|
821
|
+
|
|
822
|
+
it("moves the message when no other Active filter currently outranks it", async () => {
|
|
823
|
+
const store = createMemoryVectorStore();
|
|
824
|
+
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
825
|
+
const p = predicate({ actionMailboxId: "mbox-target" });
|
|
826
|
+
// A different standing filter matches this message too, but agrees on the
|
|
827
|
+
// same destination — nothing to defer to.
|
|
828
|
+
const agreeingFilter = filterItem({
|
|
829
|
+
filterId: "filter-agrees",
|
|
830
|
+
ruleChangedAt: 1_000,
|
|
831
|
+
actionMailboxId: "mbox-target",
|
|
832
|
+
literalClauses: [{ field: "Subject", value: "reservation" }],
|
|
833
|
+
});
|
|
834
|
+
|
|
835
|
+
const { messageIds: matched } = await matchOrganize(
|
|
836
|
+
matchDeps(store),
|
|
837
|
+
ACCOUNT_CONFIG_ID,
|
|
838
|
+
p,
|
|
839
|
+
);
|
|
840
|
+
const tracked = trackingClient({
|
|
841
|
+
activeFilters: [agreeingFilter],
|
|
842
|
+
threadMessages: { "msg-1": { subject: "Dinner reservation" } },
|
|
843
|
+
});
|
|
844
|
+
const mover = trackingMoveService();
|
|
845
|
+
const result = await applyOrganize(
|
|
846
|
+
{
|
|
847
|
+
client: tracked.client,
|
|
848
|
+
moveService: mover.moveService,
|
|
849
|
+
match: matchDeps(store),
|
|
850
|
+
},
|
|
851
|
+
ACCOUNT_CONFIG_ID,
|
|
852
|
+
matched,
|
|
853
|
+
p,
|
|
854
|
+
);
|
|
855
|
+
|
|
856
|
+
assert.equal(result.applied, 1);
|
|
857
|
+
assert.equal(result.failed, 0);
|
|
858
|
+
assert.deepEqual(
|
|
859
|
+
mover.moves.map((m) => m.messageId),
|
|
860
|
+
["msg-1"],
|
|
861
|
+
"the move proceeds exactly as it would have before this check existed",
|
|
862
|
+
);
|
|
863
|
+
});
|
|
864
|
+
|
|
865
|
+
it("suppresses an out-ranked move by a newer *semantic* filter's persisted anchor", async () => {
|
|
866
|
+
const store = createMemoryVectorStore();
|
|
867
|
+
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
868
|
+
const p = predicate({ actionMailboxId: "mbox-old" });
|
|
869
|
+
const newerSemanticFilter = filterItem({
|
|
870
|
+
filterId: "filter-newer-semantic",
|
|
871
|
+
ruleChangedAt: 1_000,
|
|
872
|
+
actionMailboxId: "mbox-new",
|
|
873
|
+
hasAnchor: true,
|
|
874
|
+
});
|
|
875
|
+
const persistedAnchor: FilterAnchorItem = {
|
|
876
|
+
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
877
|
+
filterId: "filter-newer-semantic",
|
|
878
|
+
anchorEmbedding: ANCHOR_VECTOR,
|
|
879
|
+
anchorEmbeddingId: "test-model@4",
|
|
880
|
+
anchorSourceText: "book me a table",
|
|
881
|
+
anchorMessageId: "msg-anchor-2",
|
|
882
|
+
createdAt: 0,
|
|
883
|
+
updatedAt: 0,
|
|
884
|
+
};
|
|
885
|
+
|
|
886
|
+
const { messageIds: matched } = await matchOrganize(
|
|
887
|
+
matchDeps(store),
|
|
888
|
+
ACCOUNT_CONFIG_ID,
|
|
889
|
+
p,
|
|
890
|
+
);
|
|
891
|
+
const tracked = trackingClient({
|
|
892
|
+
activeFilters: [newerSemanticFilter],
|
|
893
|
+
filterAnchorRows: [persistedAnchor],
|
|
894
|
+
threadMessages: { "msg-1": { subject: "Dinner reservation" } },
|
|
895
|
+
});
|
|
896
|
+
const mover = trackingMoveService();
|
|
897
|
+
const result = await applyOrganize(
|
|
898
|
+
{
|
|
899
|
+
client: tracked.client,
|
|
900
|
+
moveService: mover.moveService,
|
|
901
|
+
match: matchDeps(store, [], [persistedAnchor]),
|
|
902
|
+
},
|
|
903
|
+
ACCOUNT_CONFIG_ID,
|
|
904
|
+
matched,
|
|
905
|
+
p,
|
|
906
|
+
);
|
|
907
|
+
|
|
908
|
+
assert.equal(result.applied, 1);
|
|
909
|
+
assert.deepEqual(
|
|
910
|
+
mover.moves,
|
|
911
|
+
[],
|
|
912
|
+
"a newer semantic filter's own persisted anchor outranks the move",
|
|
913
|
+
);
|
|
914
|
+
});
|
|
915
|
+
});
|
|
916
|
+
|
|
608
917
|
describe("matchOrganize with ListId and FromDomain clauses", () => {
|
|
609
918
|
const senderChunk = (
|
|
610
919
|
messageId: string,
|
|
@@ -702,7 +1011,7 @@ describe("matchOrganize with ListId and FromDomain clauses", () => {
|
|
|
702
1011
|
|
|
703
1012
|
const tracked = trackingClient();
|
|
704
1013
|
const result = await applyOrganize(
|
|
705
|
-
{ client: tracked.client },
|
|
1014
|
+
{ client: tracked.client, match: deps },
|
|
706
1015
|
ACCOUNT_CONFIG_ID,
|
|
707
1016
|
applied.messageIds,
|
|
708
1017
|
p,
|
package/src/service/organize.ts
CHANGED
|
@@ -1,11 +1,19 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
1
|
+
import type {
|
|
2
|
+
FilterItem,
|
|
3
|
+
IFilterAnchorRepository,
|
|
4
|
+
OrganizeJobRequestItem,
|
|
5
|
+
} from "@remit/data-ports";
|
|
6
|
+
import { NotFoundError } from "@remit/data-ports/errors";
|
|
7
|
+
import { FilterClauseField, FilterState } from "@remit/domain-enums";
|
|
3
8
|
import {
|
|
9
|
+
buildMatchText,
|
|
10
|
+
cosineSimilarity,
|
|
4
11
|
DEFAULT_SEMANTIC_MATCH_THRESHOLD,
|
|
5
12
|
type FilterMessage,
|
|
6
13
|
literalClausesMatch,
|
|
7
14
|
NO_ACTION,
|
|
8
15
|
PlacementMoveService,
|
|
16
|
+
selectMoveWinner,
|
|
9
17
|
} from "@remit/mailbox-service";
|
|
10
18
|
import {
|
|
11
19
|
type AnchorPayload,
|
|
@@ -79,6 +87,15 @@ export interface OrganizeSemanticDeps {
|
|
|
79
87
|
anchorMessageId: string,
|
|
80
88
|
) => Promise<AnchorPayload | null>;
|
|
81
89
|
vectorStore: Pick<VectorStoreService, "query" | "getByMessage">;
|
|
90
|
+
/**
|
|
91
|
+
* Embed one candidate message's text — used only by the cross-filter
|
|
92
|
+
* precedence check ({@link findCurrentMoveWinner}) to compare a message
|
|
93
|
+
* against a *different* Active filter's persisted anchor, the same
|
|
94
|
+
* comparison `FilterPipeline.filterMatches` runs at index time. Never
|
|
95
|
+
* called by the anchor widen itself, which only ever runs a vector-store
|
|
96
|
+
* kNN read (see `semantic-capability.ts`).
|
|
97
|
+
*/
|
|
98
|
+
embed: (text: string) => Promise<number[]>;
|
|
82
99
|
}
|
|
83
100
|
|
|
84
101
|
/**
|
|
@@ -107,6 +124,17 @@ export interface OrganizeMatchDeps {
|
|
|
107
124
|
accountConfigId: string,
|
|
108
125
|
limit: number,
|
|
109
126
|
) => Promise<OrganizeCandidate[]>;
|
|
127
|
+
/**
|
|
128
|
+
* Every persisted FilterAnchor for the account — read-only, always
|
|
129
|
+
* available (never gated behind {@link semantic}, since it never touches
|
|
130
|
+
* the vector store). A back-apply predicate carries only a bare
|
|
131
|
+
* `anchorMessageId`, never a `filterId` (RFC 034 recap: this job is
|
|
132
|
+
* deliberately not a Filter), so this is how {@link matchSemantic} finds
|
|
133
|
+
* "the standing filter this anchor came from," if one still exists, to
|
|
134
|
+
* read its fixed-at-save-time vector instead of re-deriving one from the
|
|
135
|
+
* anchor message's current chunks (reader #350 / RFC 039 Decision 1).
|
|
136
|
+
*/
|
|
137
|
+
filterAnchors: Pick<IFilterAnchorRepository, "listByAccountConfig">;
|
|
110
138
|
}
|
|
111
139
|
|
|
112
140
|
/** The matched ids plus whether the semantic widen was skipped as unavailable. */
|
|
@@ -152,23 +180,61 @@ const filterMessageFromChunks = (
|
|
|
152
180
|
};
|
|
153
181
|
|
|
154
182
|
/**
|
|
155
|
-
* The
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
*
|
|
159
|
-
* the
|
|
160
|
-
*
|
|
183
|
+
* The persisted FilterAnchor whose `anchorMessageId` matches this predicate's
|
|
184
|
+
* anchor, if the account has a standing filter built from that message (RFC
|
|
185
|
+
* 034 Decision 2 / RFC 039 Decision 1, reader #350). A back-apply predicate
|
|
186
|
+
* never carries a `filterId` — this job structurally is not a Filter — so the
|
|
187
|
+
* only way to recover "the filter this anchor came from" is by the value it
|
|
188
|
+
* was anchored on. The scan is over one account's (small, bounded) anchor
|
|
189
|
+
* rows, read-only, and never touches the vector store. Returns `undefined`
|
|
190
|
+
* when no standing filter was ever anchored on this message (or it has since
|
|
191
|
+
* been deleted), in which case the caller falls back to deriving the anchor
|
|
192
|
+
* live from the message's current chunk vectors, exactly as before.
|
|
193
|
+
*/
|
|
194
|
+
const findPersistedAnchor = async (
|
|
195
|
+
filterAnchors: Pick<IFilterAnchorRepository, "listByAccountConfig">,
|
|
196
|
+
accountConfigId: string,
|
|
197
|
+
anchorMessageId: string,
|
|
198
|
+
): Promise<AnchorPayload | undefined> => {
|
|
199
|
+
const anchors = await filterAnchors.listByAccountConfig(accountConfigId);
|
|
200
|
+
const persisted = anchors.find(
|
|
201
|
+
(anchor) => anchor.anchorMessageId === anchorMessageId,
|
|
202
|
+
);
|
|
203
|
+
if (!persisted) return undefined;
|
|
204
|
+
return {
|
|
205
|
+
anchorEmbedding: persisted.anchorEmbedding,
|
|
206
|
+
anchorEmbeddingId: persisted.anchorEmbeddingId,
|
|
207
|
+
anchorSourceText: persisted.anchorSourceText,
|
|
208
|
+
};
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* The semantic (anchor) arm: read the anchor vector — the persisted
|
|
213
|
+
* `FilterAnchor` for a message a standing filter was built from (fixed at
|
|
214
|
+
* save time, unaffected by the anchor message's later deletion or
|
|
215
|
+
* re-chunking), or else pool it fresh from the anchor message's existing
|
|
216
|
+
* chunk vectors, the same as before this filter existed or ever had one.
|
|
217
|
+
* Fan out with a k-NN query gated on the cosine threshold, then refine by
|
|
218
|
+
* literal clauses reconstructed from the same chunk vectors. Every read here
|
|
219
|
+
* goes through the vector store; a deployment without the vector pipeline
|
|
220
|
+
* fails on the first call, which {@link matchOrganize} catches. Returns null
|
|
221
|
+
* when neither a persisted anchor nor the message's own chunk vectors exist
|
|
222
|
+
* to pool.
|
|
161
223
|
*/
|
|
162
224
|
const matchSemantic = async (
|
|
163
|
-
|
|
225
|
+
deps: OrganizeMatchDeps,
|
|
164
226
|
accountConfigId: string,
|
|
165
227
|
predicate: OrganizePredicate,
|
|
166
228
|
limit: number,
|
|
167
229
|
): Promise<string[] | null> => {
|
|
168
|
-
const
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
230
|
+
const semantic = deps.semantic();
|
|
231
|
+
const anchor =
|
|
232
|
+
(await findPersistedAnchor(
|
|
233
|
+
deps.filterAnchors,
|
|
234
|
+
accountConfigId,
|
|
235
|
+
predicate.anchorMessageId,
|
|
236
|
+
)) ??
|
|
237
|
+
(await semantic.buildAnchor(accountConfigId, predicate.anchorMessageId));
|
|
172
238
|
if (!anchor) return null;
|
|
173
239
|
const threshold =
|
|
174
240
|
predicate.similarityThreshold ?? DEFAULT_SEMANTIC_MATCH_THRESHOLD;
|
|
@@ -302,7 +368,7 @@ export const matchOrganize = async (
|
|
|
302
368
|
|
|
303
369
|
try {
|
|
304
370
|
const semanticIds = await matchSemantic(
|
|
305
|
-
deps
|
|
371
|
+
deps,
|
|
306
372
|
accountConfigId,
|
|
307
373
|
predicate,
|
|
308
374
|
limit,
|
|
@@ -343,6 +409,7 @@ const buildSemanticFromEnv = (): OrganizeSemanticDeps => {
|
|
|
343
409
|
buildAnchor: (accountConfigId, anchorMessageId) =>
|
|
344
410
|
buildMessageAnchor({ store }, { accountConfigId, anchorMessageId }),
|
|
345
411
|
vectorStore: store,
|
|
412
|
+
embed: async (text) => (await embedder.embed([text]))[0],
|
|
346
413
|
};
|
|
347
414
|
return cachedSemantic;
|
|
348
415
|
};
|
|
@@ -404,6 +471,7 @@ export const buildOrganizeMatchDeps = (
|
|
|
404
471
|
): OrganizeMatchDeps => ({
|
|
405
472
|
semantic: buildSemanticFromEnv,
|
|
406
473
|
listAccountFilterMessages: listAccountFilterMessagesFromClient(client),
|
|
474
|
+
filterAnchors: client.filterAnchor,
|
|
407
475
|
});
|
|
408
476
|
|
|
409
477
|
/**
|
|
@@ -433,6 +501,13 @@ export const buildOrganizeMoveService = (
|
|
|
433
501
|
export interface ApplyOrganizeDeps {
|
|
434
502
|
client: RemitClient;
|
|
435
503
|
moveService?: PlacementMoveService;
|
|
504
|
+
/**
|
|
505
|
+
* The same matcher deps {@link matchOrganize} uses — reused here only for
|
|
506
|
+
* their vector-store/embedder access, to compare a candidate message
|
|
507
|
+
* against a *different* filter's persisted anchor when checking exclusive-
|
|
508
|
+
* move precedence (RFC 039 Decision 2, reader #350).
|
|
509
|
+
*/
|
|
510
|
+
match: OrganizeMatchDeps;
|
|
436
511
|
}
|
|
437
512
|
|
|
438
513
|
export interface ApplyOrganizeResult {
|
|
@@ -440,6 +515,160 @@ export interface ApplyOrganizeResult {
|
|
|
440
515
|
failed: number;
|
|
441
516
|
}
|
|
442
517
|
|
|
518
|
+
/**
|
|
519
|
+
* The vector-free literal-match projection of one already-stored message,
|
|
520
|
+
* read from its ThreadMessage row — the same fields and the same fidelity
|
|
521
|
+
* tradeoff {@link listAccountFilterMessagesFromClient} accepts for the
|
|
522
|
+
* back-apply predicate's own literal clauses (full-fidelity From/Subject/
|
|
523
|
+
* ListId, empty body text, so a `HasWords` clause on a *different* filter
|
|
524
|
+
* cannot be proven to currently match here). `undefined` when the message no
|
|
525
|
+
* longer exists.
|
|
526
|
+
*/
|
|
527
|
+
const findFilterMessageForPrecedence = async (
|
|
528
|
+
client: Pick<RemitClient, "threadMessage">,
|
|
529
|
+
accountConfigId: string,
|
|
530
|
+
messageId: string,
|
|
531
|
+
): Promise<FilterMessage | undefined> => {
|
|
532
|
+
const row = await client.threadMessage
|
|
533
|
+
.get(accountConfigId, messageId)
|
|
534
|
+
.catch((error: unknown) => {
|
|
535
|
+
if (error instanceof NotFoundError) return undefined;
|
|
536
|
+
throw error;
|
|
537
|
+
});
|
|
538
|
+
if (!row) return undefined;
|
|
539
|
+
return {
|
|
540
|
+
from: row.fromEmail ?? "",
|
|
541
|
+
fromName: row.fromName ?? "",
|
|
542
|
+
subject: row.subject ?? "",
|
|
543
|
+
text: "",
|
|
544
|
+
listId: row.listId ?? "",
|
|
545
|
+
};
|
|
546
|
+
};
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* Whether one *other* Active filter with a move action currently matches this
|
|
550
|
+
* message — mirrors `FilterPipeline.filterMatches` (mailbox-service
|
|
551
|
+
* filters/pipeline.ts) exactly: literal clauses first, then, for a filter
|
|
552
|
+
* with a semantic anchor, its own persisted `FilterAnchor` compared against
|
|
553
|
+
* the candidate's embedding. A stale/incompatible anchor on the *other*
|
|
554
|
+
* filter is isolated to that filter (skipped, not thrown) — the same
|
|
555
|
+
* resilience `filterMatches` gives index-time matching, so one bad anchor
|
|
556
|
+
* elsewhere never breaks this back-apply's move.
|
|
557
|
+
*/
|
|
558
|
+
const filterCurrentlyMatches = async (
|
|
559
|
+
filterAnchorService: Pick<IFilterAnchorRepository, "get">,
|
|
560
|
+
accountConfigId: string,
|
|
561
|
+
filter: FilterItem,
|
|
562
|
+
msg: FilterMessage,
|
|
563
|
+
embed: () => Promise<number[] | null>,
|
|
564
|
+
): Promise<boolean> => {
|
|
565
|
+
if (!literalClausesMatch(filter.literalClauses, filter.matchOperator, msg)) {
|
|
566
|
+
return false;
|
|
567
|
+
}
|
|
568
|
+
if (!filter.hasAnchor) {
|
|
569
|
+
return filter.literalClauses.length > 0;
|
|
570
|
+
}
|
|
571
|
+
const anchor = await filterAnchorService.get(
|
|
572
|
+
accountConfigId,
|
|
573
|
+
filter.filterId,
|
|
574
|
+
);
|
|
575
|
+
if (!anchor) return false;
|
|
576
|
+
const vector = await embed();
|
|
577
|
+
if (!vector) return false;
|
|
578
|
+
try {
|
|
579
|
+
return (
|
|
580
|
+
cosineSimilarity(vector, anchor.anchorEmbedding) >=
|
|
581
|
+
DEFAULT_SEMANTIC_MATCH_THRESHOLD
|
|
582
|
+
);
|
|
583
|
+
} catch {
|
|
584
|
+
return false;
|
|
585
|
+
}
|
|
586
|
+
};
|
|
587
|
+
|
|
588
|
+
/**
|
|
589
|
+
* The filter that currently wins this message's exclusive move, evaluated
|
|
590
|
+
* fresh against every Active filter with a move action — the same
|
|
591
|
+
* cross-filter arbitration index-time matching runs
|
|
592
|
+
* (`FilterPipeline`/`selectMoveWinner`), never the back-apply job's own
|
|
593
|
+
* snapshotted predicate (RFC 039 Decision 2, reader #350). This is what lets
|
|
594
|
+
* back-apply defer to a filter created or edited *after* the job was
|
|
595
|
+
* requested. `movers` is fetched once per {@link applyOrganize} call, not
|
|
596
|
+
* once per message — a back-apply pass runs in one short burst, so the
|
|
597
|
+
* account's Active filter set does not need re-reading per message.
|
|
598
|
+
*
|
|
599
|
+
* Returns `undefined` — meaning "nothing else contests this move" — whenever
|
|
600
|
+
* there are no other Active mover filters, or the message's own projection
|
|
601
|
+
* cannot be built (deleted between match and apply). Both are the common
|
|
602
|
+
* case and the safe default: proceed with the requested move exactly as
|
|
603
|
+
* before this check existed.
|
|
604
|
+
*/
|
|
605
|
+
const findCurrentMoveWinner = async (
|
|
606
|
+
deps: {
|
|
607
|
+
client: Pick<RemitClient, "threadMessage" | "filterAnchor">;
|
|
608
|
+
match: OrganizeMatchDeps;
|
|
609
|
+
},
|
|
610
|
+
movers: readonly FilterItem[],
|
|
611
|
+
accountConfigId: string,
|
|
612
|
+
messageId: string,
|
|
613
|
+
): Promise<FilterItem | undefined> => {
|
|
614
|
+
if (movers.length === 0) return undefined;
|
|
615
|
+
const msg = await findFilterMessageForPrecedence(
|
|
616
|
+
deps.client,
|
|
617
|
+
accountConfigId,
|
|
618
|
+
messageId,
|
|
619
|
+
);
|
|
620
|
+
if (!msg) return undefined;
|
|
621
|
+
|
|
622
|
+
let messageEmbedding: number[] | null | undefined;
|
|
623
|
+
const embed = async (): Promise<number[] | null> => {
|
|
624
|
+
if (messageEmbedding !== undefined) return messageEmbedding;
|
|
625
|
+
try {
|
|
626
|
+
messageEmbedding = await deps.match.semantic().embed(buildMatchText(msg));
|
|
627
|
+
} catch (error) {
|
|
628
|
+
if (!noteSemanticCapabilityAbsence(error)) throw error;
|
|
629
|
+
messageEmbedding = null;
|
|
630
|
+
}
|
|
631
|
+
return messageEmbedding;
|
|
632
|
+
};
|
|
633
|
+
|
|
634
|
+
const matched: FilterItem[] = [];
|
|
635
|
+
for (const filter of movers) {
|
|
636
|
+
const isMatch = await filterCurrentlyMatches(
|
|
637
|
+
deps.client.filterAnchor,
|
|
638
|
+
accountConfigId,
|
|
639
|
+
filter,
|
|
640
|
+
msg,
|
|
641
|
+
embed,
|
|
642
|
+
);
|
|
643
|
+
if (isMatch) matched.push(filter);
|
|
644
|
+
}
|
|
645
|
+
return selectMoveWinner(matched);
|
|
646
|
+
};
|
|
647
|
+
|
|
648
|
+
/**
|
|
649
|
+
* Every currently-Active filter with a move action, its lazy Temporary-expiry
|
|
650
|
+
* check already applied — the fixed candidate set {@link findCurrentMoveWinner}
|
|
651
|
+
* arbitrates against for every message in this pass. Empty (and read exactly
|
|
652
|
+
* once) when no move action was requested, so a label-only back-apply never
|
|
653
|
+
* pays for this at all.
|
|
654
|
+
*/
|
|
655
|
+
const listCurrentMovers = async (
|
|
656
|
+
client: Pick<RemitClient, "filter">,
|
|
657
|
+
accountConfigId: string,
|
|
658
|
+
): Promise<FilterItem[]> => {
|
|
659
|
+
const active = await client.filter.listByAccountAndState(
|
|
660
|
+
accountConfigId,
|
|
661
|
+
FilterState.Active,
|
|
662
|
+
);
|
|
663
|
+
const movers: FilterItem[] = [];
|
|
664
|
+
for (const filter of active) {
|
|
665
|
+
if (filter.actionMailboxId === NO_ACTION) continue;
|
|
666
|
+
const usable = await client.filter.refreshExpiry(filter);
|
|
667
|
+
if (usable.state === FilterState.Active) movers.push(usable);
|
|
668
|
+
}
|
|
669
|
+
return movers;
|
|
670
|
+
};
|
|
671
|
+
|
|
443
672
|
/**
|
|
444
673
|
* Apply the back-apply action to every matched message, reusing the index-time
|
|
445
674
|
* apply plumbing: an idempotent MessageLabel upsert (additive) with
|
|
@@ -447,6 +676,13 @@ export interface ApplyOrganizeResult {
|
|
|
447
676
|
* exactly like a hand-applied label (RFC 034 Decision 3.3) — and an idempotent
|
|
448
677
|
* folder move (exclusive). One poisoned message never fails the batch; it is
|
|
449
678
|
* counted as failed and the pass continues.
|
|
679
|
+
*
|
|
680
|
+
* Before an exclusive move, resolves whether a *different* Active filter
|
|
681
|
+
* currently wins that message (RFC 039 Decision 2, reader #350): if so, the
|
|
682
|
+
* move is skipped for that message — the label action above, if requested,
|
|
683
|
+
* still applies, since labels are additive and always safe. A message with no
|
|
684
|
+
* contesting filter, or whose own back-applied filter is still the current
|
|
685
|
+
* winner, moves exactly as before this check existed.
|
|
450
686
|
*/
|
|
451
687
|
export const applyOrganize = async (
|
|
452
688
|
deps: ApplyOrganizeDeps,
|
|
@@ -454,12 +690,16 @@ export const applyOrganize = async (
|
|
|
454
690
|
messageIds: readonly string[],
|
|
455
691
|
predicate: OrganizePredicate,
|
|
456
692
|
): Promise<ApplyOrganizeResult> => {
|
|
457
|
-
const { client, moveService } = deps;
|
|
693
|
+
const { client, moveService, match } = deps;
|
|
458
694
|
const applyLabel =
|
|
459
695
|
predicate.actionLabelId !== NO_ACTION && predicate.actionLabelId !== "";
|
|
460
696
|
const applyMove =
|
|
461
697
|
predicate.actionMailboxId !== NO_ACTION && predicate.actionMailboxId !== "";
|
|
462
698
|
|
|
699
|
+
const movers = applyMove
|
|
700
|
+
? await listCurrentMovers(client, accountConfigId)
|
|
701
|
+
: [];
|
|
702
|
+
|
|
463
703
|
const applyToMessage = async (messageId: string): Promise<void> => {
|
|
464
704
|
if (applyLabel) {
|
|
465
705
|
await client.messageLabel.apply({
|
|
@@ -469,6 +709,18 @@ export const applyOrganize = async (
|
|
|
469
709
|
});
|
|
470
710
|
}
|
|
471
711
|
if (applyMove) {
|
|
712
|
+
const winner = await findCurrentMoveWinner(
|
|
713
|
+
{ client, match },
|
|
714
|
+
movers,
|
|
715
|
+
accountConfigId,
|
|
716
|
+
messageId,
|
|
717
|
+
);
|
|
718
|
+
if (winner && winner.actionMailboxId !== predicate.actionMailboxId) {
|
|
719
|
+
// A more-recently-changed filter currently claims this message's
|
|
720
|
+
// move (RFC 034 Decision 3.2) — defer to it. The label above, if
|
|
721
|
+
// requested, has already applied; only the move is suppressed.
|
|
722
|
+
return;
|
|
723
|
+
}
|
|
472
724
|
if (!moveService) {
|
|
473
725
|
// An exclusive move was requested but this caller wired no move
|
|
474
726
|
// service. Never silently pretend it applied — surface it as a
|