@remit/backend 0.0.107 → 0.0.109
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/derive/enrichThreadRows.test.ts +33 -4
- package/src/derive/enrichThreadRows.ts +9 -2
- package/src/derive/filterThreadCriteria.test.ts +2 -0
- package/src/handlers/message.test.ts +7 -7
- package/src/handlers/message.ts +15 -11
- package/src/service/create-remit-client.ts +36 -29
- package/src/service/read-path-filter-config.test.ts +243 -0
- package/src/service/semantic-capability.ts +18 -18
package/package.json
CHANGED
|
@@ -9,7 +9,11 @@ import type {
|
|
|
9
9
|
} from "@remit/data-ports";
|
|
10
10
|
import { deriveAddressId } from "@remit/data-ports/id";
|
|
11
11
|
import { hasAbandonedDelete } from "@remit/data-ports/message-settlement";
|
|
12
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
MessageMutation,
|
|
14
|
+
MessageStatus,
|
|
15
|
+
MessageSyncStatus,
|
|
16
|
+
} from "@remit/domain-enums";
|
|
13
17
|
import { type EnrichClient, enrichThreadRows } from "./enrichThreadRows.js";
|
|
14
18
|
|
|
15
19
|
const threadRow = (
|
|
@@ -273,22 +277,45 @@ describe("enrichThreadRows — the mutation pair", () => {
|
|
|
273
277
|
return result;
|
|
274
278
|
};
|
|
275
279
|
|
|
276
|
-
test("projects the pair an abandoned delete leaves behind", async () => {
|
|
280
|
+
test("projects the pair an abandoned delete leaves behind, and what it gave up on", async () => {
|
|
277
281
|
const result = await project({
|
|
278
282
|
status: MessageStatus.active,
|
|
279
|
-
syncStatus: MessageSyncStatus.
|
|
283
|
+
syncStatus: MessageSyncStatus.abandoned,
|
|
284
|
+
abandonedMutation: MessageMutation.delete,
|
|
280
285
|
});
|
|
281
286
|
assert.equal(result?.status, MessageStatus.active);
|
|
282
|
-
assert.equal(result?.syncStatus, MessageSyncStatus.
|
|
287
|
+
assert.equal(result?.syncStatus, MessageSyncStatus.abandoned);
|
|
288
|
+
assert.equal(result?.abandonedMutation, MessageMutation.delete);
|
|
283
289
|
assert.equal(
|
|
284
290
|
hasAbandonedDelete({
|
|
285
291
|
status: result?.status,
|
|
286
292
|
syncStatus: result?.syncStatus,
|
|
293
|
+
abandonedMutation: result?.abandonedMutation,
|
|
287
294
|
}),
|
|
288
295
|
true,
|
|
289
296
|
);
|
|
290
297
|
});
|
|
291
298
|
|
|
299
|
+
// Issue #1229. Both give-ups leave the same `status`/`syncStatus` pair, so
|
|
300
|
+
// the projection has to carry the third field or the client cannot tell a
|
|
301
|
+
// move that handed back from a delete that was refused.
|
|
302
|
+
test("projects a move that gave up as a move, never as a delete", async () => {
|
|
303
|
+
const result = await project({
|
|
304
|
+
status: MessageStatus.active,
|
|
305
|
+
syncStatus: MessageSyncStatus.abandoned,
|
|
306
|
+
abandonedMutation: MessageMutation.move,
|
|
307
|
+
});
|
|
308
|
+
assert.equal(result?.abandonedMutation, MessageMutation.move);
|
|
309
|
+
assert.equal(
|
|
310
|
+
hasAbandonedDelete({
|
|
311
|
+
status: result?.status,
|
|
312
|
+
syncStatus: result?.syncStatus,
|
|
313
|
+
abandonedMutation: result?.abandonedMutation,
|
|
314
|
+
}),
|
|
315
|
+
false,
|
|
316
|
+
);
|
|
317
|
+
});
|
|
318
|
+
|
|
292
319
|
test("projects a move mid-retry without calling it a give-up", async () => {
|
|
293
320
|
const result = await project({
|
|
294
321
|
status: MessageStatus.moving,
|
|
@@ -300,6 +327,7 @@ describe("enrichThreadRows — the mutation pair", () => {
|
|
|
300
327
|
hasAbandonedDelete({
|
|
301
328
|
status: result?.status,
|
|
302
329
|
syncStatus: result?.syncStatus,
|
|
330
|
+
abandonedMutation: result?.abandonedMutation,
|
|
303
331
|
}),
|
|
304
332
|
false,
|
|
305
333
|
);
|
|
@@ -324,6 +352,7 @@ describe("enrichThreadRows — the mutation pair", () => {
|
|
|
324
352
|
hasAbandonedDelete({
|
|
325
353
|
status: result?.status,
|
|
326
354
|
syncStatus: result?.syncStatus,
|
|
355
|
+
abandonedMutation: result?.abandonedMutation,
|
|
327
356
|
}),
|
|
328
357
|
false,
|
|
329
358
|
);
|
|
@@ -8,6 +8,7 @@ import type {
|
|
|
8
8
|
} from "@remit/data-ports";
|
|
9
9
|
import { deriveAddressId } from "@remit/data-ports/id";
|
|
10
10
|
import {
|
|
11
|
+
MessageMutation,
|
|
11
12
|
MessageStatus,
|
|
12
13
|
MessageSyncStatus,
|
|
13
14
|
SenderTrust,
|
|
@@ -63,6 +64,7 @@ const toResponse = (item: ThreadMessageItem): ThreadMessageResponse => ({
|
|
|
63
64
|
muted: false,
|
|
64
65
|
status: MessageStatus.active,
|
|
65
66
|
syncStatus: MessageSyncStatus.pending,
|
|
67
|
+
abandonedMutation: MessageMutation.none,
|
|
66
68
|
});
|
|
67
69
|
|
|
68
70
|
/**
|
|
@@ -110,7 +112,8 @@ export const planBatchFetch = (rows: ThreadMessageItem[]): BatchPlan => {
|
|
|
110
112
|
* Enrich a page of ThreadMessage rows with `senderTrust` and `muted` (both
|
|
111
113
|
* derived from the From Address's flags map), `authenticity`, `autoMoved` and
|
|
112
114
|
* `spamReport`, and the `status`/`syncStatus` pair an IMAP mutation leaves
|
|
113
|
-
* behind
|
|
115
|
+
* behind together with the mutation an abandoned one names (all projected
|
|
116
|
+
* straight from the Message row — no ThreadMessage
|
|
114
117
|
* column of their own, see `deriveAutoMoved`). `toResponse` seeds that pair
|
|
115
118
|
* with the ordinary inbound values, which the projection then overwrites; a
|
|
116
119
|
* ThreadMessage row whose Message row is missing from the batch is broken in a
|
|
@@ -189,7 +192,11 @@ export const enrichThreadRows = async (
|
|
|
189
192
|
const settlementByMessageId = new Map(
|
|
190
193
|
messages.map((m) => [
|
|
191
194
|
m.messageId,
|
|
192
|
-
{
|
|
195
|
+
{
|
|
196
|
+
status: m.status,
|
|
197
|
+
syncStatus: m.syncStatus,
|
|
198
|
+
abandonedMutation: m.abandonedMutation,
|
|
199
|
+
},
|
|
193
200
|
]),
|
|
194
201
|
);
|
|
195
202
|
const trustByAddressId = new Map(
|
|
@@ -3,6 +3,7 @@ import { describe, it } from "node:test";
|
|
|
3
3
|
import type { ThreadMessageResponse } from "@remit/api-openapi-types";
|
|
4
4
|
import {
|
|
5
5
|
MessageCategory,
|
|
6
|
+
MessageMutation,
|
|
6
7
|
MessageStatus,
|
|
7
8
|
MessageSyncStatus,
|
|
8
9
|
SenderTrust,
|
|
@@ -38,6 +39,7 @@ const row = (
|
|
|
38
39
|
muted: false,
|
|
39
40
|
status: MessageStatus.active,
|
|
40
41
|
syncStatus: MessageSyncStatus.pending,
|
|
42
|
+
abandonedMutation: MessageMutation.none,
|
|
41
43
|
...overrides,
|
|
42
44
|
});
|
|
43
45
|
|
|
@@ -43,7 +43,8 @@ describe("settleSpamReportBulk", () => {
|
|
|
43
43
|
});
|
|
44
44
|
|
|
45
45
|
// A refusal the move gate raises used to collapse to the generic retry copy,
|
|
46
|
-
// which is a dead end
|
|
46
|
+
// which is a dead end (#665). One reason reaches it now — a mutation still in
|
|
47
|
+
// flight — and it words the wait rather than the server's sentence.
|
|
47
48
|
it("words a placement refusal from its reason, never from the server's sentence", async () => {
|
|
48
49
|
const outcome = await settleSpamReportBulk(
|
|
49
50
|
["msg-1", "msg-2"],
|
|
@@ -52,16 +53,15 @@ describe("settleSpamReportBulk", () => {
|
|
|
52
53
|
`Message ${messageId} was not acted on: its folder and uid do not name the same message`,
|
|
53
54
|
"acc-1",
|
|
54
55
|
messageId,
|
|
55
|
-
|
|
56
|
+
"in_flight",
|
|
56
57
|
);
|
|
57
58
|
},
|
|
58
59
|
);
|
|
59
60
|
|
|
60
|
-
const
|
|
61
|
-
assert.
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
for (const reason of [inFlight, unverified]) {
|
|
61
|
+
const reasons = outcome.failures?.map((f) => f.reason) ?? [];
|
|
62
|
+
assert.equal(reasons.length, 2);
|
|
63
|
+
for (const reason of reasons) {
|
|
64
|
+
assert.match(reason ?? "", /try again in a moment/i);
|
|
65
65
|
assert.notEqual(reason, GENERIC_FAILURE_REASON);
|
|
66
66
|
assert.doesNotMatch(reason ?? "", /folder and uid/);
|
|
67
67
|
assert.doesNotMatch(reason ?? "", /acc-1/);
|
package/src/handlers/message.ts
CHANGED
|
@@ -368,17 +368,15 @@ const isDesignedFailure = (reason: unknown): reason is Error =>
|
|
|
368
368
|
|
|
369
369
|
/**
|
|
370
370
|
* The placement refusal is the one designed outcome whose own message is not
|
|
371
|
-
* the text to ship: it names a uuid and no remedy
|
|
372
|
-
*
|
|
373
|
-
*
|
|
374
|
-
*
|
|
371
|
+
* the text to ship: it names a uuid and no remedy. One remedy answers it, and
|
|
372
|
+
* only one refusal reaches it — a mutation still in flight, which settles on
|
|
373
|
+
* its own (imap-mutations R3). Worded here, the way the delete path words the
|
|
374
|
+
* same refusal for the client.
|
|
375
375
|
*/
|
|
376
376
|
const placementRefusalReason = (
|
|
377
|
-
|
|
377
|
+
_error: MessagePlacementUnsettledError,
|
|
378
378
|
): string =>
|
|
379
|
-
|
|
380
|
-
? "An earlier move of this message never finished, so where it sits is unknown. Sync the folder, then report it again."
|
|
381
|
-
: "This message is still being moved on the mail server. Try again in a moment.";
|
|
379
|
+
"This message is still being moved on the mail server. Try again in a moment.";
|
|
382
380
|
|
|
383
381
|
const failureReason = (reason: unknown): string => {
|
|
384
382
|
if (!isDesignedFailure(reason)) return GENERIC_FAILURE_REASON;
|
|
@@ -479,6 +477,7 @@ export const MessageOperations: Record<
|
|
|
479
477
|
authenticity: message.authenticity,
|
|
480
478
|
status: message.status,
|
|
481
479
|
syncStatus: message.syncStatus,
|
|
480
|
+
abandonedMutation: message.abandonedMutation,
|
|
482
481
|
...(autoMoved ? { autoMoved } : {}),
|
|
483
482
|
...(labels.length > 0 ? { labels } : {}),
|
|
484
483
|
...(message.spamReport ? { spamReport: message.spamReport } : {}),
|
|
@@ -899,7 +898,7 @@ export const MessageBulkOperations: Record<
|
|
|
899
898
|
);
|
|
900
899
|
|
|
901
900
|
// MessageMoveService handles: Message + ThreadMessage updates + SQS events
|
|
902
|
-
await client.messageMove.deleteMessages(
|
|
901
|
+
const { refusedMessageIds } = await client.messageMove.deleteMessages(
|
|
903
902
|
accountConfigId,
|
|
904
903
|
messageIds,
|
|
905
904
|
accountId,
|
|
@@ -908,9 +907,14 @@ export const MessageBulkOperations: Record<
|
|
|
908
907
|
},
|
|
909
908
|
);
|
|
910
909
|
|
|
910
|
+
// A row whose placement changed under the batch is reported, not dropped.
|
|
911
|
+
// The response already carries per-row counts, which is the honest shape
|
|
912
|
+
// for a batch that partly applied — a 409 over it would deny the deletes
|
|
913
|
+
// that did happen, and silence leaves the client's optimistic removal to
|
|
914
|
+
// reappear with nothing said (imap-mutations R3).
|
|
911
915
|
return {
|
|
912
|
-
successCount: messageIds.length,
|
|
913
|
-
failureCount:
|
|
916
|
+
successCount: messageIds.length - refusedMessageIds.length,
|
|
917
|
+
failureCount: refusedMessageIds.length,
|
|
914
918
|
};
|
|
915
919
|
},
|
|
916
920
|
|
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
BodySyncQueueService,
|
|
38
38
|
BodySyncService,
|
|
39
39
|
createConnection,
|
|
40
|
+
type FilterConfig,
|
|
40
41
|
FlagPushService,
|
|
41
42
|
FlagQueueService,
|
|
42
43
|
type IImapConnection,
|
|
@@ -47,6 +48,7 @@ import {
|
|
|
47
48
|
PlacementMoveService,
|
|
48
49
|
SpamReportService,
|
|
49
50
|
} from "@remit/mailbox-service";
|
|
51
|
+
import { buildFilterConfig } from "@remit/mailbox-service/filter-config";
|
|
50
52
|
import { createSearchService, type SearchService } from "@remit/search-service";
|
|
51
53
|
import {
|
|
52
54
|
buildEmbeddingServiceFromEnv,
|
|
@@ -339,6 +341,39 @@ export const buildSharedDeps = (): RemitClientSharedDeps => {
|
|
|
339
341
|
};
|
|
340
342
|
};
|
|
341
343
|
|
|
344
|
+
// The read-path body backfill (describeMessage / getRawMessage →
|
|
345
|
+
// fetchAndGetBody) materializes a body that was never sync-synced, and must run
|
|
346
|
+
// the account's standing filters the same way the imap-worker's sync path does.
|
|
347
|
+
// Without this, a message a user opens before background body-sync is classified
|
|
348
|
+
// but never filter-evaluated — and once its body is stored the filter-capable
|
|
349
|
+
// sync path skips it, so an explicit standing filter silently never fires (issue
|
|
350
|
+
// #223). Only the explicit user-rule half is wired here; heuristic classifier
|
|
351
|
+
// placement stays on the sync path. It shares `buildFilterConfig` with
|
|
352
|
+
// `sync-message-body.ts`, so both paths embed a candidate message under the same
|
|
353
|
+
// env-selected model that produced the anchors and a semantic (anchor-only)
|
|
354
|
+
// filter fires on either (issue #298). Over the message-management queue's
|
|
355
|
+
// local-first PlacementMoveService; filters stay off when that queue is unset.
|
|
356
|
+
export const buildReadPathFilterConfig = (
|
|
357
|
+
repositories: RemitClientRepositories,
|
|
358
|
+
): FilterConfig | undefined => {
|
|
359
|
+
const placementMoveQueueUrl = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
|
|
360
|
+
return buildFilterConfig({
|
|
361
|
+
filterService: repositories.filter,
|
|
362
|
+
filterAnchorService: repositories.filterAnchor,
|
|
363
|
+
messageLabelService: repositories.messageLabel,
|
|
364
|
+
placementMoveService: placementMoveQueueUrl
|
|
365
|
+
? new PlacementMoveService({
|
|
366
|
+
messageService: repositories.message,
|
|
367
|
+
threadMessageService: repositories.threadMessage,
|
|
368
|
+
markerService: repositories.placementMove,
|
|
369
|
+
addressService: repositories.address,
|
|
370
|
+
mailboxSpecialUseService: repositories.mailboxSpecialUse,
|
|
371
|
+
sqsQueueUrl: placementMoveQueueUrl,
|
|
372
|
+
})
|
|
373
|
+
: undefined,
|
|
374
|
+
});
|
|
375
|
+
};
|
|
376
|
+
|
|
342
377
|
// Backend-neutral composition root: given repositories (from any data-ports
|
|
343
378
|
// implementation) and the shared services, wire the domain and queue services
|
|
344
379
|
// and assemble a RemitClient. Imports neither ElectroDB nor Drizzle — the
|
|
@@ -354,35 +389,7 @@ export const createRemitClient = (deps: RemitClientDeps): RemitClient => {
|
|
|
354
389
|
bodySyncQueue,
|
|
355
390
|
} = deps;
|
|
356
391
|
|
|
357
|
-
|
|
358
|
-
// fetchAndGetBody) materializes a body that was never sync-synced, and must
|
|
359
|
-
// run the account's standing filters the same way the imap-worker's sync path
|
|
360
|
-
// does. Without this, a message a user opens before background body-sync is
|
|
361
|
-
// classified but never filter-evaluated — and once its body is stored the
|
|
362
|
-
// filter-capable sync path skips it, so an explicit standing filter silently
|
|
363
|
-
// never fires (issue #223). Only the explicit user-rule half is wired here;
|
|
364
|
-
// heuristic classifier placement stays on the sync path. Wired like
|
|
365
|
-
// `sync-message-body.ts`, over the message-management queue's local-first
|
|
366
|
-
// PlacementMoveService; filters stay off when that queue is unset.
|
|
367
|
-
const placementMoveQueueUrl = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
|
|
368
|
-
const placementMoveService = placementMoveQueueUrl
|
|
369
|
-
? new PlacementMoveService({
|
|
370
|
-
messageService: repositories.message,
|
|
371
|
-
threadMessageService: repositories.threadMessage,
|
|
372
|
-
markerService: repositories.placementMove,
|
|
373
|
-
addressService: repositories.address,
|
|
374
|
-
mailboxSpecialUseService: repositories.mailboxSpecialUse,
|
|
375
|
-
sqsQueueUrl: placementMoveQueueUrl,
|
|
376
|
-
})
|
|
377
|
-
: undefined;
|
|
378
|
-
const filterConfig = placementMoveService
|
|
379
|
-
? {
|
|
380
|
-
filterService: repositories.filter,
|
|
381
|
-
filterAnchorService: repositories.filterAnchor,
|
|
382
|
-
messageLabelService: repositories.messageLabel,
|
|
383
|
-
placementMoveService,
|
|
384
|
-
}
|
|
385
|
-
: undefined;
|
|
392
|
+
const filterConfig = buildReadPathFilterConfig(repositories);
|
|
386
393
|
|
|
387
394
|
const flagPushService = new FlagPushService({
|
|
388
395
|
markerService: repositories.flagPush,
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The read-path body backfill must run a semantic (anchor-only) standing filter,
|
|
3
|
+
* which needs the env-selected embedder wired into the filter config
|
|
4
|
+
* `createRemitClient` hands its BodySyncService (issue #298).
|
|
5
|
+
*
|
|
6
|
+
* These drive the composed client — `createRemitClient(...).bodySync` — rather
|
|
7
|
+
* than `buildReadPathFilterConfig` alone, so replacing that call with an inline
|
|
8
|
+
* filter-config literal (which carries no embedder) fails here.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import assert from "node:assert/strict";
|
|
12
|
+
import { afterEach, beforeEach, describe, it } from "node:test";
|
|
13
|
+
import type {
|
|
14
|
+
CreateMessageLabelInput,
|
|
15
|
+
FilterAnchorItem,
|
|
16
|
+
FilterItem,
|
|
17
|
+
UpdateMessageInput,
|
|
18
|
+
} from "@remit/data-ports";
|
|
19
|
+
import { NotFoundError } from "@remit/data-ports/errors";
|
|
20
|
+
import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
|
|
21
|
+
import {
|
|
22
|
+
buildMatchText,
|
|
23
|
+
type FilterMessage,
|
|
24
|
+
type IImapConnection,
|
|
25
|
+
NO_ACTION,
|
|
26
|
+
} from "@remit/mailbox-service";
|
|
27
|
+
import type { SearchService } from "@remit/search-service";
|
|
28
|
+
import { buildEmbeddingServiceFromEnv } from "@remit/search-service/from-env";
|
|
29
|
+
import type { SecretsService } from "@remit/secrets-service";
|
|
30
|
+
import type { StorageService } from "@remit/storage-service";
|
|
31
|
+
import {
|
|
32
|
+
buildReadPathFilterConfig,
|
|
33
|
+
createRemitClient,
|
|
34
|
+
type RemitClientRepositories,
|
|
35
|
+
} from "./create-remit-client.js";
|
|
36
|
+
|
|
37
|
+
const MESSAGE: FilterMessage = {
|
|
38
|
+
from: "billing@stripe.com",
|
|
39
|
+
fromName: "Stripe",
|
|
40
|
+
subject: "Your receipt",
|
|
41
|
+
text: "Thanks for your payment",
|
|
42
|
+
listId: "",
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
const eml = (message: FilterMessage): Buffer =>
|
|
46
|
+
Buffer.from(
|
|
47
|
+
[
|
|
48
|
+
`From: ${message.fromName} <${message.from}>`,
|
|
49
|
+
"To: me@example.com",
|
|
50
|
+
`Subject: ${message.subject}`,
|
|
51
|
+
"Content-Type: text/plain",
|
|
52
|
+
"",
|
|
53
|
+
message.text,
|
|
54
|
+
].join("\r\n"),
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Anchor-only: no literal clauses, so it matches on the embedding alone. Its
|
|
59
|
+
* action is a label, which the fake MessageLabel repository below records —
|
|
60
|
+
* a move would run the real PlacementMoveService against SQS.
|
|
61
|
+
*/
|
|
62
|
+
const anchorOnlyFilter: FilterItem = {
|
|
63
|
+
filterId: "flt-semantic",
|
|
64
|
+
accountConfigId: "cfg-1",
|
|
65
|
+
name: "receipts",
|
|
66
|
+
scope: "Standing",
|
|
67
|
+
state: FilterState.Active,
|
|
68
|
+
hasAnchor: true,
|
|
69
|
+
ruleChangedAt: 1,
|
|
70
|
+
matchOperator: FilterMatchOperator.Or,
|
|
71
|
+
literalClauses: [],
|
|
72
|
+
actionLabelId: "lbl-receipts",
|
|
73
|
+
actionMailboxId: NO_ACTION,
|
|
74
|
+
createdAt: 1,
|
|
75
|
+
updatedAt: 1,
|
|
76
|
+
} as unknown as FilterItem;
|
|
77
|
+
|
|
78
|
+
// The read path must embed a candidate message under the same model that wrote
|
|
79
|
+
// the anchors, so an anchor written from this text matches it exactly.
|
|
80
|
+
const anchorFor = async (message: FilterMessage): Promise<FilterAnchorItem> => {
|
|
81
|
+
const embedding = buildEmbeddingServiceFromEnv();
|
|
82
|
+
const [anchorEmbedding] = await embedding.embed([buildMatchText(message)]);
|
|
83
|
+
return {
|
|
84
|
+
accountConfigId: "cfg-1",
|
|
85
|
+
filterId: "flt-semantic",
|
|
86
|
+
anchorEmbedding,
|
|
87
|
+
anchorEmbeddingId: embedding.embeddingId,
|
|
88
|
+
anchorSourceText: buildMatchText(message),
|
|
89
|
+
anchorMessageId: "m-anchor",
|
|
90
|
+
} as unknown as FilterAnchorItem;
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
const repositoriesWithAnchor = (
|
|
94
|
+
anchor: FilterAnchorItem,
|
|
95
|
+
applied: CreateMessageLabelInput[] = [],
|
|
96
|
+
): RemitClientRepositories =>
|
|
97
|
+
({
|
|
98
|
+
filter: {
|
|
99
|
+
listByAccountAndState: async () => [anchorOnlyFilter],
|
|
100
|
+
refreshExpiry: async (filter: FilterItem) => filter,
|
|
101
|
+
},
|
|
102
|
+
filterAnchor: { get: async () => anchor },
|
|
103
|
+
messageLabel: {
|
|
104
|
+
apply: async (input: CreateMessageLabelInput) => {
|
|
105
|
+
applied.push(input);
|
|
106
|
+
},
|
|
107
|
+
},
|
|
108
|
+
message: {
|
|
109
|
+
get: async (messageId: string) => ({
|
|
110
|
+
messageId,
|
|
111
|
+
mailboxId: "mb-inbox",
|
|
112
|
+
uid: 1,
|
|
113
|
+
}),
|
|
114
|
+
update: async (_messageId: string, _input: UpdateMessageInput) => {},
|
|
115
|
+
},
|
|
116
|
+
threadMessage: {
|
|
117
|
+
findAllByMessageId: async () => [
|
|
118
|
+
{
|
|
119
|
+
threadMessageId: "tm-1",
|
|
120
|
+
sentDate: 1,
|
|
121
|
+
mailboxId: "mb-inbox",
|
|
122
|
+
isRead: false,
|
|
123
|
+
isDeleted: false,
|
|
124
|
+
hasStars: false,
|
|
125
|
+
hasAttachment: false,
|
|
126
|
+
},
|
|
127
|
+
],
|
|
128
|
+
update: async () => {},
|
|
129
|
+
},
|
|
130
|
+
address: {
|
|
131
|
+
getAddress: async () => {
|
|
132
|
+
throw new NotFoundError("Address not found");
|
|
133
|
+
},
|
|
134
|
+
incrementInboundCount: async () => {},
|
|
135
|
+
},
|
|
136
|
+
envelope: { listBodyParts: async () => [] },
|
|
137
|
+
placementMove: {},
|
|
138
|
+
mailboxSpecialUse: {},
|
|
139
|
+
}) as unknown as RemitClientRepositories;
|
|
140
|
+
|
|
141
|
+
const storage = {
|
|
142
|
+
storeMessageBody: async () => ({ uri: "s3://bodies/m-1" }),
|
|
143
|
+
storeParsedBody: async () => {},
|
|
144
|
+
listBodyParts: async () => [],
|
|
145
|
+
} as unknown as StorageService;
|
|
146
|
+
|
|
147
|
+
const readBodyThroughClient = async (
|
|
148
|
+
repositories: RemitClientRepositories,
|
|
149
|
+
message: FilterMessage,
|
|
150
|
+
): Promise<void> => {
|
|
151
|
+
const client = createRemitClient({
|
|
152
|
+
repositories,
|
|
153
|
+
storage,
|
|
154
|
+
search: {} as unknown as SearchService,
|
|
155
|
+
secrets: {} as unknown as SecretsService,
|
|
156
|
+
sqsQueueUrl: "https://sqs.eu-west-1.amazonaws.com/0/test",
|
|
157
|
+
sqsSmtpQueueUrl: "https://sqs.eu-west-1.amazonaws.com/0/test-smtp",
|
|
158
|
+
});
|
|
159
|
+
const connection = {
|
|
160
|
+
openBox: async () => {},
|
|
161
|
+
fetchMessageBody: async () => eml(message),
|
|
162
|
+
} as unknown as IImapConnection;
|
|
163
|
+
|
|
164
|
+
await client.bodySync.fetchAndGetBody(
|
|
165
|
+
"m-1",
|
|
166
|
+
"acc-1",
|
|
167
|
+
"cfg-1",
|
|
168
|
+
"INBOX",
|
|
169
|
+
async () => connection,
|
|
170
|
+
);
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
describe("createRemitClient read-path filter config", () => {
|
|
174
|
+
let previousQueueUrl: string | undefined;
|
|
175
|
+
|
|
176
|
+
beforeEach(() => {
|
|
177
|
+
previousQueueUrl = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
|
|
178
|
+
process.env.SQS_QUEUE_URL_MESSAGE_MGMT =
|
|
179
|
+
"https://sqs.eu-west-1.amazonaws.com/0/test-message-mgmt";
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
afterEach(() => {
|
|
183
|
+
if (previousQueueUrl === undefined) {
|
|
184
|
+
delete process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
process.env.SQS_QUEUE_URL_MESSAGE_MGMT = previousQueueUrl;
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
it("labels a message an anchor-only filter matches on a read-path backfill", async () => {
|
|
191
|
+
const applied: CreateMessageLabelInput[] = [];
|
|
192
|
+
|
|
193
|
+
await readBodyThroughClient(
|
|
194
|
+
repositoriesWithAnchor(await anchorFor(MESSAGE), applied),
|
|
195
|
+
MESSAGE,
|
|
196
|
+
);
|
|
197
|
+
|
|
198
|
+
assert.deepEqual(
|
|
199
|
+
applied.map(({ labelId, appliedByFilterId }) => ({
|
|
200
|
+
labelId,
|
|
201
|
+
appliedByFilterId,
|
|
202
|
+
})),
|
|
203
|
+
[{ labelId: "lbl-receipts", appliedByFilterId: "flt-semantic" }],
|
|
204
|
+
);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it("leaves a message the anchor does not match unlabelled", async () => {
|
|
208
|
+
const applied: CreateMessageLabelInput[] = [];
|
|
209
|
+
|
|
210
|
+
await readBodyThroughClient(
|
|
211
|
+
repositoriesWithAnchor(
|
|
212
|
+
await anchorFor({
|
|
213
|
+
...MESSAGE,
|
|
214
|
+
subject: "Scheduled maintenance window",
|
|
215
|
+
text: "Our service will be unavailable on Sunday",
|
|
216
|
+
}),
|
|
217
|
+
applied,
|
|
218
|
+
),
|
|
219
|
+
MESSAGE,
|
|
220
|
+
);
|
|
221
|
+
|
|
222
|
+
assert.deepEqual(applied, []);
|
|
223
|
+
});
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
describe("buildReadPathFilterConfig", () => {
|
|
227
|
+
it("keeps filters off when the message-management queue is unset", async () => {
|
|
228
|
+
const previous = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
|
|
229
|
+
delete process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
|
|
230
|
+
try {
|
|
231
|
+
assert.equal(
|
|
232
|
+
buildReadPathFilterConfig(
|
|
233
|
+
repositoriesWithAnchor(await anchorFor(MESSAGE)),
|
|
234
|
+
),
|
|
235
|
+
undefined,
|
|
236
|
+
);
|
|
237
|
+
} finally {
|
|
238
|
+
if (previous !== undefined) {
|
|
239
|
+
process.env.SQS_QUEUE_URL_MESSAGE_MGMT = previous;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
});
|
|
243
|
+
});
|
|
@@ -1,4 +1,10 @@
|
|
|
1
1
|
import { logger } from "@remit/logger-lambda";
|
|
2
|
+
import {
|
|
3
|
+
_resetEmbeddingCapabilityForTest,
|
|
4
|
+
isEmbeddingCapabilityAbsence,
|
|
5
|
+
isEmbeddingCapabilityUnavailable,
|
|
6
|
+
recordEmbeddingCapabilityAbsence,
|
|
7
|
+
} from "@remit/search-service/capability";
|
|
2
8
|
import { isSelfHostSqlBackend } from "../data-backend.js";
|
|
3
9
|
|
|
4
10
|
/**
|
|
@@ -47,23 +53,20 @@ import { isSelfHostSqlBackend } from "../data-backend.js";
|
|
|
47
53
|
* instead was considered and rejected — it would fabricate relevance scores and
|
|
48
54
|
* matched-chunk labels for the "Related" UI, and cannot serve the cross-account
|
|
49
55
|
* queries the daily brief issues.
|
|
56
|
+
*
|
|
57
|
+
* The error vocabulary and the memoized absence itself live in
|
|
58
|
+
* `@remit/search-service/capability`, shared with the filter pipeline's semantic
|
|
59
|
+
* skip: whichever path embeds first is the probe, and the other stops trying.
|
|
60
|
+
* This module adds the self-host scoping and the operator-facing log.
|
|
50
61
|
*/
|
|
51
62
|
|
|
52
|
-
const CAPABILITY_ABSENCE_CODES = new Set([
|
|
53
|
-
"ERR_MODULE_NOT_FOUND",
|
|
54
|
-
"MODULE_NOT_FOUND",
|
|
55
|
-
"ERR_DLOPEN_FAILED",
|
|
56
|
-
"ERR_EMBEDDING_MODEL_UNAVAILABLE",
|
|
57
|
-
]);
|
|
58
|
-
|
|
59
|
-
let semanticUnavailable = false;
|
|
60
|
-
|
|
61
63
|
/** Test-only reset for the memoized absence. */
|
|
62
64
|
export const _resetSemanticCapabilityForTest = (): void => {
|
|
63
|
-
|
|
65
|
+
_resetEmbeddingCapabilityForTest();
|
|
64
66
|
};
|
|
65
67
|
|
|
66
|
-
export const isSemanticSearchUnavailable = (): boolean =>
|
|
68
|
+
export const isSemanticSearchUnavailable = (): boolean =>
|
|
69
|
+
isEmbeddingCapabilityUnavailable();
|
|
67
70
|
|
|
68
71
|
/**
|
|
69
72
|
* Whether a failure is this deployment simply not carrying the semantic
|
|
@@ -71,11 +74,8 @@ export const isSemanticSearchUnavailable = (): boolean => semanticUnavailable;
|
|
|
71
74
|
* Records nothing, for the caller that must degrade one operation without
|
|
72
75
|
* disabling `/search/semantic` process-wide.
|
|
73
76
|
*/
|
|
74
|
-
export const isSemanticCapabilityAbsence = (error: unknown): boolean =>
|
|
75
|
-
|
|
76
|
-
const code = (error as { code?: unknown } | null)?.code;
|
|
77
|
-
return typeof code === "string" && CAPABILITY_ABSENCE_CODES.has(code);
|
|
78
|
-
};
|
|
77
|
+
export const isSemanticCapabilityAbsence = (error: unknown): boolean =>
|
|
78
|
+
isSelfHostSqlBackend() && isEmbeddingCapabilityAbsence(error);
|
|
79
79
|
|
|
80
80
|
/**
|
|
81
81
|
* Classify a semantic-search failure. Returns true — and remembers the
|
|
@@ -84,12 +84,12 @@ export const isSemanticCapabilityAbsence = (error: unknown): boolean => {
|
|
|
84
84
|
*/
|
|
85
85
|
export const noteSemanticCapabilityAbsence = (error: unknown): boolean => {
|
|
86
86
|
if (!isSemanticCapabilityAbsence(error)) return false;
|
|
87
|
-
if (!
|
|
87
|
+
if (!isEmbeddingCapabilityUnavailable()) {
|
|
88
88
|
logger.warn(
|
|
89
89
|
{ error: error instanceof Error ? error.message : String(error) },
|
|
90
90
|
"Semantic search pipeline unavailable in this deployment — serving empty semantic results (see deploy/vps/README.md)",
|
|
91
91
|
);
|
|
92
92
|
}
|
|
93
|
-
|
|
93
|
+
recordEmbeddingCapabilityAbsence(error);
|
|
94
94
|
return true;
|
|
95
95
|
};
|