@remit/mailbox-service 0.0.89 → 0.0.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -1
- package/src/filters/filter-config.test.ts +229 -0
- package/src/filters/filter-config.ts +69 -0
- package/src/filters/message-embedder.test.ts +39 -0
- package/src/filters/message-embedder.ts +33 -0
- package/src/filters/pipeline.ts +28 -1
- package/src/index.ts +1 -0
- package/src/message-move-empty-trash.test.ts +42 -7
- package/src/message-move.ts +14 -1
- package/src/placement-settled.test.ts +46 -1
- package/src/placement-settled.ts +17 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remit/mailbox-service",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.91",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "src/index.ts",
|
|
6
6
|
"types": "src/index.ts",
|
|
@@ -9,6 +9,10 @@
|
|
|
9
9
|
"types": "./src/index.ts",
|
|
10
10
|
"default": "./src/index.ts"
|
|
11
11
|
},
|
|
12
|
+
"./filter-config": {
|
|
13
|
+
"types": "./src/filters/filter-config.ts",
|
|
14
|
+
"default": "./src/filters/filter-config.ts"
|
|
15
|
+
},
|
|
12
16
|
"./body-sync-queue": {
|
|
13
17
|
"types": "./src/body-sync-queue.ts",
|
|
14
18
|
"default": "./src/body-sync-queue.ts"
|
|
@@ -42,6 +46,7 @@
|
|
|
42
46
|
"@remit/data-ports": "*",
|
|
43
47
|
"@remit/domain-enums": "*",
|
|
44
48
|
"@remit/mail-oauth-service": "*",
|
|
49
|
+
"@remit/search-service": "*",
|
|
45
50
|
"@remit/secrets-service": "*",
|
|
46
51
|
"@remit/sqs-client": "*",
|
|
47
52
|
"@remit/storage-service": "*",
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { describe, it } from "node:test";
|
|
3
|
+
import type {
|
|
4
|
+
FilterItem,
|
|
5
|
+
IFilterAnchorRepository,
|
|
6
|
+
IFilterRepository,
|
|
7
|
+
IMessageLabelRepository,
|
|
8
|
+
} from "@remit/data-ports";
|
|
9
|
+
import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
|
|
10
|
+
import { _resetEmbeddingCapabilityForTest } from "@remit/search-service/capability";
|
|
11
|
+
import type { PlacementMoveService } from "../placement-move.js";
|
|
12
|
+
import { buildFilterConfig, type FilterConfigDeps } from "./filter-config.js";
|
|
13
|
+
import { type FilterMessage, NO_ACTION } from "./match.js";
|
|
14
|
+
import { _resetMessageEmbedderForTest } from "./message-embedder.js";
|
|
15
|
+
import { FilterPipeline, type MessageEmbedder } from "./pipeline.js";
|
|
16
|
+
|
|
17
|
+
const anchorOnlyFilter = (destinationMailboxId: string): FilterItem =>
|
|
18
|
+
({
|
|
19
|
+
filterId: "flt-semantic",
|
|
20
|
+
accountConfigId: "cfg-1",
|
|
21
|
+
name: "receipts",
|
|
22
|
+
scope: "Standing",
|
|
23
|
+
state: FilterState.Active,
|
|
24
|
+
hasAnchor: true,
|
|
25
|
+
ruleChangedAt: 1,
|
|
26
|
+
matchOperator: FilterMatchOperator.Or,
|
|
27
|
+
literalClauses: [],
|
|
28
|
+
actionLabelId: NO_ACTION,
|
|
29
|
+
actionMailboxId: destinationMailboxId,
|
|
30
|
+
createdAt: 1,
|
|
31
|
+
updatedAt: 1,
|
|
32
|
+
}) as unknown as FilterItem;
|
|
33
|
+
|
|
34
|
+
const CURRENT_MODEL = "test-model@3";
|
|
35
|
+
|
|
36
|
+
const repos = (filter: FilterItem): FilterConfigDeps => ({
|
|
37
|
+
filterService: {
|
|
38
|
+
listByAccountAndState: async () => [filter],
|
|
39
|
+
refreshExpiry: async (f: FilterItem) => f,
|
|
40
|
+
} as unknown as IFilterRepository,
|
|
41
|
+
filterAnchorService: {
|
|
42
|
+
get: async () => ({
|
|
43
|
+
anchorEmbedding: [1, 0, 0],
|
|
44
|
+
anchorEmbeddingId: CURRENT_MODEL,
|
|
45
|
+
}),
|
|
46
|
+
} as unknown as IFilterAnchorRepository,
|
|
47
|
+
messageLabelService: {} as unknown as IMessageLabelRepository,
|
|
48
|
+
placementMoveService: {} as unknown as PlacementMoveService,
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
const message: FilterMessage = {
|
|
52
|
+
from: "billing@stripe.com",
|
|
53
|
+
fromName: "Stripe",
|
|
54
|
+
subject: "Your receipt",
|
|
55
|
+
text: "Thanks for your payment",
|
|
56
|
+
listId: "",
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
describe("buildFilterConfig", () => {
|
|
60
|
+
it("returns undefined without a placement mover — no move path, filters stay off", () => {
|
|
61
|
+
const deps = repos(anchorOnlyFilter("mbx-archive"));
|
|
62
|
+
deps.placementMoveService = undefined;
|
|
63
|
+
|
|
64
|
+
assert.equal(buildFilterConfig(deps), undefined);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it("wires an embedder into the config", () => {
|
|
68
|
+
const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
|
|
69
|
+
|
|
70
|
+
assert.ok(config);
|
|
71
|
+
assert.ok(config.embedder);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it("lights up a semantic anchor-only filter on the body-sync pass", async () => {
|
|
75
|
+
let embedCalls = 0;
|
|
76
|
+
const embedder: MessageEmbedder = {
|
|
77
|
+
embed: async () => {
|
|
78
|
+
embedCalls += 1;
|
|
79
|
+
return [1, 0, 0];
|
|
80
|
+
},
|
|
81
|
+
embeddingId: CURRENT_MODEL,
|
|
82
|
+
};
|
|
83
|
+
const config = buildFilterConfig(
|
|
84
|
+
repos(anchorOnlyFilter("mbx-archive")),
|
|
85
|
+
embedder,
|
|
86
|
+
);
|
|
87
|
+
assert.ok(config);
|
|
88
|
+
|
|
89
|
+
const decision = await new FilterPipeline(config, {
|
|
90
|
+
info: () => {},
|
|
91
|
+
}).evaluate("cfg-1", "m-1", message);
|
|
92
|
+
|
|
93
|
+
assert.deepEqual(decision.move, {
|
|
94
|
+
destinationMailboxId: "mbx-archive",
|
|
95
|
+
filterId: "flt-semantic",
|
|
96
|
+
});
|
|
97
|
+
assert.equal(embedCalls, 1);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it("does not match when the message embedding diverges from the anchor", async () => {
|
|
101
|
+
const embedder: MessageEmbedder = {
|
|
102
|
+
embed: async () => [0, 1, 0],
|
|
103
|
+
embeddingId: CURRENT_MODEL,
|
|
104
|
+
};
|
|
105
|
+
const config = buildFilterConfig(
|
|
106
|
+
repos(anchorOnlyFilter("mbx-archive")),
|
|
107
|
+
embedder,
|
|
108
|
+
);
|
|
109
|
+
assert.ok(config);
|
|
110
|
+
|
|
111
|
+
const decision = await new FilterPipeline(config, {
|
|
112
|
+
info: () => {},
|
|
113
|
+
}).evaluate("cfg-1", "m-2", message);
|
|
114
|
+
|
|
115
|
+
assert.equal(decision.move, undefined);
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
// An instance with semantic search off (issue #1068). `off` selects an embedder
|
|
120
|
+
// that throws by design, and the throw lands in the pipeline's per-filter catch:
|
|
121
|
+
// one error-level `filter_anchor_match_failed` for every semantic filter on
|
|
122
|
+
// every message it syncs. The pipeline already has a designed skip for a
|
|
123
|
+
// missing embedder, so the config passes none.
|
|
124
|
+
describe("buildFilterConfig with semantic search off", () => {
|
|
125
|
+
const withProviderOff = async (body: () => Promise<void>): Promise<void> => {
|
|
126
|
+
const previous = process.env.SEARCH_EMBEDDING_PROVIDER;
|
|
127
|
+
process.env.SEARCH_EMBEDDING_PROVIDER = "off";
|
|
128
|
+
_resetMessageEmbedderForTest();
|
|
129
|
+
try {
|
|
130
|
+
await body();
|
|
131
|
+
} finally {
|
|
132
|
+
if (previous === undefined) {
|
|
133
|
+
delete process.env.SEARCH_EMBEDDING_PROVIDER;
|
|
134
|
+
} else {
|
|
135
|
+
process.env.SEARCH_EMBEDDING_PROVIDER = previous;
|
|
136
|
+
}
|
|
137
|
+
_resetMessageEmbedderForTest();
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
it("wires no embedder", async () => {
|
|
142
|
+
await withProviderOff(async () => {
|
|
143
|
+
const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
|
|
144
|
+
|
|
145
|
+
assert.ok(config);
|
|
146
|
+
assert.equal(config.embedder, undefined);
|
|
147
|
+
});
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
it("skips the semantic filter without logging an error", async () => {
|
|
151
|
+
await withProviderOff(async () => {
|
|
152
|
+
const config = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
|
|
153
|
+
assert.ok(config);
|
|
154
|
+
|
|
155
|
+
const errors: unknown[] = [];
|
|
156
|
+
const decision = await new FilterPipeline(config, {
|
|
157
|
+
info: () => {},
|
|
158
|
+
debug: () => {},
|
|
159
|
+
warn: () => {},
|
|
160
|
+
error: (fields) => errors.push(fields.alert),
|
|
161
|
+
}).evaluate("cfg-1", "m-off", message);
|
|
162
|
+
|
|
163
|
+
assert.deepEqual(decision.labels, []);
|
|
164
|
+
assert.equal(decision.move, undefined);
|
|
165
|
+
assert.deepEqual(errors, []);
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
// The self-host `local` profile asks for the Transformers.js embedder, but the
|
|
171
|
+
// backend and imap-worker images ship without `@huggingface/transformers`
|
|
172
|
+
// (npm-scripts/docker-bundle.mjs). The first embed throws ERR_MODULE_NOT_FOUND,
|
|
173
|
+
// which is the same capability absence `off` states and must take the same
|
|
174
|
+
// designed skip — not an error-level `filter_anchor_match_failed` per semantic
|
|
175
|
+
// filter per message on a user-facing GET.
|
|
176
|
+
describe("buildFilterConfig when this deployment carries no embedder", () => {
|
|
177
|
+
const moduleNotFound = (): Error =>
|
|
178
|
+
Object.assign(
|
|
179
|
+
new Error("Cannot find package '@huggingface/transformers'"),
|
|
180
|
+
{ code: "ERR_MODULE_NOT_FOUND" },
|
|
181
|
+
);
|
|
182
|
+
|
|
183
|
+
it("skips the semantic filter, logs no error, and stops wiring an embedder", async () => {
|
|
184
|
+
const previous = process.env.SEARCH_EMBEDDING_PROVIDER;
|
|
185
|
+
process.env.SEARCH_EMBEDDING_PROVIDER = "local";
|
|
186
|
+
_resetMessageEmbedderForTest();
|
|
187
|
+
_resetEmbeddingCapabilityForTest();
|
|
188
|
+
|
|
189
|
+
try {
|
|
190
|
+
const unavailable: MessageEmbedder = {
|
|
191
|
+
embed: async () => {
|
|
192
|
+
throw moduleNotFound();
|
|
193
|
+
},
|
|
194
|
+
embeddingId: CURRENT_MODEL,
|
|
195
|
+
};
|
|
196
|
+
const config = buildFilterConfig(
|
|
197
|
+
repos(anchorOnlyFilter("mbx-archive")),
|
|
198
|
+
unavailable,
|
|
199
|
+
);
|
|
200
|
+
assert.ok(config);
|
|
201
|
+
|
|
202
|
+
const errors: unknown[] = [];
|
|
203
|
+
const decision = await new FilterPipeline(config, {
|
|
204
|
+
info: () => {},
|
|
205
|
+
debug: () => {},
|
|
206
|
+
warn: () => {},
|
|
207
|
+
error: (fields) => errors.push(fields.alert),
|
|
208
|
+
}).evaluate("cfg-1", "m-local", message);
|
|
209
|
+
|
|
210
|
+
assert.deepEqual(decision.labels, []);
|
|
211
|
+
assert.equal(decision.move, undefined);
|
|
212
|
+
assert.deepEqual(errors, []);
|
|
213
|
+
|
|
214
|
+
// Probed once, not per filter or per message: every later config on this
|
|
215
|
+
// process wires no embedder at all, so nothing else attempts the embed.
|
|
216
|
+
const next = buildFilterConfig(repos(anchorOnlyFilter("mbx-archive")));
|
|
217
|
+
assert.ok(next);
|
|
218
|
+
assert.equal(next.embedder, undefined);
|
|
219
|
+
} finally {
|
|
220
|
+
if (previous === undefined) {
|
|
221
|
+
delete process.env.SEARCH_EMBEDDING_PROVIDER;
|
|
222
|
+
} else {
|
|
223
|
+
process.env.SEARCH_EMBEDDING_PROVIDER = previous;
|
|
224
|
+
}
|
|
225
|
+
_resetMessageEmbedderForTest();
|
|
226
|
+
_resetEmbeddingCapabilityForTest();
|
|
227
|
+
}
|
|
228
|
+
});
|
|
229
|
+
});
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
IFilterAnchorRepository,
|
|
3
|
+
IFilterRepository,
|
|
4
|
+
IMessageLabelRepository,
|
|
5
|
+
} from "@remit/data-ports";
|
|
6
|
+
import { isEmbeddingCapabilityUnavailable } from "@remit/search-service/capability";
|
|
7
|
+
import {
|
|
8
|
+
EMBEDDING_PROVIDER_OFF,
|
|
9
|
+
readEmbeddingProviderFromEnv,
|
|
10
|
+
} from "@remit/search-service/from-env";
|
|
11
|
+
import type { PlacementMoveService } from "../placement-move.js";
|
|
12
|
+
import { getMessageEmbedder } from "./message-embedder.js";
|
|
13
|
+
import type { FilterConfig, MessageEmbedder } from "./pipeline.js";
|
|
14
|
+
|
|
15
|
+
export interface FilterConfigDeps {
|
|
16
|
+
filterService: IFilterRepository;
|
|
17
|
+
filterAnchorService: IFilterAnchorRepository;
|
|
18
|
+
messageLabelService: IMessageLabelRepository;
|
|
19
|
+
placementMoveService?: PlacementMoveService;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The env-selected embedder, or none on an instance that cannot embed.
|
|
24
|
+
*
|
|
25
|
+
* `off` embeds nothing by design, and its embedder says so by throwing
|
|
26
|
+
* (EmbeddingDisabledError). The pipeline has a designed skip for a *missing*
|
|
27
|
+
* embedder — semantic filters are passed over and logged at debug — but a
|
|
28
|
+
* present embedder that throws lands in the per-filter catch instead, which is
|
|
29
|
+
* an error-level `filter_anchor_match_failed` for every semantic filter on
|
|
30
|
+
* every synced message. Same outcome, a log nobody can read.
|
|
31
|
+
*
|
|
32
|
+
* `local` on the self-host profiles reaches the same place by a different road:
|
|
33
|
+
* the backend and imap-worker images ship without `@huggingface/transformers`,
|
|
34
|
+
* so the first embed throws ERR_MODULE_NOT_FOUND. The pipeline classifies that
|
|
35
|
+
* one throw through `@remit/search-service/capability` and remembers it, and
|
|
36
|
+
* from then on the config here wires no embedder either — one probe per process,
|
|
37
|
+
* not one per filter per message.
|
|
38
|
+
*/
|
|
39
|
+
const embedderFromEnv = (): MessageEmbedder | undefined =>
|
|
40
|
+
readEmbeddingProviderFromEnv() === EMBEDDING_PROVIDER_OFF ||
|
|
41
|
+
isEmbeddingCapabilityUnavailable()
|
|
42
|
+
? undefined
|
|
43
|
+
: getMessageEmbedder();
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Assemble the filter config a body-materializing pass runs (RFC 034), shared by
|
|
47
|
+
* the imap-worker's sync path and the backend's read-path backfill so both embed
|
|
48
|
+
* under the same model that produced the anchors — a semantic (anchor-only)
|
|
49
|
+
* filter is evaluated on either path instead of silently skipped.
|
|
50
|
+
*
|
|
51
|
+
* Absent the placement mover there is no move path, so filters stay off — a
|
|
52
|
+
* matched filter's actions reuse the same enqueue plumbing the placement mover
|
|
53
|
+
* owns.
|
|
54
|
+
*/
|
|
55
|
+
export const buildFilterConfig = (
|
|
56
|
+
deps: FilterConfigDeps,
|
|
57
|
+
embedder?: MessageEmbedder,
|
|
58
|
+
): FilterConfig | undefined => {
|
|
59
|
+
const { placementMoveService } = deps;
|
|
60
|
+
if (!placementMoveService) return undefined;
|
|
61
|
+
const resolved = embedder ?? embedderFromEnv();
|
|
62
|
+
return {
|
|
63
|
+
filterService: deps.filterService,
|
|
64
|
+
filterAnchorService: deps.filterAnchorService,
|
|
65
|
+
messageLabelService: deps.messageLabelService,
|
|
66
|
+
placementMoveService,
|
|
67
|
+
...(resolved ? { embedder: resolved } : {}),
|
|
68
|
+
};
|
|
69
|
+
};
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { afterEach, describe, it } from "node:test";
|
|
3
|
+
import {
|
|
4
|
+
_resetMessageEmbedderForTest,
|
|
5
|
+
getMessageEmbedder,
|
|
6
|
+
} from "./message-embedder.js";
|
|
7
|
+
|
|
8
|
+
describe("getMessageEmbedder", () => {
|
|
9
|
+
afterEach(() => {
|
|
10
|
+
_resetMessageEmbedderForTest();
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it("adapts the batch embedding service into a single-text message embedder", async () => {
|
|
14
|
+
const vector = await getMessageEmbedder().embed("invoice from stripe");
|
|
15
|
+
|
|
16
|
+
assert.ok(Array.isArray(vector));
|
|
17
|
+
assert.ok(vector.length > 0);
|
|
18
|
+
assert.ok(vector.every((value) => typeof value === "number"));
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it("embeds the same text deterministically", async () => {
|
|
22
|
+
const embedder = getMessageEmbedder();
|
|
23
|
+
const a = await embedder.embed("your monthly receipt");
|
|
24
|
+
const b = await embedder.embed("your monthly receipt");
|
|
25
|
+
|
|
26
|
+
assert.deepEqual(a, b);
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
it("memoizes the embedder across calls", () => {
|
|
30
|
+
assert.strictEqual(getMessageEmbedder(), getMessageEmbedder());
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it("rebuilds after a reset", () => {
|
|
34
|
+
const first = getMessageEmbedder();
|
|
35
|
+
_resetMessageEmbedderForTest();
|
|
36
|
+
|
|
37
|
+
assert.notStrictEqual(getMessageEmbedder(), first);
|
|
38
|
+
});
|
|
39
|
+
});
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { buildEmbeddingServiceFromEnv } from "@remit/search-service/from-env";
|
|
2
|
+
import type { MessageEmbedder } from "./pipeline.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Adapt the batch {@link EmbeddingService} the rest of the stack composes from
|
|
6
|
+
* env into the single-text {@link MessageEmbedder} the filter pipeline needs: one
|
|
7
|
+
* incoming message becomes one message-level vector to compare against a filter's
|
|
8
|
+
* persisted anchor (RFC 034 Decision 2.1/2.3).
|
|
9
|
+
*
|
|
10
|
+
* The embedder is selected by the same `SEARCH_EMBEDDING_*` env the search-index
|
|
11
|
+
* worker reads, so every filter pass embeds under the identical model the anchors
|
|
12
|
+
* were built with. The instance is memoized across warm invocations so a local
|
|
13
|
+
* Transformers.js model loads once per container rather than per event.
|
|
14
|
+
*/
|
|
15
|
+
let cached: MessageEmbedder | undefined;
|
|
16
|
+
|
|
17
|
+
export const getMessageEmbedder = (): MessageEmbedder => {
|
|
18
|
+
if (cached) return cached;
|
|
19
|
+
const service = buildEmbeddingServiceFromEnv();
|
|
20
|
+
cached = {
|
|
21
|
+
embed: async (text: string): Promise<number[]> => {
|
|
22
|
+
const [vector] = await service.embed([text]);
|
|
23
|
+
return vector;
|
|
24
|
+
},
|
|
25
|
+
embeddingId: service.embeddingId,
|
|
26
|
+
};
|
|
27
|
+
return cached;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
/** Reset the singleton — test use only. */
|
|
31
|
+
export const _resetMessageEmbedderForTest = (): void => {
|
|
32
|
+
cached = undefined;
|
|
33
|
+
};
|
package/src/filters/pipeline.ts
CHANGED
|
@@ -6,6 +6,11 @@ import type {
|
|
|
6
6
|
IMessageLabelRepository,
|
|
7
7
|
} from "@remit/data-ports";
|
|
8
8
|
import { FilterState } from "@remit/domain-enums";
|
|
9
|
+
import {
|
|
10
|
+
isEmbeddingCapabilityAbsence,
|
|
11
|
+
isEmbeddingCapabilityUnavailable,
|
|
12
|
+
recordEmbeddingCapabilityAbsence,
|
|
13
|
+
} from "@remit/search-service/capability";
|
|
9
14
|
import type { PlacementMoveService } from "../placement-move.js";
|
|
10
15
|
import {
|
|
11
16
|
type AnchorEmbedder,
|
|
@@ -131,7 +136,29 @@ export class FilterPipeline {
|
|
|
131
136
|
messageEmbedding = null;
|
|
132
137
|
return null;
|
|
133
138
|
}
|
|
134
|
-
messageEmbedding = await embedder
|
|
139
|
+
messageEmbedding = await embedder
|
|
140
|
+
.embed(buildMatchText(msg))
|
|
141
|
+
.catch((error: unknown) => {
|
|
142
|
+
// This deployment carries no embedding pipeline at all — the
|
|
143
|
+
// container images ship without `@huggingface/transformers`. That is
|
|
144
|
+
// the same absence `SEARCH_EMBEDDING_PROVIDER=off` states, so it takes
|
|
145
|
+
// the same designed skip rather than an error-level
|
|
146
|
+
// `filter_anchor_match_failed` per semantic filter per message. The
|
|
147
|
+
// absence is remembered process-wide, so this is the only embed that
|
|
148
|
+
// pays for it: `buildFilterConfig` wires no embedder afterwards.
|
|
149
|
+
if (!isEmbeddingCapabilityAbsence(error)) throw error;
|
|
150
|
+
if (!isEmbeddingCapabilityUnavailable()) {
|
|
151
|
+
this.log.warn?.(
|
|
152
|
+
{
|
|
153
|
+
accountConfigId,
|
|
154
|
+
error: inspect(error),
|
|
155
|
+
},
|
|
156
|
+
"No embedding pipeline in this deployment; semantic filters are skipped (literal filters still run)",
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
recordEmbeddingCapabilityAbsence(error);
|
|
160
|
+
return null;
|
|
161
|
+
});
|
|
135
162
|
return messageEmbedding;
|
|
136
163
|
};
|
|
137
164
|
|
package/src/index.ts
CHANGED
|
@@ -46,8 +46,27 @@ const proposedDeletedFolder: RoleResolution<TrashMailbox> = {
|
|
|
46
46
|
interface TrashMessage {
|
|
47
47
|
messageId: string;
|
|
48
48
|
syncStatus: string;
|
|
49
|
+
status?: string;
|
|
50
|
+
mailboxId?: string;
|
|
51
|
+
uid?: number;
|
|
52
|
+
originalMailboxId?: string;
|
|
53
|
+
originalUid?: number;
|
|
49
54
|
}
|
|
50
55
|
|
|
56
|
+
/**
|
|
57
|
+
* A row mid-move into Trash: the folder is already written, the uid is still
|
|
58
|
+
* the inbox's, and only `status: moving` says so.
|
|
59
|
+
*/
|
|
60
|
+
const movingIntoTrash = (messageId: string, uid: number): TrashMessage => ({
|
|
61
|
+
messageId,
|
|
62
|
+
syncStatus: "pending",
|
|
63
|
+
status: "moving",
|
|
64
|
+
mailboxId: REAL_TRASH,
|
|
65
|
+
uid,
|
|
66
|
+
originalMailboxId: "mbx-inbox",
|
|
67
|
+
originalUid: uid,
|
|
68
|
+
});
|
|
69
|
+
|
|
51
70
|
interface EnqueuedEvent {
|
|
52
71
|
type: string;
|
|
53
72
|
schemaVersion?: number;
|
|
@@ -194,22 +213,38 @@ describe("MessageMoveService.emptyTrash", () => {
|
|
|
194
213
|
assert.equal(deletedCount, 3);
|
|
195
214
|
});
|
|
196
215
|
|
|
197
|
-
it("marks and counts a message whose
|
|
198
|
-
//
|
|
199
|
-
//
|
|
200
|
-
//
|
|
201
|
-
//
|
|
216
|
+
it("marks and counts a message whose sync is merely pending", async () => {
|
|
217
|
+
// `syncStatus: pending` is where every freshly synced inbound row sits
|
|
218
|
+
// forever; it says nothing about where the message is. The user saw it in
|
|
219
|
+
// Trash and asked for the folder to be emptied, and skipping it would
|
|
220
|
+
// report a number the folder contradicts.
|
|
202
221
|
const { service, markedDeleting } = buildWorld(appointedTrash, [
|
|
203
222
|
{ messageId: "settled-1", syncStatus: "synced" },
|
|
204
|
-
{ messageId: "
|
|
223
|
+
{ messageId: "pending-1", syncStatus: "pending" },
|
|
205
224
|
]);
|
|
206
225
|
|
|
207
226
|
const { deletedCount } = await service.emptyTrash(ACCOUNT_CONFIG, ACCOUNT);
|
|
208
227
|
|
|
209
|
-
assert.deepEqual(markedDeleting, ["settled-1", "
|
|
228
|
+
assert.deepEqual(markedDeleting, ["settled-1", "pending-1"]);
|
|
210
229
|
assert.equal(deletedCount, 2);
|
|
211
230
|
});
|
|
212
231
|
|
|
232
|
+
it("leaves a row whose move into Trash has not settled unmarked and uncounted", async () => {
|
|
233
|
+
// Issue #1217. Marking it `deleting` overwrites the `moving` its own
|
|
234
|
+
// MESSAGE_MOVE reads to decide there is still work to do, so that move
|
|
235
|
+
// returns without touching IMAP and the worker's expunge then binds the
|
|
236
|
+
// inbox's uid against whatever Trash really holds at it.
|
|
237
|
+
const { service, markedDeleting } = buildWorld(appointedTrash, [
|
|
238
|
+
{ messageId: "settled-1", syncStatus: "synced" },
|
|
239
|
+
movingIntoTrash("moving-1", 10),
|
|
240
|
+
]);
|
|
241
|
+
|
|
242
|
+
const { deletedCount } = await service.emptyTrash(ACCOUNT_CONFIG, ACCOUNT);
|
|
243
|
+
|
|
244
|
+
assert.deepEqual(markedDeleting, ["settled-1"]);
|
|
245
|
+
assert.equal(deletedCount, 1);
|
|
246
|
+
});
|
|
247
|
+
|
|
213
248
|
it("reports the same count when pressed twice before the worker runs", async () => {
|
|
214
249
|
// The rows it marked are still in that folder, so N stays true and the
|
|
215
250
|
// re-mark is idempotent. Reporting 0 the second time while still
|
package/src/message-move.ts
CHANGED
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
} from "@remit/domain-enums";
|
|
35
35
|
import { createQueueProducer } from "@remit/sqs-client/producer";
|
|
36
36
|
import {
|
|
37
|
+
bindsForeignUid,
|
|
37
38
|
type PlacementBinding,
|
|
38
39
|
placementBindingOf,
|
|
39
40
|
waitForPlacementToSettle,
|
|
@@ -917,10 +918,22 @@ export class MessageMoveService {
|
|
|
917
918
|
trashMailbox.mailboxId,
|
|
918
919
|
);
|
|
919
920
|
|
|
920
|
-
const
|
|
921
|
+
const rows = await this.messageService.listAllByMailbox(
|
|
921
922
|
trashMailbox.mailboxId,
|
|
922
923
|
);
|
|
923
924
|
|
|
925
|
+
// Reconciles, never waits (imap-mutations R2). A row whose move into
|
|
926
|
+
// Trash has not settled names this folder while still carrying the SOURCE
|
|
927
|
+
// folder's uid, and marking it `deleting` overwrites the only marker that
|
|
928
|
+
// says so: its own MESSAGE_MOVE then reads the row as settled and returns
|
|
929
|
+
// without moving anything, and the worker's expunge binds that borrowed
|
|
930
|
+
// uid against whatever Trash really holds at it. Left as it stands, that
|
|
931
|
+
// move runs ahead of this expunge in the account's FIFO group and settles
|
|
932
|
+
// the row, so the message is in Trash by the time the sweep reaches it and
|
|
933
|
+
// goes with the rest of this same press — only the count below is one
|
|
934
|
+
// short of what the sweep ends up removing.
|
|
935
|
+
const messages = rows.filter((message) => !bindsForeignUid(message));
|
|
936
|
+
|
|
924
937
|
// Mark all as deleting locally
|
|
925
938
|
for (const message of messages) {
|
|
926
939
|
await this.messageService.update(message.messageId, {
|
|
@@ -2,7 +2,7 @@ import assert from "node:assert/strict";
|
|
|
2
2
|
import { describe, test } from "node:test";
|
|
3
3
|
import type { MessageItem } from "@remit/data-ports";
|
|
4
4
|
import { MessageStatus, MessageSyncStatus } from "@remit/domain-enums";
|
|
5
|
-
import { placementBindingOf } from "./placement-settled.js";
|
|
5
|
+
import { carriesForeignUid, placementBindingOf } from "./placement-settled.js";
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* Pins `placementBindingOf` directly. It is the guard #1098/#1143 depend on and
|
|
@@ -70,3 +70,48 @@ describe("placementBindingOf", () => {
|
|
|
70
70
|
);
|
|
71
71
|
});
|
|
72
72
|
});
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The same question with the `status` gate off, for callers that resolve a uid
|
|
76
|
+
* off a row some other operation may have re-marked on its way past (#1217).
|
|
77
|
+
*/
|
|
78
|
+
describe("carriesForeignUid", () => {
|
|
79
|
+
test("holds whatever an operation has since written to `status`", () => {
|
|
80
|
+
for (const status of [
|
|
81
|
+
MessageStatus.moving,
|
|
82
|
+
MessageStatus.deleting,
|
|
83
|
+
MessageStatus.active,
|
|
84
|
+
]) {
|
|
85
|
+
assert.equal(carriesForeignUid(row({ status })), true, status);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("is false once `updateUid` settles the move", () => {
|
|
90
|
+
// The settle writes the destination's own uid and drops `originalUid` in
|
|
91
|
+
// the same statement, so a destination counter that hands back the
|
|
92
|
+
// source's number cannot leave the row reading foreign forever.
|
|
93
|
+
assert.equal(
|
|
94
|
+
carriesForeignUid(
|
|
95
|
+
row({
|
|
96
|
+
status: MessageStatus.active,
|
|
97
|
+
syncStatus: MessageSyncStatus.synced,
|
|
98
|
+
originalUid: undefined,
|
|
99
|
+
}),
|
|
100
|
+
),
|
|
101
|
+
false,
|
|
102
|
+
);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test("is false for a row that was never moved", () => {
|
|
106
|
+
assert.equal(
|
|
107
|
+
carriesForeignUid(
|
|
108
|
+
row({ originalUid: undefined, originalMailboxId: undefined }),
|
|
109
|
+
),
|
|
110
|
+
false,
|
|
111
|
+
);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("is false once a restore points the row back at that folder", () => {
|
|
115
|
+
assert.equal(carriesForeignUid(row({ mailboxId: "mbx-src" })), false);
|
|
116
|
+
});
|
|
117
|
+
});
|
package/src/placement-settled.ts
CHANGED
|
@@ -29,19 +29,33 @@ export const isPlacementUnsettled = (
|
|
|
29
29
|
* never moved, so its uid names nothing but itself — a freshly copied row is
|
|
30
30
|
* `moving` with `uid: 0` until COPYUID lands, which is a uid that is not ready,
|
|
31
31
|
* not a uid that names somebody else's message.
|
|
32
|
+
*
|
|
33
|
+
* Answered without reference to `status`, because `status` is not what clears
|
|
34
|
+
* it: `updateUid` does, dropping `originalUid` in the same statement that
|
|
35
|
+
* writes the destination's own, and a restore does by pointing `mailboxId`
|
|
36
|
+
* back at `originalMailboxId`. An operation that rewrites `status` on its way
|
|
37
|
+
* past — Empty Trash marking a whole folder `deleting` — leaves the pair
|
|
38
|
+
* untouched, so anything that resolves a uid off a row it did not settle
|
|
39
|
+
* itself asks this rather than the binding below (issue #1217).
|
|
32
40
|
*/
|
|
33
|
-
export const
|
|
41
|
+
export const carriesForeignUid = (
|
|
34
42
|
message: Pick<
|
|
35
43
|
MessageItem,
|
|
36
|
-
"
|
|
44
|
+
"mailboxId" | "uid" | "originalMailboxId" | "originalUid"
|
|
37
45
|
>,
|
|
38
46
|
): boolean =>
|
|
39
|
-
isPlacementUnsettled(message) &&
|
|
40
47
|
message.originalUid !== undefined &&
|
|
41
48
|
message.originalUid === message.uid &&
|
|
42
49
|
message.originalMailboxId !== undefined &&
|
|
43
50
|
message.originalMailboxId !== message.mailboxId;
|
|
44
51
|
|
|
52
|
+
export const bindsForeignUid = (
|
|
53
|
+
message: Pick<
|
|
54
|
+
MessageItem,
|
|
55
|
+
"status" | "mailboxId" | "uid" | "originalMailboxId" | "originalUid"
|
|
56
|
+
>,
|
|
57
|
+
): boolean => isPlacementUnsettled(message) && carriesForeignUid(message);
|
|
58
|
+
|
|
45
59
|
/**
|
|
46
60
|
* `in_flight` — a mover is still working on the row, so waiting resolves it.
|
|
47
61
|
* `abandoned` — the last attempt failed (`syncStatus: failed`) and the row is
|