@remit/backend 0.0.142 → 0.0.144
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dev-server/content-handler.d.ts +38 -0
- package/dist/dev-server/content-handler.js +46 -0
- package/dist/src/api.d.ts +7 -0
- package/dist/src/api.js +144 -0
- package/dist/src/auth.d.ts +30 -0
- package/dist/src/auth.js +73 -0
- package/dist/src/config/msoauth.d.ts +30 -0
- package/dist/src/config/msoauth.js +30 -0
- package/{src/data-backend.ts → dist/src/data-backend.d.ts} +2 -4
- package/dist/src/data-backend.js +12 -0
- package/{src/derive/autoMoved.ts → dist/src/derive/autoMoved.d.ts} +1 -29
- package/dist/src/derive/autoMoved.js +49 -0
- package/dist/src/derive/contentSignature.d.ts +41 -0
- package/{src/derive/contentSignature.ts → dist/src/derive/contentSignature.js} +27 -72
- package/dist/src/derive/contentUrl.d.ts +46 -0
- package/dist/src/derive/contentUrl.js +46 -0
- package/{src/derive/deriveMuted.ts → dist/src/derive/deriveMuted.d.ts} +1 -3
- package/dist/src/derive/deriveMuted.js +7 -0
- package/dist/src/derive/enrichThreadRows.d.ts +118 -0
- package/dist/src/derive/enrichThreadRows.js +151 -0
- package/dist/src/derive/filterThreadCriteria.d.ts +23 -0
- package/dist/src/derive/filterThreadCriteria.js +24 -0
- package/{src/derive/pendingMoveCounts.ts → dist/src/derive/pendingMoveCounts.d.ts} +5 -53
- package/dist/src/derive/pendingMoveCounts.js +53 -0
- package/{src/derive/senderTrust.ts → dist/src/derive/senderTrust.d.ts} +1 -9
- package/dist/src/derive/senderTrust.js +19 -0
- package/dist/src/error.d.ts +21 -0
- package/dist/src/error.js +85 -0
- package/dist/src/handlers/account-guards.d.ts +36 -0
- package/dist/src/handlers/account-guards.js +107 -0
- package/dist/src/handlers/account-oauth.d.ts +14 -0
- package/dist/src/handlers/account-oauth.js +387 -0
- package/dist/src/handlers/account-overrides.d.ts +53 -0
- package/dist/src/handlers/account-overrides.js +233 -0
- package/dist/src/handlers/account-signature.d.ts +28 -0
- package/dist/src/handlers/account-signature.js +85 -0
- package/dist/src/handlers/account.d.ts +6 -0
- package/dist/src/handlers/account.js +350 -0
- package/dist/src/handlers/address.d.ts +35 -0
- package/dist/src/handlers/address.js +141 -0
- package/dist/src/handlers/calendar-event.d.ts +70 -0
- package/dist/src/handlers/calendar-event.js +425 -0
- package/dist/src/handlers/calendar-feed.d.ts +47 -0
- package/dist/src/handlers/calendar-feed.js +192 -0
- package/dist/src/handlers/calendar-suggestion.d.ts +47 -0
- package/dist/src/handlers/calendar-suggestion.js +212 -0
- package/dist/src/handlers/calendar.d.ts +116 -0
- package/dist/src/handlers/calendar.js +330 -0
- package/dist/src/handlers/config.d.ts +43 -0
- package/dist/src/handlers/config.js +176 -0
- package/dist/src/handlers/ensure-account-config.d.ts +9 -0
- package/dist/src/handlers/ensure-account-config.js +27 -0
- package/dist/src/handlers/filter.d.ts +3 -0
- package/dist/src/handlers/filter.js +103 -0
- package/dist/src/handlers/folder-role-appointments.d.ts +68 -0
- package/dist/src/handlers/folder-role-appointments.js +168 -0
- package/dist/src/handlers/folder-role-labels.d.ts +26 -0
- package/dist/src/handlers/folder-role-labels.js +40 -0
- package/dist/src/handlers/folder-role.d.ts +9 -0
- package/dist/src/handlers/folder-role.js +40 -0
- package/dist/src/handlers/index.d.ts +2 -0
- package/dist/src/handlers/index.js +60 -0
- package/dist/src/handlers/label.d.ts +43 -0
- package/dist/src/handlers/label.js +114 -0
- package/dist/src/handlers/mailbox-bindings.d.ts +41 -0
- package/dist/src/handlers/mailbox-bindings.js +61 -0
- package/dist/src/handlers/mailbox.d.ts +23 -0
- package/dist/src/handlers/mailbox.js +219 -0
- package/dist/src/handlers/me.d.ts +2 -0
- package/dist/src/handlers/me.js +138 -0
- package/dist/src/handlers/message.d.ts +84 -0
- package/dist/src/handlers/message.js +607 -0
- package/dist/src/handlers/organize.d.ts +10 -0
- package/dist/src/handlers/organize.js +112 -0
- package/dist/src/handlers/outbox-attachment.d.ts +23 -0
- package/dist/src/handlers/outbox-attachment.js +81 -0
- package/dist/src/handlers/outbox.d.ts +4 -0
- package/dist/src/handlers/outbox.js +144 -0
- package/dist/src/handlers/sync-progress.d.ts +46 -0
- package/dist/src/handlers/sync-progress.js +63 -0
- package/dist/src/handlers/sync.d.ts +28 -0
- package/dist/src/handlers/sync.js +82 -0
- package/dist/src/handlers/system-update.d.ts +2 -0
- package/dist/src/handlers/system-update.js +172 -0
- package/dist/src/handlers/thread.d.ts +148 -0
- package/dist/src/handlers/thread.js +233 -0
- package/dist/src/handlers/unified-threads.d.ts +247 -0
- package/dist/src/handlers/unified-threads.js +409 -0
- package/{src/handlers/vip-suggestions.ts → dist/src/handlers/vip-suggestions.d.ts} +1 -13
- package/dist/src/handlers/vip-suggestions.js +16 -0
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.js +37 -0
- package/{src/json.ts → dist/src/json.d.ts} +1 -4
- package/dist/src/json.js +10 -0
- package/dist/src/jwt-auth.d.ts +18 -0
- package/dist/src/jwt-auth.js +97 -0
- package/dist/src/request-context.d.ts +9 -0
- package/dist/src/request-context.js +22 -0
- package/dist/src/request.d.ts +3 -0
- package/dist/src/request.js +7 -0
- package/dist/src/response.d.ts +27 -0
- package/dist/src/response.js +87 -0
- package/dist/src/service/account-deletion.d.ts +22 -0
- package/dist/src/service/account-deletion.js +39 -0
- package/dist/src/service/compose-sqlite.d.ts +2 -0
- package/dist/src/service/compose-sqlite.js +11 -0
- package/dist/src/service/create-remit-client.d.ts +95 -0
- package/dist/src/service/create-remit-client.js +207 -0
- package/dist/src/service/data-client.d.ts +12 -0
- package/dist/src/service/data-client.js +44 -0
- package/dist/src/service/export-identity.d.ts +3 -0
- package/dist/src/service/export-identity.js +35 -0
- package/dist/src/service/fire-and-forget.d.ts +54 -0
- package/dist/src/service/fire-and-forget.js +38 -0
- package/dist/src/service/organize.d.ts +184 -0
- package/dist/src/service/organize.js +384 -0
- package/dist/src/service/refusal.d.ts +4 -0
- package/dist/src/service/refusal.js +20 -0
- package/dist/src/service/sender-category.d.ts +51 -0
- package/{src/service/sender-category.ts → dist/src/service/sender-category.js} +40 -81
- package/dist/src/service/sqs.d.ts +1 -0
- package/{src/service/sqs.ts → dist/src/service/sqs.js} +0 -1
- package/dist/src/service/trigger-sync.d.ts +51 -0
- package/dist/src/service/trigger-sync.js +39 -0
- package/dist/src/types.d.ts +49 -0
- package/dist/src/types.js +1 -0
- package/package.json +113 -98
- package/.env.test +0 -4
- package/dev-server/content-auth.test.ts +0 -113
- package/dev-server/content-auth.ts +0 -54
- package/dev-server/content-handler.test.ts +0 -105
- package/dev-server/content-handler.ts +0 -79
- package/dev-server/content-path.test.ts +0 -37
- package/dev-server/content-path.ts +0 -27
- package/dev-server/cors.test.ts +0 -48
- package/dev-server/cors.ts +0 -25
- package/dev-server/keep-alive.test.ts +0 -24
- package/dev-server/lambda-helpers.ts +0 -69
- package/dev-server/log-output.test.ts +0 -173
- package/dev-server/oauth-callback-route.test.ts +0 -66
- package/dev-server/relational-health.test.ts +0 -83
- package/dev-server/relational-health.ts +0 -39
- package/dev-server/server.ts +0 -384
- package/dev-server/sync-age.test.ts +0 -91
- package/dev-server/sync-age.ts +0 -55
- package/dev-server/upload-handler.ts +0 -132
- package/dev-server/upload-route.test.ts +0 -196
- package/scripts/backfill-classification.ts +0 -91
- package/scripts/backfill-list-id.ts +0 -83
- package/scripts/config-save.ts +0 -128
- package/src/auth.test.ts +0 -12
- package/src/auth.ts +0 -92
- package/src/config/msoauth.test.ts +0 -48
- package/src/config/msoauth.ts +0 -48
- package/src/data-backend.test.ts +0 -23
- package/src/derive/autoMoved.test.ts +0 -104
- package/src/derive/contentSignature.test.ts +0 -161
- package/src/derive/contentUrl.test.ts +0 -172
- package/src/derive/contentUrl.ts +0 -86
- package/src/derive/deriveMuted.test.ts +0 -34
- package/src/derive/enrichThreadRows.test.ts +0 -360
- package/src/derive/enrichThreadRows.ts +0 -234
- package/src/derive/filterThreadCriteria.test.ts +0 -126
- package/src/derive/filterThreadCriteria.ts +0 -51
- package/src/derive/senderTrust.test.ts +0 -67
- package/src/error.test.ts +0 -201
- package/src/error.ts +0 -123
- package/src/handlers/account-guards.test.ts +0 -67
- package/src/handlers/account-guards.ts +0 -169
- package/src/handlers/account-oauth-callback.test.ts +0 -155
- package/src/handlers/account-oauth-consent.test.ts +0 -336
- package/src/handlers/account-oauth-start.test.ts +0 -191
- package/src/handlers/account-oauth.test.ts +0 -439
- package/src/handlers/account-oauth.ts +0 -635
- package/src/handlers/account-overrides.test.ts +0 -88
- package/src/handlers/account-overrides.ts +0 -323
- package/src/handlers/account-ownership.ts +0 -25
- package/src/handlers/account-signature.ts +0 -117
- package/src/handlers/account-synced-services.test.ts +0 -245
- package/src/handlers/account.ts +0 -552
- package/src/handlers/address.test.ts +0 -381
- package/src/handlers/address.ts +0 -200
- package/src/handlers/calendar-event.test.ts +0 -685
- package/src/handlers/calendar-event.ts +0 -738
- package/src/handlers/calendar-feed-validators.test.ts +0 -228
- package/src/handlers/calendar-feed.test.ts +0 -439
- package/src/handlers/calendar-feed.ts +0 -291
- package/src/handlers/calendar-sqlite-fixture.ts +0 -36
- package/src/handlers/calendar-suggestion.test.ts +0 -1269
- package/src/handlers/calendar-suggestion.ts +0 -425
- package/src/handlers/calendar.test.ts +0 -1080
- package/src/handlers/calendar.ts +0 -517
- package/src/handlers/config.test.ts +0 -245
- package/src/handlers/config.ts +0 -332
- package/src/handlers/ensure-account-config.ts +0 -30
- package/src/handlers/filter.test.ts +0 -644
- package/src/handlers/filter.ts +0 -518
- package/src/handlers/folder-role-appointments.test.ts +0 -438
- package/src/handlers/folder-role-appointments.ts +0 -249
- package/src/handlers/folder-role-labels.ts +0 -63
- package/src/handlers/folder-role.test.ts +0 -209
- package/src/handlers/folder-role.ts +0 -80
- package/src/handlers/index.ts +0 -83
- package/src/handlers/label.test.ts +0 -112
- package/src/handlers/label.ts +0 -210
- package/src/handlers/mailbox-bindings.ts +0 -111
- package/src/handlers/mailbox-rename-refusal.test.ts +0 -247
- package/src/handlers/mailbox-settled-target.test.ts +0 -251
- package/src/handlers/mailbox.test.ts +0 -280
- package/src/handlers/mailbox.ts +0 -581
- package/src/handlers/me.test.ts +0 -81
- package/src/handlers/me.ts +0 -203
- package/src/handlers/message-read-backfill.test.ts +0 -103
- package/src/handlers/message.test.ts +0 -131
- package/src/handlers/message.ts +0 -1131
- package/src/handlers/organize.test.ts +0 -277
- package/src/handlers/organize.ts +0 -197
- package/src/handlers/outbox-attachment.test.ts +0 -765
- package/src/handlers/outbox-attachment.ts +0 -126
- package/src/handlers/outbox-list.test.ts +0 -179
- package/src/handlers/outbox.test.ts +0 -772
- package/src/handlers/outbox.ts +0 -258
- package/src/handlers/search.test.ts +0 -161
- package/src/handlers/search.ts +0 -105
- package/src/handlers/sync-progress.test.ts +0 -205
- package/src/handlers/sync-progress.ts +0 -99
- package/src/handlers/sync.test.ts +0 -168
- package/src/handlers/sync.ts +0 -118
- package/src/handlers/system-update.test.ts +0 -497
- package/src/handlers/system-update.ts +0 -230
- package/src/handlers/thread.test.ts +0 -396
- package/src/handlers/thread.ts +0 -427
- package/src/handlers/unified-threads-mail-sources.sqlite.test.ts +0 -197
- package/src/handlers/unified-threads.test.ts +0 -785
- package/src/handlers/unified-threads.ts +0 -669
- package/src/index.test.ts +0 -83
- package/src/index.ts +0 -228
- package/src/jwt-auth.test.ts +0 -152
- package/src/jwt-auth.ts +0 -132
- package/src/log-redaction.test.ts +0 -144
- package/src/openapi-spec.test.ts +0 -118
- package/src/request-context.test.ts +0 -61
- package/src/request-context.ts +0 -33
- package/src/request-pipeline.test.ts +0 -88
- package/src/request.ts +0 -10
- package/src/response.test.ts +0 -130
- package/src/response.ts +0 -144
- package/src/service/compose-sqlite.ts +0 -98
- package/src/service/config-import.ts +0 -39
- package/src/service/create-remit-client.ts +0 -521
- package/src/service/data-client.test.ts +0 -97
- package/src/service/data-client.ts +0 -58
- package/src/service/export-identity.ts +0 -39
- package/src/service/filter.ts +0 -40
- package/src/service/fire-and-forget.test.ts +0 -126
- package/src/service/fire-and-forget.ts +0 -79
- package/src/service/localhost-env-config.test.ts +0 -36
- package/src/service/organize-queue.ts +0 -98
- package/src/service/organize.test.ts +0 -1451
- package/src/service/organize.ts +0 -1025
- package/src/service/read-path-filter-config.test.ts +0 -243
- package/src/service/semantic-capability.test.ts +0 -78
- package/src/service/semantic-capability.ts +0 -95
- package/src/service/sender-category.test.ts +0 -235
- package/src/service/trigger-sync.test.ts +0 -205
- package/src/service/trigger-sync.ts +0 -100
- package/src/stdout-capture-fixture.ts +0 -40
- package/src/types.ts +0 -271
- package/tsconfig.json +0 -7
|
@@ -1,1451 +0,0 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import { afterEach, beforeEach, describe, it } from "node:test";
|
|
3
|
-
import type {
|
|
4
|
-
CreateFilterAnchorInput,
|
|
5
|
-
FilterAnchorItem,
|
|
6
|
-
FilterItem,
|
|
7
|
-
} from "@remit/data-ports";
|
|
8
|
-
import { NotFoundError } from "@remit/data-ports/errors";
|
|
9
|
-
import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
|
|
10
|
-
import type {
|
|
11
|
-
AnchorPayload,
|
|
12
|
-
ChunkMetadata,
|
|
13
|
-
VectorRecord,
|
|
14
|
-
} from "@remit/search-service";
|
|
15
|
-
import { createMemoryVectorStore } from "@remit/search-service";
|
|
16
|
-
import type { RemitClient } from "./data-client.js";
|
|
17
|
-
import {
|
|
18
|
-
applyOrganize,
|
|
19
|
-
BODY_CONTENT_REJECTION_MESSAGE,
|
|
20
|
-
matchOrganize,
|
|
21
|
-
ORGANIZE_MATCH_LIMIT,
|
|
22
|
-
type OrganizeCandidate,
|
|
23
|
-
type OrganizeMatchDeps,
|
|
24
|
-
type OrganizeMatched,
|
|
25
|
-
type OrganizePredicate,
|
|
26
|
-
} from "./organize.js";
|
|
27
|
-
import {
|
|
28
|
-
_resetSemanticCapabilityForTest,
|
|
29
|
-
isSemanticSearchUnavailable,
|
|
30
|
-
} from "./semantic-capability.js";
|
|
31
|
-
|
|
32
|
-
const moduleNotFound = (): Error => {
|
|
33
|
-
const error = new Error("Cannot find package 'sqlite-vec' imported from …");
|
|
34
|
-
(error as Error & { code: string }).code = "ERR_MODULE_NOT_FOUND";
|
|
35
|
-
return error;
|
|
36
|
-
};
|
|
37
|
-
|
|
38
|
-
const ACCOUNT_CONFIG_ID = "cfg-1";
|
|
39
|
-
const ANCHOR_VECTOR = [1, 0, 0, 0];
|
|
40
|
-
const ORTHOGONAL_VECTOR = [0, 1, 0, 0];
|
|
41
|
-
/** The model the fixtures' embedder is currently configured with. */
|
|
42
|
-
const CURRENT_EMBEDDING_ID = "test-model@4";
|
|
43
|
-
/** The model a persisted anchor was written under before a same-dimension swap. */
|
|
44
|
-
const STALE_EMBEDDING_ID = "older-model@4";
|
|
45
|
-
const ANCHOR_SOURCE_TEXT = "book me a table";
|
|
46
|
-
|
|
47
|
-
const anchorPayload: AnchorPayload = {
|
|
48
|
-
anchorEmbedding: ANCHOR_VECTOR,
|
|
49
|
-
anchorEmbeddingId: CURRENT_EMBEDDING_ID,
|
|
50
|
-
anchorSourceText: ANCHOR_SOURCE_TEXT,
|
|
51
|
-
};
|
|
52
|
-
|
|
53
|
-
const metadata = (over: Partial<ChunkMetadata>): ChunkMetadata => ({
|
|
54
|
-
messageId: "msg-x",
|
|
55
|
-
threadId: "thread-x",
|
|
56
|
-
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
57
|
-
mailboxIds: ["mbox-1"],
|
|
58
|
-
chunkType: "body",
|
|
59
|
-
sentDate: 1_700_000_000,
|
|
60
|
-
isRead: false,
|
|
61
|
-
hasAttachment: false,
|
|
62
|
-
hasStars: false,
|
|
63
|
-
...over,
|
|
64
|
-
});
|
|
65
|
-
|
|
66
|
-
const bodyChunk = (
|
|
67
|
-
messageId: string,
|
|
68
|
-
vector: number[],
|
|
69
|
-
over: Partial<ChunkMetadata> = {},
|
|
70
|
-
): VectorRecord => ({
|
|
71
|
-
chunkId: `${messageId}#body`,
|
|
72
|
-
vector,
|
|
73
|
-
metadata: metadata({ messageId, ...over }),
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
const predicate = (
|
|
77
|
-
over: Partial<OrganizePredicate> = {},
|
|
78
|
-
): OrganizePredicate => ({
|
|
79
|
-
anchorMessageId: "msg-anchor",
|
|
80
|
-
matchOperator: FilterMatchOperator.And,
|
|
81
|
-
literalClauses: [],
|
|
82
|
-
similarityThreshold: 0.75,
|
|
83
|
-
actionLabelId: "None",
|
|
84
|
-
actionMailboxId: "None",
|
|
85
|
-
...over,
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
/**
|
|
89
|
-
* Run the matcher and assert it accepted the predicate, so every test that only
|
|
90
|
-
* cares about the matched set reads the accepted arm directly. A rejection is a
|
|
91
|
-
* result, not a throw (reader #463), so without this the two would be silently
|
|
92
|
-
* interchangeable at the call site.
|
|
93
|
-
*/
|
|
94
|
-
const matchAccepted = async (
|
|
95
|
-
...args: Parameters<typeof matchOrganize>
|
|
96
|
-
): Promise<OrganizeMatched> => {
|
|
97
|
-
const result = await matchOrganize(...args);
|
|
98
|
-
assert.ok(
|
|
99
|
-
result.rejected === null,
|
|
100
|
-
`expected an accepted match, got: ${result.rejected?.message}`,
|
|
101
|
-
);
|
|
102
|
-
return result;
|
|
103
|
-
};
|
|
104
|
-
|
|
105
|
-
/** A standing filter fixture — the account's already-existing rules. */
|
|
106
|
-
const filterItem = (over: Partial<FilterItem> = {}): FilterItem => ({
|
|
107
|
-
filterId: "filter-other",
|
|
108
|
-
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
109
|
-
name: "other filter",
|
|
110
|
-
scope: "Standing",
|
|
111
|
-
state: FilterState.Active,
|
|
112
|
-
disabledReason: "None",
|
|
113
|
-
hasAnchor: false,
|
|
114
|
-
ruleChangedAt: 0,
|
|
115
|
-
actionChangedAt: 0,
|
|
116
|
-
matchOperator: FilterMatchOperator.And,
|
|
117
|
-
literalClauses: [],
|
|
118
|
-
actionLabelId: "None",
|
|
119
|
-
actionMailboxId: "None",
|
|
120
|
-
createdAt: 0,
|
|
121
|
-
updatedAt: 0,
|
|
122
|
-
...over,
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
/**
|
|
126
|
-
* A client that records MessageLabel writes and blows up if the back-apply path
|
|
127
|
-
* ever touches Filter/FilterAnchor — the RFC 034 guardrail: this scope never
|
|
128
|
-
* persists a standing rule.
|
|
129
|
-
*
|
|
130
|
-
* `activeFilters`/`filterAnchorRows`/`threadMessages` model the account's
|
|
131
|
-
* *other*, already-existing standing filters and message rows. A back-apply
|
|
132
|
-
* never arbitrates against them (reader #497), so `filterReads` pins that the
|
|
133
|
-
* Active filter set is not even consulted.
|
|
134
|
-
*/
|
|
135
|
-
const trackingClient = (
|
|
136
|
-
seed: {
|
|
137
|
-
activeFilters?: FilterItem[];
|
|
138
|
-
filterAnchorRows?: FilterAnchorItem[];
|
|
139
|
-
threadMessages?: Record<
|
|
140
|
-
string,
|
|
141
|
-
{ fromEmail?: string; fromName?: string; subject?: string }
|
|
142
|
-
>;
|
|
143
|
-
} = {},
|
|
144
|
-
) => {
|
|
145
|
-
const labeled: Array<{ messageId: string; labelId: string }> = [];
|
|
146
|
-
let filterWrites = 0;
|
|
147
|
-
let filterReads = 0;
|
|
148
|
-
let filterAnchorWrites = 0;
|
|
149
|
-
const anchorPuts: CreateFilterAnchorInput[] = [];
|
|
150
|
-
const activeFilters = seed.activeFilters ?? [];
|
|
151
|
-
const filterAnchorRows = seed.filterAnchorRows ?? [];
|
|
152
|
-
const threadMessages = seed.threadMessages ?? {};
|
|
153
|
-
const client = {
|
|
154
|
-
messageLabel: {
|
|
155
|
-
apply: async (input: {
|
|
156
|
-
messageId: string;
|
|
157
|
-
labelId: string;
|
|
158
|
-
accountConfigId: string;
|
|
159
|
-
appliedByFilterId?: string;
|
|
160
|
-
}) => {
|
|
161
|
-
assert.equal(
|
|
162
|
-
input.appliedByFilterId,
|
|
163
|
-
undefined,
|
|
164
|
-
"back-apply must never attribute a filter",
|
|
165
|
-
);
|
|
166
|
-
labeled.push({ messageId: input.messageId, labelId: input.labelId });
|
|
167
|
-
return {} as never;
|
|
168
|
-
},
|
|
169
|
-
},
|
|
170
|
-
message: {
|
|
171
|
-
get: async (messageId: string) => ({ messageId, mailboxId: "mbox-src" }),
|
|
172
|
-
},
|
|
173
|
-
mailbox: {
|
|
174
|
-
resolveAccountId: async () => "acct-1",
|
|
175
|
-
},
|
|
176
|
-
threadMessage: {
|
|
177
|
-
get: async (_accountConfigId: string, messageId: string) => {
|
|
178
|
-
const row = threadMessages[messageId];
|
|
179
|
-
if (!row) {
|
|
180
|
-
throw new NotFoundError("ThreadMessage not found");
|
|
181
|
-
}
|
|
182
|
-
return {
|
|
183
|
-
threadMessageId: messageId,
|
|
184
|
-
fromEmail: row.fromEmail,
|
|
185
|
-
fromName: row.fromName,
|
|
186
|
-
subject: row.subject,
|
|
187
|
-
} as never;
|
|
188
|
-
},
|
|
189
|
-
},
|
|
190
|
-
filter: {
|
|
191
|
-
create: async () => {
|
|
192
|
-
filterWrites += 1;
|
|
193
|
-
return {} as never;
|
|
194
|
-
},
|
|
195
|
-
listByAccountAndState: async () => {
|
|
196
|
-
filterReads += 1;
|
|
197
|
-
return activeFilters;
|
|
198
|
-
},
|
|
199
|
-
refreshExpiry: async (filter: FilterItem) => filter,
|
|
200
|
-
},
|
|
201
|
-
filterAnchor: {
|
|
202
|
-
put: async (input: CreateFilterAnchorInput) => {
|
|
203
|
-
filterAnchorWrites += 1;
|
|
204
|
-
anchorPuts.push(input);
|
|
205
|
-
const row: FilterAnchorItem = { ...input, createdAt: 0, updatedAt: 1 };
|
|
206
|
-
const at = filterAnchorRows.findIndex(
|
|
207
|
-
(existing) => existing.filterId === input.filterId,
|
|
208
|
-
);
|
|
209
|
-
if (at === -1) filterAnchorRows.push(row);
|
|
210
|
-
else filterAnchorRows[at] = row;
|
|
211
|
-
return row;
|
|
212
|
-
},
|
|
213
|
-
get: async (_accountConfigId: string, filterId: string) =>
|
|
214
|
-
filterAnchorRows.find((row) => row.filterId === filterId) ?? null,
|
|
215
|
-
listByAccountConfig: async () => filterAnchorRows,
|
|
216
|
-
},
|
|
217
|
-
} as unknown as RemitClient;
|
|
218
|
-
return {
|
|
219
|
-
client,
|
|
220
|
-
labeled,
|
|
221
|
-
filterWrites: () => filterWrites,
|
|
222
|
-
filterReads: () => filterReads,
|
|
223
|
-
filterAnchorWrites: () => filterAnchorWrites,
|
|
224
|
-
anchorPuts,
|
|
225
|
-
};
|
|
226
|
-
};
|
|
227
|
-
|
|
228
|
-
/**
|
|
229
|
-
* A stand-in for the local-first placement mover. `moveMessage` is idempotent
|
|
230
|
-
* on the (messageId, destination) pair — mirroring the real
|
|
231
|
-
* `PlacementMoveService` marker engine (#1297) — so a redelivered back-apply
|
|
232
|
-
* re-issues the same move without double-enqueuing. Records every call and the
|
|
233
|
-
* final resting mailbox per message.
|
|
234
|
-
*/
|
|
235
|
-
const trackingMoveService = () => {
|
|
236
|
-
const moves: Array<{
|
|
237
|
-
messageId: string;
|
|
238
|
-
destinationMailboxId: string;
|
|
239
|
-
accountId: string;
|
|
240
|
-
}> = [];
|
|
241
|
-
const destinationOf = new Map<string, string>();
|
|
242
|
-
let enqueues = 0;
|
|
243
|
-
return {
|
|
244
|
-
moveService: {
|
|
245
|
-
moveMessage: async (
|
|
246
|
-
_accountConfigId: string,
|
|
247
|
-
messageId: string,
|
|
248
|
-
destinationMailboxId: string,
|
|
249
|
-
accountId: string,
|
|
250
|
-
): Promise<void> => {
|
|
251
|
-
moves.push({ messageId, destinationMailboxId, accountId });
|
|
252
|
-
if (destinationOf.get(messageId) === destinationMailboxId) return;
|
|
253
|
-
destinationOf.set(messageId, destinationMailboxId);
|
|
254
|
-
enqueues += 1;
|
|
255
|
-
},
|
|
256
|
-
} as unknown as import("@remit/mailbox-service").PlacementMoveService,
|
|
257
|
-
moves,
|
|
258
|
-
enqueues: () => enqueues,
|
|
259
|
-
destinations: () => destinationOf,
|
|
260
|
-
};
|
|
261
|
-
};
|
|
262
|
-
|
|
263
|
-
/**
|
|
264
|
-
* Deps whose semantic side is the in-memory vector store — the vector-backed
|
|
265
|
-
* deployment. `listAccountFilterMessages` returns the given corpus (empty by
|
|
266
|
-
* default), and constructing the semantic side is tracked so a literal-only
|
|
267
|
-
* predicate can be shown never to build it. `filterAnchorRows` seeds the
|
|
268
|
-
* account's *persisted* FilterAnchor rows (empty by default) — the reverse
|
|
269
|
-
* lookup back-apply's anchor consultation reads (reader #350) — and `embed`
|
|
270
|
-
* is a deterministic stand-in for the configured embedding model, used only
|
|
271
|
-
* by the exclusive-move precedence check.
|
|
272
|
-
*/
|
|
273
|
-
const matchDeps = (
|
|
274
|
-
store: ReturnType<typeof createMemoryVectorStore>,
|
|
275
|
-
corpus: OrganizeCandidate[] = [],
|
|
276
|
-
filterAnchorRows: FilterAnchorItem[] = [],
|
|
277
|
-
): OrganizeMatchDeps & { semanticBuilds: () => number } => {
|
|
278
|
-
let semanticBuilds = 0;
|
|
279
|
-
return {
|
|
280
|
-
semantic: () => {
|
|
281
|
-
semanticBuilds += 1;
|
|
282
|
-
return {
|
|
283
|
-
buildAnchor: async () => anchorPayload,
|
|
284
|
-
vectorStore: store,
|
|
285
|
-
embed: async (text: string) =>
|
|
286
|
-
text.includes("reservation") ? ANCHOR_VECTOR : ORTHOGONAL_VECTOR,
|
|
287
|
-
embeddingId: CURRENT_EMBEDDING_ID,
|
|
288
|
-
};
|
|
289
|
-
},
|
|
290
|
-
listAccountFilterMessages: async () => ({ items: corpus }),
|
|
291
|
-
filterAnchors: {
|
|
292
|
-
get: async () => null,
|
|
293
|
-
listByAccountConfig: async () => filterAnchorRows,
|
|
294
|
-
put: async () => {
|
|
295
|
-
throw new Error("matchDeps must not repair an anchor");
|
|
296
|
-
},
|
|
297
|
-
},
|
|
298
|
-
semanticBuilds: () => semanticBuilds,
|
|
299
|
-
};
|
|
300
|
-
};
|
|
301
|
-
|
|
302
|
-
/**
|
|
303
|
-
* Deps modelling a deployment that ships no vector pipeline: any attempt to
|
|
304
|
-
* build or use the semantic side raises the missing-module shape, and the
|
|
305
|
-
* literal corpus is served from plain rows.
|
|
306
|
-
*/
|
|
307
|
-
const vectorlessDeps = (
|
|
308
|
-
corpus: OrganizeCandidate[],
|
|
309
|
-
): OrganizeMatchDeps & { semanticUsed: () => boolean } => {
|
|
310
|
-
let semanticUsed = false;
|
|
311
|
-
return {
|
|
312
|
-
semantic: () => {
|
|
313
|
-
semanticUsed = true;
|
|
314
|
-
return {
|
|
315
|
-
buildAnchor: async () => {
|
|
316
|
-
throw moduleNotFound();
|
|
317
|
-
},
|
|
318
|
-
vectorStore: {
|
|
319
|
-
query: async () => {
|
|
320
|
-
throw moduleNotFound();
|
|
321
|
-
},
|
|
322
|
-
getByMessage: async () => {
|
|
323
|
-
throw moduleNotFound();
|
|
324
|
-
},
|
|
325
|
-
},
|
|
326
|
-
embed: async () => {
|
|
327
|
-
throw moduleNotFound();
|
|
328
|
-
},
|
|
329
|
-
embeddingId: CURRENT_EMBEDDING_ID,
|
|
330
|
-
};
|
|
331
|
-
},
|
|
332
|
-
listAccountFilterMessages: async () => ({ items: corpus }),
|
|
333
|
-
filterAnchors: {
|
|
334
|
-
get: async () => null,
|
|
335
|
-
listByAccountConfig: async () => [],
|
|
336
|
-
put: async () => {
|
|
337
|
-
throw moduleNotFound();
|
|
338
|
-
},
|
|
339
|
-
},
|
|
340
|
-
semanticUsed: () => semanticUsed,
|
|
341
|
-
};
|
|
342
|
-
};
|
|
343
|
-
|
|
344
|
-
const candidate = (
|
|
345
|
-
messageId: string,
|
|
346
|
-
over: Partial<OrganizeCandidate["message"]> = {},
|
|
347
|
-
): OrganizeCandidate => ({
|
|
348
|
-
messageId,
|
|
349
|
-
message: {
|
|
350
|
-
from: "",
|
|
351
|
-
fromName: "",
|
|
352
|
-
subject: "",
|
|
353
|
-
text: "",
|
|
354
|
-
listId: "",
|
|
355
|
-
...over,
|
|
356
|
-
},
|
|
357
|
-
});
|
|
358
|
-
|
|
359
|
-
describe("matchOrganize", () => {
|
|
360
|
-
it("returns every semantically matching message and excludes the misses", async () => {
|
|
361
|
-
const store = createMemoryVectorStore();
|
|
362
|
-
const matching = ["msg-1", "msg-2", "msg-3", "msg-4", "msg-5"];
|
|
363
|
-
await store.upsert([
|
|
364
|
-
...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
|
|
365
|
-
bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
|
|
366
|
-
]);
|
|
367
|
-
|
|
368
|
-
const { messageIds, semanticUnavailable } = await matchAccepted(
|
|
369
|
-
matchDeps(store),
|
|
370
|
-
ACCOUNT_CONFIG_ID,
|
|
371
|
-
predicate({ actionLabelId: "lbl-1" }),
|
|
372
|
-
);
|
|
373
|
-
|
|
374
|
-
assert.deepEqual([...messageIds].sort(), matching);
|
|
375
|
-
assert.equal(semanticUnavailable, false);
|
|
376
|
-
});
|
|
377
|
-
|
|
378
|
-
it("matches nothing when the predicate has neither an anchor nor a clause", async () => {
|
|
379
|
-
const store = createMemoryVectorStore();
|
|
380
|
-
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
381
|
-
|
|
382
|
-
const { messageIds } = await matchAccepted(
|
|
383
|
-
matchDeps(store),
|
|
384
|
-
ACCOUNT_CONFIG_ID,
|
|
385
|
-
{
|
|
386
|
-
...predicate(),
|
|
387
|
-
anchorMessageId: "None",
|
|
388
|
-
},
|
|
389
|
-
);
|
|
390
|
-
|
|
391
|
-
assert.deepEqual(messageIds, []);
|
|
392
|
-
});
|
|
393
|
-
|
|
394
|
-
it("refines the semantic set by literal clauses", async () => {
|
|
395
|
-
const store = createMemoryVectorStore();
|
|
396
|
-
await store.upsert([
|
|
397
|
-
bodyChunk("msg-1", ANCHOR_VECTOR, {
|
|
398
|
-
subject: "Dinner reservation",
|
|
399
|
-
textPreview: "your table is booked",
|
|
400
|
-
}),
|
|
401
|
-
bodyChunk("msg-2", ANCHOR_VECTOR, {
|
|
402
|
-
subject: "Newsletter",
|
|
403
|
-
textPreview: "weekly digest",
|
|
404
|
-
}),
|
|
405
|
-
]);
|
|
406
|
-
|
|
407
|
-
const { messageIds } = await matchAccepted(
|
|
408
|
-
matchDeps(store),
|
|
409
|
-
ACCOUNT_CONFIG_ID,
|
|
410
|
-
{
|
|
411
|
-
...predicate(),
|
|
412
|
-
literalClauses: [{ field: "Subject", value: "reservation" }],
|
|
413
|
-
},
|
|
414
|
-
);
|
|
415
|
-
|
|
416
|
-
assert.deepEqual(messageIds, ["msg-1"]);
|
|
417
|
-
});
|
|
418
|
-
|
|
419
|
-
it("never builds the semantic side for a purely-literal predicate", async () => {
|
|
420
|
-
const store = createMemoryVectorStore();
|
|
421
|
-
const deps = matchDeps(store, [
|
|
422
|
-
candidate("msg-1", { subject: "Dinner reservation" }),
|
|
423
|
-
candidate("msg-2", { subject: "Newsletter" }),
|
|
424
|
-
]);
|
|
425
|
-
|
|
426
|
-
const { messageIds } = await matchAccepted(deps, ACCOUNT_CONFIG_ID, {
|
|
427
|
-
...predicate(),
|
|
428
|
-
anchorMessageId: "None",
|
|
429
|
-
literalClauses: [{ field: "Subject", value: "reservation" }],
|
|
430
|
-
});
|
|
431
|
-
|
|
432
|
-
assert.deepEqual(messageIds, ["msg-1"]);
|
|
433
|
-
assert.equal(
|
|
434
|
-
deps.semanticBuilds(),
|
|
435
|
-
0,
|
|
436
|
-
"a literal-only predicate must not construct the vector-backed side",
|
|
437
|
-
);
|
|
438
|
-
});
|
|
439
|
-
});
|
|
440
|
-
|
|
441
|
-
describe("matchOrganize honors the persisted FilterAnchor (reader #350)", () => {
|
|
442
|
-
it("still matches by reading the persisted anchor after the anchor message is purged", async () => {
|
|
443
|
-
const store = createMemoryVectorStore();
|
|
444
|
-
const matching = ["msg-1", "msg-2"];
|
|
445
|
-
await store.upsert([
|
|
446
|
-
...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
|
|
447
|
-
bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
|
|
448
|
-
]);
|
|
449
|
-
|
|
450
|
-
const persistedAnchor: FilterAnchorItem = {
|
|
451
|
-
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
452
|
-
filterId: "filter-a",
|
|
453
|
-
anchorEmbedding: ANCHOR_VECTOR,
|
|
454
|
-
anchorEmbeddingId: CURRENT_EMBEDDING_ID,
|
|
455
|
-
anchorSourceText: ANCHOR_SOURCE_TEXT,
|
|
456
|
-
anchorMessageId: "msg-anchor",
|
|
457
|
-
createdAt: 0,
|
|
458
|
-
updatedAt: 0,
|
|
459
|
-
};
|
|
460
|
-
|
|
461
|
-
let buildAnchorCalls = 0;
|
|
462
|
-
const deps: OrganizeMatchDeps = {
|
|
463
|
-
semantic: () => ({
|
|
464
|
-
buildAnchor: async () => {
|
|
465
|
-
// The live path: the anchor message's chunks are gone (purged),
|
|
466
|
-
// exactly what buildMessageAnchor returns in that case today.
|
|
467
|
-
buildAnchorCalls += 1;
|
|
468
|
-
return null;
|
|
469
|
-
},
|
|
470
|
-
vectorStore: store,
|
|
471
|
-
embed: async () => ANCHOR_VECTOR,
|
|
472
|
-
embeddingId: CURRENT_EMBEDDING_ID,
|
|
473
|
-
}),
|
|
474
|
-
listAccountFilterMessages: async () => ({ items: [] }),
|
|
475
|
-
filterAnchors: {
|
|
476
|
-
get: async () => null,
|
|
477
|
-
listByAccountConfig: async () => [persistedAnchor],
|
|
478
|
-
put: async () => {
|
|
479
|
-
throw new Error("a current anchor must not be rewritten");
|
|
480
|
-
},
|
|
481
|
-
},
|
|
482
|
-
};
|
|
483
|
-
|
|
484
|
-
const { messageIds } = await matchAccepted(
|
|
485
|
-
deps,
|
|
486
|
-
ACCOUNT_CONFIG_ID,
|
|
487
|
-
predicate(),
|
|
488
|
-
);
|
|
489
|
-
|
|
490
|
-
assert.deepEqual(
|
|
491
|
-
[...messageIds].sort(),
|
|
492
|
-
matching,
|
|
493
|
-
"the persisted anchor vector still finds the same matches the live message would have",
|
|
494
|
-
);
|
|
495
|
-
assert.equal(
|
|
496
|
-
buildAnchorCalls,
|
|
497
|
-
0,
|
|
498
|
-
"a persisted anchor must be read instead of re-deriving one from the (now-gone) live message",
|
|
499
|
-
);
|
|
500
|
-
});
|
|
501
|
-
|
|
502
|
-
it("falls back to deriving the anchor from the live message when no filter was ever anchored on it", async () => {
|
|
503
|
-
const store = createMemoryVectorStore();
|
|
504
|
-
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
505
|
-
|
|
506
|
-
// No persisted FilterAnchor names this anchorMessageId — an ad hoc "all
|
|
507
|
-
// like these" widen over a bare message selection, never tied to a
|
|
508
|
-
// standing filter.
|
|
509
|
-
const { messageIds } = await matchAccepted(
|
|
510
|
-
matchDeps(store),
|
|
511
|
-
ACCOUNT_CONFIG_ID,
|
|
512
|
-
predicate(),
|
|
513
|
-
);
|
|
514
|
-
|
|
515
|
-
assert.deepEqual(messageIds, ["msg-1"]);
|
|
516
|
-
});
|
|
517
|
-
});
|
|
518
|
-
|
|
519
|
-
/**
|
|
520
|
-
* A zero from the widen has two meanings and the preview could not tell them
|
|
521
|
-
* apart (issue #452): no mail resembles the anchor, or the index holds nothing
|
|
522
|
-
* to compare it against. Only the second is a reason to keep the rule.
|
|
523
|
-
*/
|
|
524
|
-
describe("matchOrganize reporting an empty semantic index", () => {
|
|
525
|
-
it("flags a widen answered by an index with no vectors at all", async () => {
|
|
526
|
-
const { messageIds, semanticIndexEmpty, semanticUnavailable } =
|
|
527
|
-
await matchAccepted(
|
|
528
|
-
matchDeps(createMemoryVectorStore()),
|
|
529
|
-
ACCOUNT_CONFIG_ID,
|
|
530
|
-
predicate(),
|
|
531
|
-
);
|
|
532
|
-
|
|
533
|
-
assert.deepEqual(messageIds, []);
|
|
534
|
-
assert.equal(semanticIndexEmpty, true);
|
|
535
|
-
assert.equal(
|
|
536
|
-
semanticUnavailable,
|
|
537
|
-
false,
|
|
538
|
-
"an empty index is not a missing vector pipeline",
|
|
539
|
-
);
|
|
540
|
-
});
|
|
541
|
-
|
|
542
|
-
it("flags a widen whose anchor message has no chunk vectors to pool", async () => {
|
|
543
|
-
const store = createMemoryVectorStore();
|
|
544
|
-
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
545
|
-
const deps: OrganizeMatchDeps = {
|
|
546
|
-
...matchDeps(store),
|
|
547
|
-
semantic: () => ({
|
|
548
|
-
buildAnchor: async () => null,
|
|
549
|
-
vectorStore: store,
|
|
550
|
-
embed: async () => ANCHOR_VECTOR,
|
|
551
|
-
embeddingId: CURRENT_EMBEDDING_ID,
|
|
552
|
-
}),
|
|
553
|
-
};
|
|
554
|
-
|
|
555
|
-
const { messageIds, semanticIndexEmpty } = await matchAccepted(
|
|
556
|
-
deps,
|
|
557
|
-
ACCOUNT_CONFIG_ID,
|
|
558
|
-
predicate(),
|
|
559
|
-
);
|
|
560
|
-
|
|
561
|
-
assert.deepEqual(messageIds, []);
|
|
562
|
-
assert.equal(semanticIndexEmpty, true);
|
|
563
|
-
});
|
|
564
|
-
|
|
565
|
-
it("does not flag a populated index that simply holds nothing similar enough", async () => {
|
|
566
|
-
const store = createMemoryVectorStore();
|
|
567
|
-
await store.upsert([bodyChunk("msg-1", ORTHOGONAL_VECTOR)]);
|
|
568
|
-
|
|
569
|
-
const { messageIds, semanticIndexEmpty } = await matchAccepted(
|
|
570
|
-
matchDeps(store),
|
|
571
|
-
ACCOUNT_CONFIG_ID,
|
|
572
|
-
predicate(),
|
|
573
|
-
);
|
|
574
|
-
|
|
575
|
-
assert.deepEqual(
|
|
576
|
-
messageIds,
|
|
577
|
-
[],
|
|
578
|
-
"the one indexed message is below threshold",
|
|
579
|
-
);
|
|
580
|
-
assert.equal(
|
|
581
|
-
semanticIndexEmpty,
|
|
582
|
-
false,
|
|
583
|
-
"the index answered; the rule is what matched nothing",
|
|
584
|
-
);
|
|
585
|
-
});
|
|
586
|
-
|
|
587
|
-
it("does not flag a widen that matched", async () => {
|
|
588
|
-
const store = createMemoryVectorStore();
|
|
589
|
-
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
590
|
-
|
|
591
|
-
const { messageIds, semanticIndexEmpty } = await matchAccepted(
|
|
592
|
-
matchDeps(store),
|
|
593
|
-
ACCOUNT_CONFIG_ID,
|
|
594
|
-
predicate(),
|
|
595
|
-
);
|
|
596
|
-
|
|
597
|
-
assert.deepEqual(messageIds, ["msg-1"]);
|
|
598
|
-
assert.equal(semanticIndexEmpty, false);
|
|
599
|
-
});
|
|
600
|
-
|
|
601
|
-
it("does not flag a literal-only predicate, which never consults the index", async () => {
|
|
602
|
-
const { semanticIndexEmpty } = await matchAccepted(
|
|
603
|
-
matchDeps(createMemoryVectorStore(), [
|
|
604
|
-
candidate("msg-1", { subject: "Weekly newsletter" }),
|
|
605
|
-
]),
|
|
606
|
-
ACCOUNT_CONFIG_ID,
|
|
607
|
-
{
|
|
608
|
-
...predicate(),
|
|
609
|
-
anchorMessageId: "None",
|
|
610
|
-
literalClauses: [{ field: "Subject", value: "reservation" }],
|
|
611
|
-
},
|
|
612
|
-
);
|
|
613
|
-
|
|
614
|
-
assert.equal(semanticIndexEmpty, false);
|
|
615
|
-
});
|
|
616
|
-
});
|
|
617
|
-
|
|
618
|
-
describe("matchOrganize on a deployment without the vector pipeline", () => {
|
|
619
|
-
const ORIGINAL = process.env.DATA_BACKEND;
|
|
620
|
-
beforeEach(() => {
|
|
621
|
-
process.env.DATA_BACKEND = "sqlite";
|
|
622
|
-
_resetSemanticCapabilityForTest();
|
|
623
|
-
});
|
|
624
|
-
afterEach(() => {
|
|
625
|
-
if (ORIGINAL === undefined) delete process.env.DATA_BACKEND;
|
|
626
|
-
else process.env.DATA_BACKEND = ORIGINAL;
|
|
627
|
-
_resetSemanticCapabilityForTest();
|
|
628
|
-
});
|
|
629
|
-
|
|
630
|
-
it("matches a literal predicate from the corpus without touching the vector store", async () => {
|
|
631
|
-
const deps = vectorlessDeps([
|
|
632
|
-
candidate("msg-1", { subject: "Dinner reservation" }),
|
|
633
|
-
candidate("msg-2", { subject: "Weekly newsletter" }),
|
|
634
|
-
candidate("msg-3", { subject: "Table reservation confirmed" }),
|
|
635
|
-
]);
|
|
636
|
-
|
|
637
|
-
const { messageIds, semanticUnavailable } = await matchAccepted(
|
|
638
|
-
deps,
|
|
639
|
-
ACCOUNT_CONFIG_ID,
|
|
640
|
-
{
|
|
641
|
-
...predicate(),
|
|
642
|
-
anchorMessageId: "None",
|
|
643
|
-
literalClauses: [{ field: "Subject", value: "reservation" }],
|
|
644
|
-
},
|
|
645
|
-
);
|
|
646
|
-
|
|
647
|
-
assert.deepEqual(messageIds, ["msg-1", "msg-3"]);
|
|
648
|
-
assert.equal(semanticUnavailable, false);
|
|
649
|
-
assert.equal(
|
|
650
|
-
deps.semanticUsed(),
|
|
651
|
-
false,
|
|
652
|
-
"a literal-only predicate must never reach the vector store",
|
|
653
|
-
);
|
|
654
|
-
});
|
|
655
|
-
|
|
656
|
-
it("refuses a body-content (HasWords) clause as a result rather than matching it against a preview", async () => {
|
|
657
|
-
const deps = vectorlessDeps([
|
|
658
|
-
candidate("msg-1", { subject: "Dinner reservation" }),
|
|
659
|
-
]);
|
|
660
|
-
|
|
661
|
-
const result = await matchOrganize(deps, ACCOUNT_CONFIG_ID, {
|
|
662
|
-
...predicate(),
|
|
663
|
-
anchorMessageId: "None",
|
|
664
|
-
literalClauses: [{ field: "HasWords", value: "invoice" }],
|
|
665
|
-
});
|
|
666
|
-
|
|
667
|
-
assert.deepEqual(
|
|
668
|
-
result.rejected,
|
|
669
|
-
{
|
|
670
|
-
reason: "BodyContentWithoutVectorPipeline",
|
|
671
|
-
message: BODY_CONTENT_REJECTION_MESSAGE,
|
|
672
|
-
},
|
|
673
|
-
"the vector-free literal path must not silently narrow a body match to a preview",
|
|
674
|
-
);
|
|
675
|
-
assert.equal(deps.semanticUsed(), false);
|
|
676
|
-
});
|
|
677
|
-
|
|
678
|
-
// The refusal is the same whichever way the vector-free path is reached, so
|
|
679
|
-
// an anchored predicate that degrades onto it cannot land in the corpus scan
|
|
680
|
-
// with a body clause it can only mis-evaluate (reader #463).
|
|
681
|
-
it("refuses a body-content clause reached through the degraded widen fallback", async () => {
|
|
682
|
-
const deps = vectorlessDeps([
|
|
683
|
-
candidate("msg-1", { subject: "Dinner reservation" }),
|
|
684
|
-
]);
|
|
685
|
-
|
|
686
|
-
const result = await matchOrganize(deps, ACCOUNT_CONFIG_ID, {
|
|
687
|
-
...predicate(),
|
|
688
|
-
literalClauses: [{ field: "HasWords", value: "invoice" }],
|
|
689
|
-
});
|
|
690
|
-
|
|
691
|
-
assert.equal(result.rejected?.reason, "BodyContentWithoutVectorPipeline");
|
|
692
|
-
});
|
|
693
|
-
|
|
694
|
-
it("degrades an anchor+clauses widen to the literal matches, flagged, instead of crashing", async () => {
|
|
695
|
-
const deps = vectorlessDeps([
|
|
696
|
-
candidate("msg-1", { subject: "Dinner reservation" }),
|
|
697
|
-
candidate("msg-2", { subject: "Weekly newsletter" }),
|
|
698
|
-
]);
|
|
699
|
-
|
|
700
|
-
const { messageIds, semanticUnavailable } = await matchAccepted(
|
|
701
|
-
deps,
|
|
702
|
-
ACCOUNT_CONFIG_ID,
|
|
703
|
-
{
|
|
704
|
-
...predicate(),
|
|
705
|
-
literalClauses: [{ field: "Subject", value: "reservation" }],
|
|
706
|
-
},
|
|
707
|
-
);
|
|
708
|
-
|
|
709
|
-
assert.deepEqual(messageIds, ["msg-1"]);
|
|
710
|
-
assert.equal(semanticUnavailable, true);
|
|
711
|
-
});
|
|
712
|
-
|
|
713
|
-
it("degrades an anchor-only widen to an empty flagged result instead of crashing", async () => {
|
|
714
|
-
const deps = vectorlessDeps([candidate("msg-1"), candidate("msg-2")]);
|
|
715
|
-
|
|
716
|
-
const { messageIds, semanticUnavailable, semanticIndexEmpty } =
|
|
717
|
-
await matchAccepted(deps, ACCOUNT_CONFIG_ID, predicate());
|
|
718
|
-
|
|
719
|
-
assert.deepEqual(messageIds, []);
|
|
720
|
-
assert.equal(semanticUnavailable, true);
|
|
721
|
-
assert.equal(
|
|
722
|
-
semanticIndexEmpty,
|
|
723
|
-
false,
|
|
724
|
-
"a deployment with no vector pipeline has no index to call empty",
|
|
725
|
-
);
|
|
726
|
-
});
|
|
727
|
-
|
|
728
|
-
it("keeps widening from a drifted persisted anchor when no embedding model is available", async () => {
|
|
729
|
-
// Self-host: sqlite-vec is present, `@huggingface/transformers` is not.
|
|
730
|
-
// The anchor's stamp will never match — after a model swap, or forever
|
|
731
|
-
// for one pooled from pre-#349 chunks and stamped
|
|
732
|
-
// UNKNOWN_CHUNK_EMBEDDING_ID — and there is no embedder to repair it
|
|
733
|
-
// with, so the widen must query with the vector it has rather than go
|
|
734
|
-
// dark and take every later /search/semantic with it.
|
|
735
|
-
const store = createMemoryVectorStore();
|
|
736
|
-
await store.upsert([
|
|
737
|
-
bodyChunk("msg-1", ANCHOR_VECTOR),
|
|
738
|
-
bodyChunk("msg-stored-space", ORTHOGONAL_VECTOR),
|
|
739
|
-
]);
|
|
740
|
-
const drifted: FilterAnchorItem = {
|
|
741
|
-
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
742
|
-
filterId: "filter-a",
|
|
743
|
-
anchorEmbedding: ORTHOGONAL_VECTOR,
|
|
744
|
-
anchorEmbeddingId: STALE_EMBEDDING_ID,
|
|
745
|
-
anchorSourceText: ANCHOR_SOURCE_TEXT,
|
|
746
|
-
anchorMessageId: "msg-anchor",
|
|
747
|
-
createdAt: 0,
|
|
748
|
-
updatedAt: 0,
|
|
749
|
-
};
|
|
750
|
-
const deps: OrganizeMatchDeps = {
|
|
751
|
-
semantic: () => ({
|
|
752
|
-
buildAnchor: async () => {
|
|
753
|
-
throw new Error("a persisted anchor must not be re-derived");
|
|
754
|
-
},
|
|
755
|
-
vectorStore: store,
|
|
756
|
-
embed: async () => {
|
|
757
|
-
throw moduleNotFound();
|
|
758
|
-
},
|
|
759
|
-
embeddingId: CURRENT_EMBEDDING_ID,
|
|
760
|
-
}),
|
|
761
|
-
listAccountFilterMessages: async () => ({ items: [] }),
|
|
762
|
-
filterAnchors: {
|
|
763
|
-
get: async () => null,
|
|
764
|
-
listByAccountConfig: async () => [drifted],
|
|
765
|
-
put: async () => {
|
|
766
|
-
throw new Error("an unrepairable anchor must not be rewritten");
|
|
767
|
-
},
|
|
768
|
-
},
|
|
769
|
-
};
|
|
770
|
-
|
|
771
|
-
const { messageIds, semanticUnavailable } = await matchAccepted(
|
|
772
|
-
deps,
|
|
773
|
-
ACCOUNT_CONFIG_ID,
|
|
774
|
-
predicate(),
|
|
775
|
-
);
|
|
776
|
-
|
|
777
|
-
assert.deepEqual(
|
|
778
|
-
messageIds,
|
|
779
|
-
["msg-stored-space"],
|
|
780
|
-
"the kNN read needs no embedder, so the stored anchor vector still returns real hits",
|
|
781
|
-
);
|
|
782
|
-
assert.equal(semanticUnavailable, false);
|
|
783
|
-
assert.equal(
|
|
784
|
-
isSemanticSearchUnavailable(),
|
|
785
|
-
false,
|
|
786
|
-
"one unrepairable anchor must not disable free-text semantic search for the rest of the process",
|
|
787
|
-
);
|
|
788
|
-
});
|
|
789
|
-
|
|
790
|
-
it("propagates a genuine (non-capability) semantic failure loudly", async () => {
|
|
791
|
-
const deps: OrganizeMatchDeps = {
|
|
792
|
-
semantic: () => ({
|
|
793
|
-
buildAnchor: async () => {
|
|
794
|
-
throw new Error("SQLITE_BUSY");
|
|
795
|
-
},
|
|
796
|
-
vectorStore: {
|
|
797
|
-
query: async () => [],
|
|
798
|
-
getByMessage: async () => [],
|
|
799
|
-
},
|
|
800
|
-
embed: async () => [],
|
|
801
|
-
embeddingId: CURRENT_EMBEDDING_ID,
|
|
802
|
-
}),
|
|
803
|
-
listAccountFilterMessages: async () => ({ items: [] }),
|
|
804
|
-
filterAnchors: {
|
|
805
|
-
get: async () => null,
|
|
806
|
-
listByAccountConfig: async () => [],
|
|
807
|
-
put: async () => {
|
|
808
|
-
throw new Error("unreachable");
|
|
809
|
-
},
|
|
810
|
-
},
|
|
811
|
-
};
|
|
812
|
-
|
|
813
|
-
await assert.rejects(
|
|
814
|
-
() => matchOrganize(deps, ACCOUNT_CONFIG_ID, predicate()),
|
|
815
|
-
/SQLITE_BUSY/,
|
|
816
|
-
);
|
|
817
|
-
});
|
|
818
|
-
});
|
|
819
|
-
|
|
820
|
-
describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
|
|
821
|
-
it("applies the label to all N matches in one pass and writes zero Filter rows", async () => {
|
|
822
|
-
const store = createMemoryVectorStore();
|
|
823
|
-
const matching = ["msg-1", "msg-2", "msg-3", "msg-4", "msg-5"];
|
|
824
|
-
await store.upsert([
|
|
825
|
-
...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
|
|
826
|
-
bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
|
|
827
|
-
]);
|
|
828
|
-
|
|
829
|
-
const p = predicate({ actionLabelId: "lbl-1" });
|
|
830
|
-
|
|
831
|
-
// The preview and the apply share the same matcher — the previewed set is
|
|
832
|
-
// exactly what gets applied.
|
|
833
|
-
const previewed = await matchAccepted(
|
|
834
|
-
matchDeps(store),
|
|
835
|
-
ACCOUNT_CONFIG_ID,
|
|
836
|
-
p,
|
|
837
|
-
);
|
|
838
|
-
const applied = await matchAccepted(matchDeps(store), ACCOUNT_CONFIG_ID, p);
|
|
839
|
-
assert.deepEqual(previewed, applied);
|
|
840
|
-
|
|
841
|
-
const tracked = trackingClient();
|
|
842
|
-
const result = await applyOrganize(
|
|
843
|
-
{ client: tracked.client },
|
|
844
|
-
ACCOUNT_CONFIG_ID,
|
|
845
|
-
applied.messageIds,
|
|
846
|
-
p,
|
|
847
|
-
);
|
|
848
|
-
|
|
849
|
-
assert.equal(result.applied, matching.length);
|
|
850
|
-
assert.equal(result.failed, 0);
|
|
851
|
-
assert.deepEqual(
|
|
852
|
-
tracked.labeled.map((row) => row.messageId).sort(),
|
|
853
|
-
matching,
|
|
854
|
-
);
|
|
855
|
-
assert.ok(
|
|
856
|
-
tracked.labeled.every((row) => row.labelId === "lbl-1"),
|
|
857
|
-
"every matching message gets the requested label",
|
|
858
|
-
);
|
|
859
|
-
assert.equal(tracked.filterWrites(), 0, "no Filter row is ever created");
|
|
860
|
-
assert.equal(
|
|
861
|
-
tracked.filterAnchorWrites(),
|
|
862
|
-
0,
|
|
863
|
-
"no FilterAnchor row is ever created",
|
|
864
|
-
);
|
|
865
|
-
});
|
|
866
|
-
|
|
867
|
-
it("counts a requested move as failed when no move service is wired", async () => {
|
|
868
|
-
const store = createMemoryVectorStore();
|
|
869
|
-
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
870
|
-
const p = predicate({ actionMailboxId: "mbox-target" });
|
|
871
|
-
|
|
872
|
-
const { messageIds: matched } = await matchAccepted(
|
|
873
|
-
matchDeps(store),
|
|
874
|
-
ACCOUNT_CONFIG_ID,
|
|
875
|
-
p,
|
|
876
|
-
);
|
|
877
|
-
const tracked = trackingClient();
|
|
878
|
-
const result = await applyOrganize(
|
|
879
|
-
{ client: tracked.client },
|
|
880
|
-
ACCOUNT_CONFIG_ID,
|
|
881
|
-
matched,
|
|
882
|
-
p,
|
|
883
|
-
);
|
|
884
|
-
|
|
885
|
-
assert.equal(result.applied, 0);
|
|
886
|
-
assert.equal(result.failed, 1);
|
|
887
|
-
});
|
|
888
|
-
|
|
889
|
-
it("moves every match through the wired move service and writes zero Filter rows", async () => {
|
|
890
|
-
const store = createMemoryVectorStore();
|
|
891
|
-
const matching = ["msg-1", "msg-2", "msg-3"];
|
|
892
|
-
await store.upsert([
|
|
893
|
-
...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
|
|
894
|
-
bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
|
|
895
|
-
]);
|
|
896
|
-
const p = predicate({ actionMailboxId: "mbox-target" });
|
|
897
|
-
|
|
898
|
-
const { messageIds: matched } = await matchAccepted(
|
|
899
|
-
matchDeps(store),
|
|
900
|
-
ACCOUNT_CONFIG_ID,
|
|
901
|
-
p,
|
|
902
|
-
);
|
|
903
|
-
const tracked = trackingClient();
|
|
904
|
-
const mover = trackingMoveService();
|
|
905
|
-
const result = await applyOrganize(
|
|
906
|
-
{
|
|
907
|
-
client: tracked.client,
|
|
908
|
-
moveService: mover.moveService,
|
|
909
|
-
},
|
|
910
|
-
ACCOUNT_CONFIG_ID,
|
|
911
|
-
matched,
|
|
912
|
-
p,
|
|
913
|
-
);
|
|
914
|
-
|
|
915
|
-
assert.equal(result.applied, matching.length);
|
|
916
|
-
assert.equal(result.failed, 0);
|
|
917
|
-
assert.deepEqual(
|
|
918
|
-
mover.moves.map((m) => m.messageId).sort(),
|
|
919
|
-
matching,
|
|
920
|
-
"every matched message is moved once",
|
|
921
|
-
);
|
|
922
|
-
assert.ok(
|
|
923
|
-
mover.moves.every((m) => m.destinationMailboxId === "mbox-target"),
|
|
924
|
-
"every move targets the requested mailbox",
|
|
925
|
-
);
|
|
926
|
-
assert.equal(
|
|
927
|
-
tracked.labeled.length,
|
|
928
|
-
0,
|
|
929
|
-
"a move-only action applies no label",
|
|
930
|
-
);
|
|
931
|
-
assert.equal(tracked.filterWrites(), 0, "no Filter row is ever created");
|
|
932
|
-
assert.equal(
|
|
933
|
-
tracked.filterAnchorWrites(),
|
|
934
|
-
0,
|
|
935
|
-
"no FilterAnchor row is ever created",
|
|
936
|
-
);
|
|
937
|
-
});
|
|
938
|
-
|
|
939
|
-
it("applies both a label and a move when both actions are requested", async () => {
|
|
940
|
-
const store = createMemoryVectorStore();
|
|
941
|
-
await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
|
|
942
|
-
const p = predicate({
|
|
943
|
-
actionLabelId: "lbl-1",
|
|
944
|
-
actionMailboxId: "mbox-target",
|
|
945
|
-
});
|
|
946
|
-
|
|
947
|
-
const { messageIds: matched } = await matchAccepted(
|
|
948
|
-
matchDeps(store),
|
|
949
|
-
ACCOUNT_CONFIG_ID,
|
|
950
|
-
p,
|
|
951
|
-
);
|
|
952
|
-
const tracked = trackingClient();
|
|
953
|
-
const mover = trackingMoveService();
|
|
954
|
-
const result = await applyOrganize(
|
|
955
|
-
{
|
|
956
|
-
client: tracked.client,
|
|
957
|
-
moveService: mover.moveService,
|
|
958
|
-
},
|
|
959
|
-
ACCOUNT_CONFIG_ID,
|
|
960
|
-
matched,
|
|
961
|
-
p,
|
|
962
|
-
);
|
|
963
|
-
|
|
964
|
-
assert.equal(result.applied, 1);
|
|
965
|
-
assert.equal(result.failed, 0);
|
|
966
|
-
assert.deepEqual(tracked.labeled, [
|
|
967
|
-
{ messageId: "msg-1", labelId: "lbl-1" },
|
|
968
|
-
]);
|
|
969
|
-
assert.deepEqual(
|
|
970
|
-
mover.moves.map((m) => m.messageId),
|
|
971
|
-
["msg-1"],
|
|
972
|
-
);
|
|
973
|
-
});
|
|
974
|
-
|
|
975
|
-
it("is idempotent on redelivery: a re-run re-issues the same move without double-enqueuing", async () => {
|
|
976
|
-
const store = createMemoryVectorStore();
|
|
977
|
-
const matching = ["msg-1", "msg-2"];
|
|
978
|
-
await store.upsert(matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)));
|
|
979
|
-
const p = predicate({ actionMailboxId: "mbox-target" });
|
|
980
|
-
|
|
981
|
-
const { messageIds: matched } = await matchAccepted(
|
|
982
|
-
matchDeps(store),
|
|
983
|
-
ACCOUNT_CONFIG_ID,
|
|
984
|
-
p,
|
|
985
|
-
);
|
|
986
|
-
const tracked = trackingClient();
|
|
987
|
-
const mover = trackingMoveService();
|
|
988
|
-
|
|
989
|
-
const first = await applyOrganize(
|
|
990
|
-
{
|
|
991
|
-
client: tracked.client,
|
|
992
|
-
moveService: mover.moveService,
|
|
993
|
-
},
|
|
994
|
-
ACCOUNT_CONFIG_ID,
|
|
995
|
-
matched,
|
|
996
|
-
p,
|
|
997
|
-
);
|
|
998
|
-
const second = await applyOrganize(
|
|
999
|
-
{
|
|
1000
|
-
client: tracked.client,
|
|
1001
|
-
moveService: mover.moveService,
|
|
1002
|
-
},
|
|
1003
|
-
ACCOUNT_CONFIG_ID,
|
|
1004
|
-
matched,
|
|
1005
|
-
p,
|
|
1006
|
-
);
|
|
1007
|
-
|
|
1008
|
-
assert.equal(first.applied, matching.length);
|
|
1009
|
-
assert.equal(second.applied, matching.length);
|
|
1010
|
-
assert.equal(
|
|
1011
|
-
mover.enqueues(),
|
|
1012
|
-
matching.length,
|
|
1013
|
-
"the second pass drives the marker forward without a fresh enqueue",
|
|
1014
|
-
);
|
|
1015
|
-
assert.deepEqual(
|
|
1016
|
-
[...mover.destinations().entries()].sort(),
|
|
1017
|
-
matching.map((id): [string, string] => [id, "mbox-target"]),
|
|
1018
|
-
"each message rests in the requested mailbox exactly once",
|
|
1019
|
-
);
|
|
1020
|
-
});
|
|
1021
|
-
});
|
|
1022
|
-
|
|
1023
|
-
/**
|
|
1024
|
-
* A standing filter acts when a message is first seen and when the user presses
|
|
1025
|
-
* "run rules now", and holds no claim afterwards. A later apply is the newer
|
|
1026
|
-
* event, so it moves the mail — and because nothing is suppressed, `applied`
|
|
1027
|
-
* counts moves that happened rather than moves that were skipped (reader #497).
|
|
1028
|
-
*/
|
|
1029
|
-
describe("a user-initiated apply outranks every standing filter (reader #497)", () => {
|
|
1030
|
-
it("moves a hand-picked selection a standing filter claims for a different mailbox", async () => {
|
|
1031
|
-
const store = createMemoryVectorStore();
|
|
1032
|
-
const picked = ["msg-1", "msg-2", "msg-3"];
|
|
1033
|
-
await store.upsert(picked.map((id) => bodyChunk(id, ANCHOR_VECTOR)));
|
|
1034
|
-
const p = predicate({ actionMailboxId: "mbox-projects" });
|
|
1035
|
-
// The standing rule that filed all three into Archive in the first place —
|
|
1036
|
-
// moving them back out is the whole point of the button.
|
|
1037
|
-
const standing = filterItem({
|
|
1038
|
-
filterId: "filter-github",
|
|
1039
|
-
ruleChangedAt: 1_000,
|
|
1040
|
-
actionChangedAt: 1_000,
|
|
1041
|
-
actionMailboxId: "mbox-archive",
|
|
1042
|
-
literalClauses: [{ field: "Subject", value: "reservation" }],
|
|
1043
|
-
});
|
|
1044
|
-
|
|
1045
|
-
const { messageIds: matched } = await matchAccepted(
|
|
1046
|
-
matchDeps(store),
|
|
1047
|
-
ACCOUNT_CONFIG_ID,
|
|
1048
|
-
p,
|
|
1049
|
-
);
|
|
1050
|
-
const tracked = trackingClient({
|
|
1051
|
-
activeFilters: [standing],
|
|
1052
|
-
threadMessages: Object.fromEntries(
|
|
1053
|
-
picked.map((id) => [id, { subject: "Dinner reservation" }]),
|
|
1054
|
-
),
|
|
1055
|
-
});
|
|
1056
|
-
const mover = trackingMoveService();
|
|
1057
|
-
const result = await applyOrganize(
|
|
1058
|
-
{ client: tracked.client, moveService: mover.moveService },
|
|
1059
|
-
ACCOUNT_CONFIG_ID,
|
|
1060
|
-
matched,
|
|
1061
|
-
p,
|
|
1062
|
-
);
|
|
1063
|
-
|
|
1064
|
-
assert.deepEqual(
|
|
1065
|
-
mover.moves.map((move) => move.messageId).sort(),
|
|
1066
|
-
picked,
|
|
1067
|
-
"every hand-picked message moves",
|
|
1068
|
-
);
|
|
1069
|
-
assert.ok(
|
|
1070
|
-
mover.moves.every(
|
|
1071
|
-
(move) => move.destinationMailboxId === "mbox-projects",
|
|
1072
|
-
),
|
|
1073
|
-
"the destination is the one the user picked, not the standing filter's",
|
|
1074
|
-
);
|
|
1075
|
-
assert.equal(
|
|
1076
|
-
result.applied,
|
|
1077
|
-
mover.moves.length,
|
|
1078
|
-
"the reported count equals the moves that actually happened",
|
|
1079
|
-
);
|
|
1080
|
-
assert.equal(result.failed, 0);
|
|
1081
|
-
assert.equal(
|
|
1082
|
-
tracked.filterReads(),
|
|
1083
|
-
0,
|
|
1084
|
-
"the account's standing filters are not consulted at all",
|
|
1085
|
-
);
|
|
1086
|
-
});
|
|
1087
|
-
});
|
|
1088
|
-
|
|
1089
|
-
/**
|
|
1090
|
-
* A same-dimension embedding-model swap leaves every persisted FilterAnchor
|
|
1091
|
-
* stamped with the old model's id and its vector in the old model's space.
|
|
1092
|
-
* Index-time matching repairs that lazily (RFC 039 Decision 1a); the
|
|
1093
|
-
* back-apply widen must do the same, or it queries with a stale-model vector
|
|
1094
|
-
* and silently disagrees with the pipeline it claims to mirror (reader #399).
|
|
1095
|
-
*/
|
|
1096
|
-
describe("back-apply repairs a drifted anchor the way index-time matching does (reader #399)", () => {
|
|
1097
|
-
// Written under the previous model: same dimensions, different space, so
|
|
1098
|
-
// nothing but the id stamp distinguishes it from a usable anchor.
|
|
1099
|
-
const staleAnchor = (filterId: string): FilterAnchorItem => ({
|
|
1100
|
-
accountConfigId: ACCOUNT_CONFIG_ID,
|
|
1101
|
-
filterId,
|
|
1102
|
-
anchorEmbedding: ORTHOGONAL_VECTOR,
|
|
1103
|
-
anchorEmbeddingId: STALE_EMBEDDING_ID,
|
|
1104
|
-
anchorSourceText: ANCHOR_SOURCE_TEXT,
|
|
1105
|
-
anchorMessageId: "msg-anchor",
|
|
1106
|
-
createdAt: 0,
|
|
1107
|
-
updatedAt: 0,
|
|
1108
|
-
});
|
|
1109
|
-
|
|
1110
|
-
/** Every text embeds into the current model's space. */
|
|
1111
|
-
const currentModelDeps = (
|
|
1112
|
-
store: ReturnType<typeof createMemoryVectorStore>,
|
|
1113
|
-
anchors: FilterAnchorItem[],
|
|
1114
|
-
embed: (text: string) => Promise<number[]> = async () => ANCHOR_VECTOR,
|
|
1115
|
-
): OrganizeMatchDeps & { anchorPuts: CreateFilterAnchorInput[] } => {
|
|
1116
|
-
const anchorPuts: CreateFilterAnchorInput[] = [];
|
|
1117
|
-
return {
|
|
1118
|
-
semantic: () => ({
|
|
1119
|
-
buildAnchor: async () => {
|
|
1120
|
-
throw new Error("a persisted anchor must be repaired, not replaced");
|
|
1121
|
-
},
|
|
1122
|
-
vectorStore: store,
|
|
1123
|
-
embed,
|
|
1124
|
-
embeddingId: CURRENT_EMBEDDING_ID,
|
|
1125
|
-
}),
|
|
1126
|
-
listAccountFilterMessages: async () => ({ items: [] }),
|
|
1127
|
-
filterAnchors: {
|
|
1128
|
-
get: async () => null,
|
|
1129
|
-
listByAccountConfig: async () => anchors,
|
|
1130
|
-
put: async (input: CreateFilterAnchorInput) => {
|
|
1131
|
-
anchorPuts.push(input);
|
|
1132
|
-
return { ...input, createdAt: 0, updatedAt: 1 };
|
|
1133
|
-
},
|
|
1134
|
-
},
|
|
1135
|
-
anchorPuts,
|
|
1136
|
-
};
|
|
1137
|
-
};
|
|
1138
|
-
|
|
1139
|
-
it("matchSemantic re-embeds a drifted persisted anchor instead of querying with the stale vector", async () => {
|
|
1140
|
-
const store = createMemoryVectorStore();
|
|
1141
|
-
await store.upsert([
|
|
1142
|
-
bodyChunk("msg-1", ANCHOR_VECTOR),
|
|
1143
|
-
bodyChunk("msg-stale-space", ORTHOGONAL_VECTOR),
|
|
1144
|
-
]);
|
|
1145
|
-
const deps = currentModelDeps(store, [staleAnchor("filter-a")]);
|
|
1146
|
-
|
|
1147
|
-
const { messageIds } = await matchAccepted(
|
|
1148
|
-
deps,
|
|
1149
|
-
ACCOUNT_CONFIG_ID,
|
|
1150
|
-
predicate(),
|
|
1151
|
-
);
|
|
1152
|
-
|
|
1153
|
-
assert.deepEqual(
|
|
1154
|
-
messageIds,
|
|
1155
|
-
["msg-1"],
|
|
1156
|
-
"the widen runs on the re-embedded anchor, not the stale-space vector that would have matched msg-stale-space",
|
|
1157
|
-
);
|
|
1158
|
-
assert.deepEqual(
|
|
1159
|
-
deps.anchorPuts.map((put) => [put.filterId, put.anchorEmbeddingId]),
|
|
1160
|
-
[["filter-a", CURRENT_EMBEDDING_ID]],
|
|
1161
|
-
"the repair is written back in place under the current model's id",
|
|
1162
|
-
);
|
|
1163
|
-
assert.deepEqual(deps.anchorPuts[0].anchorEmbedding, ANCHOR_VECTOR);
|
|
1164
|
-
assert.equal(deps.anchorPuts[0].anchorSourceText, ANCHOR_SOURCE_TEXT);
|
|
1165
|
-
});
|
|
1166
|
-
});
|
|
1167
|
-
|
|
1168
|
-
describe("matchOrganize with ListId and FromDomain clauses", () => {
|
|
1169
|
-
const senderChunk = (
|
|
1170
|
-
messageId: string,
|
|
1171
|
-
fromEmail: string,
|
|
1172
|
-
over: Partial<ChunkMetadata> = {},
|
|
1173
|
-
): VectorRecord => ({
|
|
1174
|
-
chunkId: `${messageId}#sender`,
|
|
1175
|
-
vector: ANCHOR_VECTOR,
|
|
1176
|
-
metadata: metadata({
|
|
1177
|
-
messageId,
|
|
1178
|
-
chunkType: "sender",
|
|
1179
|
-
textPreview: fromEmail,
|
|
1180
|
-
...over,
|
|
1181
|
-
}),
|
|
1182
|
-
});
|
|
1183
|
-
|
|
1184
|
-
it("matches a ListId clause exactly off the vector-free projection", async () => {
|
|
1185
|
-
const deps = vectorlessDeps([
|
|
1186
|
-
candidate("msg-1", { listId: "weekly.news.example.com" }),
|
|
1187
|
-
candidate("msg-2", { listId: "news.example.com" }),
|
|
1188
|
-
candidate("msg-3", {}),
|
|
1189
|
-
]);
|
|
1190
|
-
|
|
1191
|
-
const { messageIds } = await matchAccepted(deps, ACCOUNT_CONFIG_ID, {
|
|
1192
|
-
...predicate(),
|
|
1193
|
-
anchorMessageId: "None",
|
|
1194
|
-
literalClauses: [{ field: "ListId", value: "weekly.news.example.com" }],
|
|
1195
|
-
});
|
|
1196
|
-
|
|
1197
|
-
assert.deepEqual(messageIds, ["msg-1"]);
|
|
1198
|
-
});
|
|
1199
|
-
|
|
1200
|
-
it("matches a FromDomain clause public-suffix aware off the vector-free projection", async () => {
|
|
1201
|
-
const deps = vectorlessDeps([
|
|
1202
|
-
candidate("msg-1", { from: "notifications@github.com" }),
|
|
1203
|
-
candidate("msg-2", { from: "ci@sub.github.com" }),
|
|
1204
|
-
candidate("msg-3", { from: "attacker@github.com.evil.example" }),
|
|
1205
|
-
]);
|
|
1206
|
-
|
|
1207
|
-
const { messageIds } = await matchAccepted(deps, ACCOUNT_CONFIG_ID, {
|
|
1208
|
-
...predicate(),
|
|
1209
|
-
anchorMessageId: "None",
|
|
1210
|
-
literalClauses: [{ field: "FromDomain", value: "github.com" }],
|
|
1211
|
-
});
|
|
1212
|
-
|
|
1213
|
-
assert.deepEqual(messageIds, ["msg-1", "msg-2"]);
|
|
1214
|
-
});
|
|
1215
|
-
|
|
1216
|
-
it("round-trips ListId and FromDomain through the semantic-arm chunk projection", async () => {
|
|
1217
|
-
const store = createMemoryVectorStore();
|
|
1218
|
-
await store.upsert([
|
|
1219
|
-
senderChunk("msg-1", "ci@github.com", {
|
|
1220
|
-
listId: "actions.github.com",
|
|
1221
|
-
}),
|
|
1222
|
-
bodyChunk("msg-1", ANCHOR_VECTOR, { listId: "actions.github.com" }),
|
|
1223
|
-
senderChunk("msg-2", "hi@othersender.example", {
|
|
1224
|
-
listId: "other.list.example",
|
|
1225
|
-
}),
|
|
1226
|
-
bodyChunk("msg-2", ANCHOR_VECTOR, { listId: "other.list.example" }),
|
|
1227
|
-
]);
|
|
1228
|
-
|
|
1229
|
-
const byListId = await matchAccepted(matchDeps(store), ACCOUNT_CONFIG_ID, {
|
|
1230
|
-
...predicate(),
|
|
1231
|
-
literalClauses: [{ field: "ListId", value: "actions.github.com" }],
|
|
1232
|
-
});
|
|
1233
|
-
assert.deepEqual(byListId.messageIds, ["msg-1"]);
|
|
1234
|
-
|
|
1235
|
-
const byFromDomain = await matchAccepted(
|
|
1236
|
-
matchDeps(store),
|
|
1237
|
-
ACCOUNT_CONFIG_ID,
|
|
1238
|
-
{
|
|
1239
|
-
...predicate(),
|
|
1240
|
-
literalClauses: [{ field: "FromDomain", value: "github.com" }],
|
|
1241
|
-
},
|
|
1242
|
-
);
|
|
1243
|
-
assert.deepEqual(byFromDomain.messageIds, ["msg-1"]);
|
|
1244
|
-
});
|
|
1245
|
-
|
|
1246
|
-
it("applies a ListId predicate to exactly the previewed set (preview == apply)", async () => {
|
|
1247
|
-
const deps = vectorlessDeps([
|
|
1248
|
-
candidate("msg-1", { listId: "weekly.news.example.com" }),
|
|
1249
|
-
candidate("msg-2", { listId: "weekly.news.example.com" }),
|
|
1250
|
-
candidate("msg-3", { listId: "other.example.com" }),
|
|
1251
|
-
]);
|
|
1252
|
-
const p = predicate({
|
|
1253
|
-
anchorMessageId: "None",
|
|
1254
|
-
actionLabelId: "lbl-list",
|
|
1255
|
-
literalClauses: [{ field: "ListId", value: "weekly.news.example.com" }],
|
|
1256
|
-
});
|
|
1257
|
-
|
|
1258
|
-
const previewed = await matchAccepted(deps, ACCOUNT_CONFIG_ID, p);
|
|
1259
|
-
const applied = await matchAccepted(deps, ACCOUNT_CONFIG_ID, p);
|
|
1260
|
-
assert.deepEqual(previewed, applied);
|
|
1261
|
-
assert.deepEqual(previewed.messageIds, ["msg-1", "msg-2"]);
|
|
1262
|
-
|
|
1263
|
-
const tracked = trackingClient();
|
|
1264
|
-
const result = await applyOrganize(
|
|
1265
|
-
{ client: tracked.client },
|
|
1266
|
-
ACCOUNT_CONFIG_ID,
|
|
1267
|
-
applied.messageIds,
|
|
1268
|
-
p,
|
|
1269
|
-
);
|
|
1270
|
-
|
|
1271
|
-
assert.equal(result.applied, 2);
|
|
1272
|
-
assert.equal(result.failed, 0);
|
|
1273
|
-
assert.deepEqual(tracked.labeled.map((row) => row.messageId).sort(), [
|
|
1274
|
-
"msg-1",
|
|
1275
|
-
"msg-2",
|
|
1276
|
-
]);
|
|
1277
|
-
});
|
|
1278
|
-
});
|
|
1279
|
-
|
|
1280
|
-
/**
|
|
1281
|
-
* The corpus read a real deployment gets: the store evaluates the terms and
|
|
1282
|
-
* the caller pages the MATCHES. Modelled here rather than stubbed with a fixed
|
|
1283
|
-
* array so a matcher that reads only the newest page cannot pass.
|
|
1284
|
-
*/
|
|
1285
|
-
const storeBackedDeps = (
|
|
1286
|
-
rows: readonly OrganizeCandidate[],
|
|
1287
|
-
): OrganizeMatchDeps & { pagesRead: () => number } => {
|
|
1288
|
-
let pagesRead = 0;
|
|
1289
|
-
const termMatches = (
|
|
1290
|
-
term: { field: string; contains: string },
|
|
1291
|
-
row: OrganizeCandidate,
|
|
1292
|
-
): boolean => {
|
|
1293
|
-
const needle = term.contains.toLowerCase();
|
|
1294
|
-
if (term.field === "subject") {
|
|
1295
|
-
return row.message.subject.toLowerCase().includes(needle);
|
|
1296
|
-
}
|
|
1297
|
-
if (term.field === "listId") {
|
|
1298
|
-
return row.message.listId.toLowerCase().includes(needle);
|
|
1299
|
-
}
|
|
1300
|
-
return `${row.message.fromName} ${row.message.from}`
|
|
1301
|
-
.toLowerCase()
|
|
1302
|
-
.includes(needle);
|
|
1303
|
-
};
|
|
1304
|
-
return {
|
|
1305
|
-
semantic: () => {
|
|
1306
|
-
throw moduleNotFound();
|
|
1307
|
-
},
|
|
1308
|
-
listAccountFilterMessages: async (_accountConfigId, query) => {
|
|
1309
|
-
pagesRead += 1;
|
|
1310
|
-
const narrowed = rows.filter((row) =>
|
|
1311
|
-
query.terms.length === 0
|
|
1312
|
-
? true
|
|
1313
|
-
: query.operator === "or"
|
|
1314
|
-
? query.terms.some((term) => termMatches(term, row))
|
|
1315
|
-
: query.terms.every((term) => termMatches(term, row)),
|
|
1316
|
-
);
|
|
1317
|
-
const offset = query.continuationToken
|
|
1318
|
-
? Number(query.continuationToken)
|
|
1319
|
-
: 0;
|
|
1320
|
-
const items = narrowed.slice(offset, offset + query.limit);
|
|
1321
|
-
const next = offset + items.length;
|
|
1322
|
-
return {
|
|
1323
|
-
items,
|
|
1324
|
-
continuationToken: next < narrowed.length ? String(next) : undefined,
|
|
1325
|
-
};
|
|
1326
|
-
},
|
|
1327
|
-
filterAnchors: {
|
|
1328
|
-
get: async () => null,
|
|
1329
|
-
listByAccountConfig: async () => [],
|
|
1330
|
-
put: async () => {
|
|
1331
|
-
throw new Error("unreachable");
|
|
1332
|
-
},
|
|
1333
|
-
},
|
|
1334
|
-
pagesRead: () => pagesRead,
|
|
1335
|
-
};
|
|
1336
|
-
};
|
|
1337
|
-
|
|
1338
|
-
describe("literal matching bounds the result, not the corpus (reader #459)", () => {
|
|
1339
|
-
const quietSender = candidate("quiet-sender", {
|
|
1340
|
-
from: "noreply@bank.example",
|
|
1341
|
-
fromName: "Statements",
|
|
1342
|
-
subject: "Your March statement",
|
|
1343
|
-
});
|
|
1344
|
-
const newerNoise = Array.from({ length: ORGANIZE_MATCH_LIMIT + 1 }, (_, i) =>
|
|
1345
|
-
candidate(`noise-${i}`, {
|
|
1346
|
-
from: "digest@other.example",
|
|
1347
|
-
fromName: "Digest",
|
|
1348
|
-
subject: `Daily digest ${i}`,
|
|
1349
|
-
}),
|
|
1350
|
-
);
|
|
1351
|
-
|
|
1352
|
-
it("matches a domain rule against mail older than a whole page of newer misses", async () => {
|
|
1353
|
-
const deps = storeBackedDeps([...newerNoise, quietSender]);
|
|
1354
|
-
|
|
1355
|
-
const { messageIds } = await matchAccepted(
|
|
1356
|
-
deps,
|
|
1357
|
-
ACCOUNT_CONFIG_ID,
|
|
1358
|
-
predicate({
|
|
1359
|
-
anchorMessageId: "None",
|
|
1360
|
-
literalClauses: [{ field: "FromDomain", value: "bank.example" }],
|
|
1361
|
-
}),
|
|
1362
|
-
);
|
|
1363
|
-
|
|
1364
|
-
assert.deepEqual(messageIds, ["quiet-sender"]);
|
|
1365
|
-
});
|
|
1366
|
-
|
|
1367
|
-
it("reads one narrowed page rather than paging the whole mailbox", async () => {
|
|
1368
|
-
const deps = storeBackedDeps([...newerNoise, quietSender]);
|
|
1369
|
-
|
|
1370
|
-
await matchAccepted(
|
|
1371
|
-
deps,
|
|
1372
|
-
ACCOUNT_CONFIG_ID,
|
|
1373
|
-
predicate({
|
|
1374
|
-
anchorMessageId: "None",
|
|
1375
|
-
literalClauses: [{ field: "From", value: "noreply@bank.example" }],
|
|
1376
|
-
}),
|
|
1377
|
-
);
|
|
1378
|
-
|
|
1379
|
-
assert.equal(deps.pagesRead(), 1);
|
|
1380
|
-
});
|
|
1381
|
-
|
|
1382
|
-
it("still refines what the store returned: a term is a narrowing, not the verdict", async () => {
|
|
1383
|
-
const deps = storeBackedDeps([
|
|
1384
|
-
candidate("spoofed", { from: "billing@bank.example.evil.test" }),
|
|
1385
|
-
quietSender,
|
|
1386
|
-
]);
|
|
1387
|
-
|
|
1388
|
-
const { messageIds } = await matchAccepted(
|
|
1389
|
-
deps,
|
|
1390
|
-
ACCOUNT_CONFIG_ID,
|
|
1391
|
-
predicate({
|
|
1392
|
-
anchorMessageId: "None",
|
|
1393
|
-
literalClauses: [{ field: "FromDomain", value: "bank.example" }],
|
|
1394
|
-
}),
|
|
1395
|
-
);
|
|
1396
|
-
|
|
1397
|
-
assert.deepEqual(messageIds, ["quiet-sender"]);
|
|
1398
|
-
});
|
|
1399
|
-
|
|
1400
|
-
it("caps the matches at the limit when more of the corpus matches", async () => {
|
|
1401
|
-
const matching = Array.from({ length: ORGANIZE_MATCH_LIMIT * 2 }, (_, i) =>
|
|
1402
|
-
candidate(`hit-${i}`, { from: "noreply@bank.example" }),
|
|
1403
|
-
);
|
|
1404
|
-
const deps = storeBackedDeps(matching);
|
|
1405
|
-
|
|
1406
|
-
const { messageIds } = await matchAccepted(
|
|
1407
|
-
deps,
|
|
1408
|
-
ACCOUNT_CONFIG_ID,
|
|
1409
|
-
predicate({
|
|
1410
|
-
anchorMessageId: "None",
|
|
1411
|
-
literalClauses: [{ field: "FromDomain", value: "bank.example" }],
|
|
1412
|
-
}),
|
|
1413
|
-
);
|
|
1414
|
-
|
|
1415
|
-
assert.equal(messageIds.length, ORGANIZE_MATCH_LIMIT);
|
|
1416
|
-
});
|
|
1417
|
-
|
|
1418
|
-
it("never queries for clauses nothing can satisfy", async () => {
|
|
1419
|
-
const deps = storeBackedDeps([quietSender]);
|
|
1420
|
-
|
|
1421
|
-
const { messageIds } = await matchAccepted(
|
|
1422
|
-
deps,
|
|
1423
|
-
ACCOUNT_CONFIG_ID,
|
|
1424
|
-
predicate({
|
|
1425
|
-
anchorMessageId: "None",
|
|
1426
|
-
literalClauses: [{ field: "From", value: " " }],
|
|
1427
|
-
}),
|
|
1428
|
-
);
|
|
1429
|
-
|
|
1430
|
-
assert.deepEqual(messageIds, []);
|
|
1431
|
-
assert.equal(deps.pagesRead(), 0);
|
|
1432
|
-
});
|
|
1433
|
-
|
|
1434
|
-
it("counts one match for mail filed in two folders", async () => {
|
|
1435
|
-
const deps = storeBackedDeps([
|
|
1436
|
-
candidate("filed-twice", { from: "noreply@bank.example" }),
|
|
1437
|
-
candidate("filed-twice", { from: "noreply@bank.example" }),
|
|
1438
|
-
]);
|
|
1439
|
-
|
|
1440
|
-
const { messageIds } = await matchAccepted(
|
|
1441
|
-
deps,
|
|
1442
|
-
ACCOUNT_CONFIG_ID,
|
|
1443
|
-
predicate({
|
|
1444
|
-
anchorMessageId: "None",
|
|
1445
|
-
literalClauses: [{ field: "FromDomain", value: "bank.example" }],
|
|
1446
|
-
}),
|
|
1447
|
-
);
|
|
1448
|
-
|
|
1449
|
-
assert.deepEqual(messageIds, ["filed-twice"]);
|
|
1450
|
-
});
|
|
1451
|
-
});
|