@remit/backend 0.0.142 → 0.0.144

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (269) hide show
  1. package/dist/dev-server/content-handler.d.ts +38 -0
  2. package/dist/dev-server/content-handler.js +46 -0
  3. package/dist/src/api.d.ts +7 -0
  4. package/dist/src/api.js +144 -0
  5. package/dist/src/auth.d.ts +30 -0
  6. package/dist/src/auth.js +73 -0
  7. package/dist/src/config/msoauth.d.ts +30 -0
  8. package/dist/src/config/msoauth.js +30 -0
  9. package/{src/data-backend.ts → dist/src/data-backend.d.ts} +2 -4
  10. package/dist/src/data-backend.js +12 -0
  11. package/{src/derive/autoMoved.ts → dist/src/derive/autoMoved.d.ts} +1 -29
  12. package/dist/src/derive/autoMoved.js +49 -0
  13. package/dist/src/derive/contentSignature.d.ts +41 -0
  14. package/{src/derive/contentSignature.ts → dist/src/derive/contentSignature.js} +27 -72
  15. package/dist/src/derive/contentUrl.d.ts +46 -0
  16. package/dist/src/derive/contentUrl.js +46 -0
  17. package/{src/derive/deriveMuted.ts → dist/src/derive/deriveMuted.d.ts} +1 -3
  18. package/dist/src/derive/deriveMuted.js +7 -0
  19. package/dist/src/derive/enrichThreadRows.d.ts +118 -0
  20. package/dist/src/derive/enrichThreadRows.js +151 -0
  21. package/dist/src/derive/filterThreadCriteria.d.ts +23 -0
  22. package/dist/src/derive/filterThreadCriteria.js +24 -0
  23. package/{src/derive/pendingMoveCounts.ts → dist/src/derive/pendingMoveCounts.d.ts} +5 -53
  24. package/dist/src/derive/pendingMoveCounts.js +53 -0
  25. package/{src/derive/senderTrust.ts → dist/src/derive/senderTrust.d.ts} +1 -9
  26. package/dist/src/derive/senderTrust.js +19 -0
  27. package/dist/src/error.d.ts +21 -0
  28. package/dist/src/error.js +85 -0
  29. package/dist/src/handlers/account-guards.d.ts +36 -0
  30. package/dist/src/handlers/account-guards.js +107 -0
  31. package/dist/src/handlers/account-oauth.d.ts +14 -0
  32. package/dist/src/handlers/account-oauth.js +387 -0
  33. package/dist/src/handlers/account-overrides.d.ts +53 -0
  34. package/dist/src/handlers/account-overrides.js +233 -0
  35. package/dist/src/handlers/account-signature.d.ts +28 -0
  36. package/dist/src/handlers/account-signature.js +85 -0
  37. package/dist/src/handlers/account.d.ts +6 -0
  38. package/dist/src/handlers/account.js +350 -0
  39. package/dist/src/handlers/address.d.ts +35 -0
  40. package/dist/src/handlers/address.js +141 -0
  41. package/dist/src/handlers/calendar-event.d.ts +70 -0
  42. package/dist/src/handlers/calendar-event.js +425 -0
  43. package/dist/src/handlers/calendar-feed.d.ts +47 -0
  44. package/dist/src/handlers/calendar-feed.js +192 -0
  45. package/dist/src/handlers/calendar-suggestion.d.ts +47 -0
  46. package/dist/src/handlers/calendar-suggestion.js +212 -0
  47. package/dist/src/handlers/calendar.d.ts +116 -0
  48. package/dist/src/handlers/calendar.js +330 -0
  49. package/dist/src/handlers/config.d.ts +43 -0
  50. package/dist/src/handlers/config.js +176 -0
  51. package/dist/src/handlers/ensure-account-config.d.ts +9 -0
  52. package/dist/src/handlers/ensure-account-config.js +27 -0
  53. package/dist/src/handlers/filter.d.ts +3 -0
  54. package/dist/src/handlers/filter.js +103 -0
  55. package/dist/src/handlers/folder-role-appointments.d.ts +68 -0
  56. package/dist/src/handlers/folder-role-appointments.js +168 -0
  57. package/dist/src/handlers/folder-role-labels.d.ts +26 -0
  58. package/dist/src/handlers/folder-role-labels.js +40 -0
  59. package/dist/src/handlers/folder-role.d.ts +9 -0
  60. package/dist/src/handlers/folder-role.js +40 -0
  61. package/dist/src/handlers/index.d.ts +2 -0
  62. package/dist/src/handlers/index.js +60 -0
  63. package/dist/src/handlers/label.d.ts +43 -0
  64. package/dist/src/handlers/label.js +114 -0
  65. package/dist/src/handlers/mailbox-bindings.d.ts +41 -0
  66. package/dist/src/handlers/mailbox-bindings.js +61 -0
  67. package/dist/src/handlers/mailbox.d.ts +23 -0
  68. package/dist/src/handlers/mailbox.js +219 -0
  69. package/dist/src/handlers/me.d.ts +2 -0
  70. package/dist/src/handlers/me.js +138 -0
  71. package/dist/src/handlers/message.d.ts +84 -0
  72. package/dist/src/handlers/message.js +607 -0
  73. package/dist/src/handlers/organize.d.ts +10 -0
  74. package/dist/src/handlers/organize.js +112 -0
  75. package/dist/src/handlers/outbox-attachment.d.ts +23 -0
  76. package/dist/src/handlers/outbox-attachment.js +81 -0
  77. package/dist/src/handlers/outbox.d.ts +4 -0
  78. package/dist/src/handlers/outbox.js +144 -0
  79. package/dist/src/handlers/sync-progress.d.ts +46 -0
  80. package/dist/src/handlers/sync-progress.js +63 -0
  81. package/dist/src/handlers/sync.d.ts +28 -0
  82. package/dist/src/handlers/sync.js +82 -0
  83. package/dist/src/handlers/system-update.d.ts +2 -0
  84. package/dist/src/handlers/system-update.js +172 -0
  85. package/dist/src/handlers/thread.d.ts +148 -0
  86. package/dist/src/handlers/thread.js +233 -0
  87. package/dist/src/handlers/unified-threads.d.ts +247 -0
  88. package/dist/src/handlers/unified-threads.js +409 -0
  89. package/{src/handlers/vip-suggestions.ts → dist/src/handlers/vip-suggestions.d.ts} +1 -13
  90. package/dist/src/handlers/vip-suggestions.js +16 -0
  91. package/dist/src/index.d.ts +2 -0
  92. package/dist/src/index.js +37 -0
  93. package/{src/json.ts → dist/src/json.d.ts} +1 -4
  94. package/dist/src/json.js +10 -0
  95. package/dist/src/jwt-auth.d.ts +18 -0
  96. package/dist/src/jwt-auth.js +97 -0
  97. package/dist/src/request-context.d.ts +9 -0
  98. package/dist/src/request-context.js +22 -0
  99. package/dist/src/request.d.ts +3 -0
  100. package/dist/src/request.js +7 -0
  101. package/dist/src/response.d.ts +27 -0
  102. package/dist/src/response.js +87 -0
  103. package/dist/src/service/account-deletion.d.ts +22 -0
  104. package/dist/src/service/account-deletion.js +39 -0
  105. package/dist/src/service/compose-sqlite.d.ts +2 -0
  106. package/dist/src/service/compose-sqlite.js +11 -0
  107. package/dist/src/service/create-remit-client.d.ts +95 -0
  108. package/dist/src/service/create-remit-client.js +207 -0
  109. package/dist/src/service/data-client.d.ts +12 -0
  110. package/dist/src/service/data-client.js +44 -0
  111. package/dist/src/service/export-identity.d.ts +3 -0
  112. package/dist/src/service/export-identity.js +35 -0
  113. package/dist/src/service/fire-and-forget.d.ts +54 -0
  114. package/dist/src/service/fire-and-forget.js +38 -0
  115. package/dist/src/service/organize.d.ts +184 -0
  116. package/dist/src/service/organize.js +384 -0
  117. package/dist/src/service/refusal.d.ts +4 -0
  118. package/dist/src/service/refusal.js +20 -0
  119. package/dist/src/service/sender-category.d.ts +51 -0
  120. package/{src/service/sender-category.ts → dist/src/service/sender-category.js} +40 -81
  121. package/dist/src/service/sqs.d.ts +1 -0
  122. package/{src/service/sqs.ts → dist/src/service/sqs.js} +0 -1
  123. package/dist/src/service/trigger-sync.d.ts +51 -0
  124. package/dist/src/service/trigger-sync.js +39 -0
  125. package/dist/src/types.d.ts +49 -0
  126. package/dist/src/types.js +1 -0
  127. package/package.json +113 -98
  128. package/.env.test +0 -4
  129. package/dev-server/content-auth.test.ts +0 -113
  130. package/dev-server/content-auth.ts +0 -54
  131. package/dev-server/content-handler.test.ts +0 -105
  132. package/dev-server/content-handler.ts +0 -79
  133. package/dev-server/content-path.test.ts +0 -37
  134. package/dev-server/content-path.ts +0 -27
  135. package/dev-server/cors.test.ts +0 -48
  136. package/dev-server/cors.ts +0 -25
  137. package/dev-server/keep-alive.test.ts +0 -24
  138. package/dev-server/lambda-helpers.ts +0 -69
  139. package/dev-server/log-output.test.ts +0 -173
  140. package/dev-server/oauth-callback-route.test.ts +0 -66
  141. package/dev-server/relational-health.test.ts +0 -83
  142. package/dev-server/relational-health.ts +0 -39
  143. package/dev-server/server.ts +0 -384
  144. package/dev-server/sync-age.test.ts +0 -91
  145. package/dev-server/sync-age.ts +0 -55
  146. package/dev-server/upload-handler.ts +0 -132
  147. package/dev-server/upload-route.test.ts +0 -196
  148. package/scripts/backfill-classification.ts +0 -91
  149. package/scripts/backfill-list-id.ts +0 -83
  150. package/scripts/config-save.ts +0 -128
  151. package/src/auth.test.ts +0 -12
  152. package/src/auth.ts +0 -92
  153. package/src/config/msoauth.test.ts +0 -48
  154. package/src/config/msoauth.ts +0 -48
  155. package/src/data-backend.test.ts +0 -23
  156. package/src/derive/autoMoved.test.ts +0 -104
  157. package/src/derive/contentSignature.test.ts +0 -161
  158. package/src/derive/contentUrl.test.ts +0 -172
  159. package/src/derive/contentUrl.ts +0 -86
  160. package/src/derive/deriveMuted.test.ts +0 -34
  161. package/src/derive/enrichThreadRows.test.ts +0 -360
  162. package/src/derive/enrichThreadRows.ts +0 -234
  163. package/src/derive/filterThreadCriteria.test.ts +0 -126
  164. package/src/derive/filterThreadCriteria.ts +0 -51
  165. package/src/derive/senderTrust.test.ts +0 -67
  166. package/src/error.test.ts +0 -201
  167. package/src/error.ts +0 -123
  168. package/src/handlers/account-guards.test.ts +0 -67
  169. package/src/handlers/account-guards.ts +0 -169
  170. package/src/handlers/account-oauth-callback.test.ts +0 -155
  171. package/src/handlers/account-oauth-consent.test.ts +0 -336
  172. package/src/handlers/account-oauth-start.test.ts +0 -191
  173. package/src/handlers/account-oauth.test.ts +0 -439
  174. package/src/handlers/account-oauth.ts +0 -635
  175. package/src/handlers/account-overrides.test.ts +0 -88
  176. package/src/handlers/account-overrides.ts +0 -323
  177. package/src/handlers/account-ownership.ts +0 -25
  178. package/src/handlers/account-signature.ts +0 -117
  179. package/src/handlers/account-synced-services.test.ts +0 -245
  180. package/src/handlers/account.ts +0 -552
  181. package/src/handlers/address.test.ts +0 -381
  182. package/src/handlers/address.ts +0 -200
  183. package/src/handlers/calendar-event.test.ts +0 -685
  184. package/src/handlers/calendar-event.ts +0 -738
  185. package/src/handlers/calendar-feed-validators.test.ts +0 -228
  186. package/src/handlers/calendar-feed.test.ts +0 -439
  187. package/src/handlers/calendar-feed.ts +0 -291
  188. package/src/handlers/calendar-sqlite-fixture.ts +0 -36
  189. package/src/handlers/calendar-suggestion.test.ts +0 -1269
  190. package/src/handlers/calendar-suggestion.ts +0 -425
  191. package/src/handlers/calendar.test.ts +0 -1080
  192. package/src/handlers/calendar.ts +0 -517
  193. package/src/handlers/config.test.ts +0 -245
  194. package/src/handlers/config.ts +0 -332
  195. package/src/handlers/ensure-account-config.ts +0 -30
  196. package/src/handlers/filter.test.ts +0 -644
  197. package/src/handlers/filter.ts +0 -518
  198. package/src/handlers/folder-role-appointments.test.ts +0 -438
  199. package/src/handlers/folder-role-appointments.ts +0 -249
  200. package/src/handlers/folder-role-labels.ts +0 -63
  201. package/src/handlers/folder-role.test.ts +0 -209
  202. package/src/handlers/folder-role.ts +0 -80
  203. package/src/handlers/index.ts +0 -83
  204. package/src/handlers/label.test.ts +0 -112
  205. package/src/handlers/label.ts +0 -210
  206. package/src/handlers/mailbox-bindings.ts +0 -111
  207. package/src/handlers/mailbox-rename-refusal.test.ts +0 -247
  208. package/src/handlers/mailbox-settled-target.test.ts +0 -251
  209. package/src/handlers/mailbox.test.ts +0 -280
  210. package/src/handlers/mailbox.ts +0 -581
  211. package/src/handlers/me.test.ts +0 -81
  212. package/src/handlers/me.ts +0 -203
  213. package/src/handlers/message-read-backfill.test.ts +0 -103
  214. package/src/handlers/message.test.ts +0 -131
  215. package/src/handlers/message.ts +0 -1131
  216. package/src/handlers/organize.test.ts +0 -277
  217. package/src/handlers/organize.ts +0 -197
  218. package/src/handlers/outbox-attachment.test.ts +0 -765
  219. package/src/handlers/outbox-attachment.ts +0 -126
  220. package/src/handlers/outbox-list.test.ts +0 -179
  221. package/src/handlers/outbox.test.ts +0 -772
  222. package/src/handlers/outbox.ts +0 -258
  223. package/src/handlers/search.test.ts +0 -161
  224. package/src/handlers/search.ts +0 -105
  225. package/src/handlers/sync-progress.test.ts +0 -205
  226. package/src/handlers/sync-progress.ts +0 -99
  227. package/src/handlers/sync.test.ts +0 -168
  228. package/src/handlers/sync.ts +0 -118
  229. package/src/handlers/system-update.test.ts +0 -497
  230. package/src/handlers/system-update.ts +0 -230
  231. package/src/handlers/thread.test.ts +0 -396
  232. package/src/handlers/thread.ts +0 -427
  233. package/src/handlers/unified-threads-mail-sources.sqlite.test.ts +0 -197
  234. package/src/handlers/unified-threads.test.ts +0 -785
  235. package/src/handlers/unified-threads.ts +0 -669
  236. package/src/index.test.ts +0 -83
  237. package/src/index.ts +0 -228
  238. package/src/jwt-auth.test.ts +0 -152
  239. package/src/jwt-auth.ts +0 -132
  240. package/src/log-redaction.test.ts +0 -144
  241. package/src/openapi-spec.test.ts +0 -118
  242. package/src/request-context.test.ts +0 -61
  243. package/src/request-context.ts +0 -33
  244. package/src/request-pipeline.test.ts +0 -88
  245. package/src/request.ts +0 -10
  246. package/src/response.test.ts +0 -130
  247. package/src/response.ts +0 -144
  248. package/src/service/compose-sqlite.ts +0 -98
  249. package/src/service/config-import.ts +0 -39
  250. package/src/service/create-remit-client.ts +0 -521
  251. package/src/service/data-client.test.ts +0 -97
  252. package/src/service/data-client.ts +0 -58
  253. package/src/service/export-identity.ts +0 -39
  254. package/src/service/filter.ts +0 -40
  255. package/src/service/fire-and-forget.test.ts +0 -126
  256. package/src/service/fire-and-forget.ts +0 -79
  257. package/src/service/localhost-env-config.test.ts +0 -36
  258. package/src/service/organize-queue.ts +0 -98
  259. package/src/service/organize.test.ts +0 -1451
  260. package/src/service/organize.ts +0 -1025
  261. package/src/service/read-path-filter-config.test.ts +0 -243
  262. package/src/service/semantic-capability.test.ts +0 -78
  263. package/src/service/semantic-capability.ts +0 -95
  264. package/src/service/sender-category.test.ts +0 -235
  265. package/src/service/trigger-sync.test.ts +0 -205
  266. package/src/service/trigger-sync.ts +0 -100
  267. package/src/stdout-capture-fixture.ts +0 -40
  268. package/src/types.ts +0 -271
  269. package/tsconfig.json +0 -7
@@ -1,1451 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { afterEach, beforeEach, describe, it } from "node:test";
3
- import type {
4
- CreateFilterAnchorInput,
5
- FilterAnchorItem,
6
- FilterItem,
7
- } from "@remit/data-ports";
8
- import { NotFoundError } from "@remit/data-ports/errors";
9
- import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
10
- import type {
11
- AnchorPayload,
12
- ChunkMetadata,
13
- VectorRecord,
14
- } from "@remit/search-service";
15
- import { createMemoryVectorStore } from "@remit/search-service";
16
- import type { RemitClient } from "./data-client.js";
17
- import {
18
- applyOrganize,
19
- BODY_CONTENT_REJECTION_MESSAGE,
20
- matchOrganize,
21
- ORGANIZE_MATCH_LIMIT,
22
- type OrganizeCandidate,
23
- type OrganizeMatchDeps,
24
- type OrganizeMatched,
25
- type OrganizePredicate,
26
- } from "./organize.js";
27
- import {
28
- _resetSemanticCapabilityForTest,
29
- isSemanticSearchUnavailable,
30
- } from "./semantic-capability.js";
31
-
32
- const moduleNotFound = (): Error => {
33
- const error = new Error("Cannot find package 'sqlite-vec' imported from …");
34
- (error as Error & { code: string }).code = "ERR_MODULE_NOT_FOUND";
35
- return error;
36
- };
37
-
38
- const ACCOUNT_CONFIG_ID = "cfg-1";
39
- const ANCHOR_VECTOR = [1, 0, 0, 0];
40
- const ORTHOGONAL_VECTOR = [0, 1, 0, 0];
41
- /** The model the fixtures' embedder is currently configured with. */
42
- const CURRENT_EMBEDDING_ID = "test-model@4";
43
- /** The model a persisted anchor was written under before a same-dimension swap. */
44
- const STALE_EMBEDDING_ID = "older-model@4";
45
- const ANCHOR_SOURCE_TEXT = "book me a table";
46
-
47
- const anchorPayload: AnchorPayload = {
48
- anchorEmbedding: ANCHOR_VECTOR,
49
- anchorEmbeddingId: CURRENT_EMBEDDING_ID,
50
- anchorSourceText: ANCHOR_SOURCE_TEXT,
51
- };
52
-
53
- const metadata = (over: Partial<ChunkMetadata>): ChunkMetadata => ({
54
- messageId: "msg-x",
55
- threadId: "thread-x",
56
- accountConfigId: ACCOUNT_CONFIG_ID,
57
- mailboxIds: ["mbox-1"],
58
- chunkType: "body",
59
- sentDate: 1_700_000_000,
60
- isRead: false,
61
- hasAttachment: false,
62
- hasStars: false,
63
- ...over,
64
- });
65
-
66
- const bodyChunk = (
67
- messageId: string,
68
- vector: number[],
69
- over: Partial<ChunkMetadata> = {},
70
- ): VectorRecord => ({
71
- chunkId: `${messageId}#body`,
72
- vector,
73
- metadata: metadata({ messageId, ...over }),
74
- });
75
-
76
- const predicate = (
77
- over: Partial<OrganizePredicate> = {},
78
- ): OrganizePredicate => ({
79
- anchorMessageId: "msg-anchor",
80
- matchOperator: FilterMatchOperator.And,
81
- literalClauses: [],
82
- similarityThreshold: 0.75,
83
- actionLabelId: "None",
84
- actionMailboxId: "None",
85
- ...over,
86
- });
87
-
88
- /**
89
- * Run the matcher and assert it accepted the predicate, so every test that only
90
- * cares about the matched set reads the accepted arm directly. A rejection is a
91
- * result, not a throw (reader #463), so without this the two would be silently
92
- * interchangeable at the call site.
93
- */
94
- const matchAccepted = async (
95
- ...args: Parameters<typeof matchOrganize>
96
- ): Promise<OrganizeMatched> => {
97
- const result = await matchOrganize(...args);
98
- assert.ok(
99
- result.rejected === null,
100
- `expected an accepted match, got: ${result.rejected?.message}`,
101
- );
102
- return result;
103
- };
104
-
105
- /** A standing filter fixture — the account's already-existing rules. */
106
- const filterItem = (over: Partial<FilterItem> = {}): FilterItem => ({
107
- filterId: "filter-other",
108
- accountConfigId: ACCOUNT_CONFIG_ID,
109
- name: "other filter",
110
- scope: "Standing",
111
- state: FilterState.Active,
112
- disabledReason: "None",
113
- hasAnchor: false,
114
- ruleChangedAt: 0,
115
- actionChangedAt: 0,
116
- matchOperator: FilterMatchOperator.And,
117
- literalClauses: [],
118
- actionLabelId: "None",
119
- actionMailboxId: "None",
120
- createdAt: 0,
121
- updatedAt: 0,
122
- ...over,
123
- });
124
-
125
- /**
126
- * A client that records MessageLabel writes and blows up if the back-apply path
127
- * ever touches Filter/FilterAnchor — the RFC 034 guardrail: this scope never
128
- * persists a standing rule.
129
- *
130
- * `activeFilters`/`filterAnchorRows`/`threadMessages` model the account's
131
- * *other*, already-existing standing filters and message rows. A back-apply
132
- * never arbitrates against them (reader #497), so `filterReads` pins that the
133
- * Active filter set is not even consulted.
134
- */
135
- const trackingClient = (
136
- seed: {
137
- activeFilters?: FilterItem[];
138
- filterAnchorRows?: FilterAnchorItem[];
139
- threadMessages?: Record<
140
- string,
141
- { fromEmail?: string; fromName?: string; subject?: string }
142
- >;
143
- } = {},
144
- ) => {
145
- const labeled: Array<{ messageId: string; labelId: string }> = [];
146
- let filterWrites = 0;
147
- let filterReads = 0;
148
- let filterAnchorWrites = 0;
149
- const anchorPuts: CreateFilterAnchorInput[] = [];
150
- const activeFilters = seed.activeFilters ?? [];
151
- const filterAnchorRows = seed.filterAnchorRows ?? [];
152
- const threadMessages = seed.threadMessages ?? {};
153
- const client = {
154
- messageLabel: {
155
- apply: async (input: {
156
- messageId: string;
157
- labelId: string;
158
- accountConfigId: string;
159
- appliedByFilterId?: string;
160
- }) => {
161
- assert.equal(
162
- input.appliedByFilterId,
163
- undefined,
164
- "back-apply must never attribute a filter",
165
- );
166
- labeled.push({ messageId: input.messageId, labelId: input.labelId });
167
- return {} as never;
168
- },
169
- },
170
- message: {
171
- get: async (messageId: string) => ({ messageId, mailboxId: "mbox-src" }),
172
- },
173
- mailbox: {
174
- resolveAccountId: async () => "acct-1",
175
- },
176
- threadMessage: {
177
- get: async (_accountConfigId: string, messageId: string) => {
178
- const row = threadMessages[messageId];
179
- if (!row) {
180
- throw new NotFoundError("ThreadMessage not found");
181
- }
182
- return {
183
- threadMessageId: messageId,
184
- fromEmail: row.fromEmail,
185
- fromName: row.fromName,
186
- subject: row.subject,
187
- } as never;
188
- },
189
- },
190
- filter: {
191
- create: async () => {
192
- filterWrites += 1;
193
- return {} as never;
194
- },
195
- listByAccountAndState: async () => {
196
- filterReads += 1;
197
- return activeFilters;
198
- },
199
- refreshExpiry: async (filter: FilterItem) => filter,
200
- },
201
- filterAnchor: {
202
- put: async (input: CreateFilterAnchorInput) => {
203
- filterAnchorWrites += 1;
204
- anchorPuts.push(input);
205
- const row: FilterAnchorItem = { ...input, createdAt: 0, updatedAt: 1 };
206
- const at = filterAnchorRows.findIndex(
207
- (existing) => existing.filterId === input.filterId,
208
- );
209
- if (at === -1) filterAnchorRows.push(row);
210
- else filterAnchorRows[at] = row;
211
- return row;
212
- },
213
- get: async (_accountConfigId: string, filterId: string) =>
214
- filterAnchorRows.find((row) => row.filterId === filterId) ?? null,
215
- listByAccountConfig: async () => filterAnchorRows,
216
- },
217
- } as unknown as RemitClient;
218
- return {
219
- client,
220
- labeled,
221
- filterWrites: () => filterWrites,
222
- filterReads: () => filterReads,
223
- filterAnchorWrites: () => filterAnchorWrites,
224
- anchorPuts,
225
- };
226
- };
227
-
228
- /**
229
- * A stand-in for the local-first placement mover. `moveMessage` is idempotent
230
- * on the (messageId, destination) pair — mirroring the real
231
- * `PlacementMoveService` marker engine (#1297) — so a redelivered back-apply
232
- * re-issues the same move without double-enqueuing. Records every call and the
233
- * final resting mailbox per message.
234
- */
235
- const trackingMoveService = () => {
236
- const moves: Array<{
237
- messageId: string;
238
- destinationMailboxId: string;
239
- accountId: string;
240
- }> = [];
241
- const destinationOf = new Map<string, string>();
242
- let enqueues = 0;
243
- return {
244
- moveService: {
245
- moveMessage: async (
246
- _accountConfigId: string,
247
- messageId: string,
248
- destinationMailboxId: string,
249
- accountId: string,
250
- ): Promise<void> => {
251
- moves.push({ messageId, destinationMailboxId, accountId });
252
- if (destinationOf.get(messageId) === destinationMailboxId) return;
253
- destinationOf.set(messageId, destinationMailboxId);
254
- enqueues += 1;
255
- },
256
- } as unknown as import("@remit/mailbox-service").PlacementMoveService,
257
- moves,
258
- enqueues: () => enqueues,
259
- destinations: () => destinationOf,
260
- };
261
- };
262
-
263
- /**
264
- * Deps whose semantic side is the in-memory vector store — the vector-backed
265
- * deployment. `listAccountFilterMessages` returns the given corpus (empty by
266
- * default), and constructing the semantic side is tracked so a literal-only
267
- * predicate can be shown never to build it. `filterAnchorRows` seeds the
268
- * account's *persisted* FilterAnchor rows (empty by default) — the reverse
269
- * lookup back-apply's anchor consultation reads (reader #350) — and `embed`
270
- * is a deterministic stand-in for the configured embedding model, used only
271
- * by the exclusive-move precedence check.
272
- */
273
- const matchDeps = (
274
- store: ReturnType<typeof createMemoryVectorStore>,
275
- corpus: OrganizeCandidate[] = [],
276
- filterAnchorRows: FilterAnchorItem[] = [],
277
- ): OrganizeMatchDeps & { semanticBuilds: () => number } => {
278
- let semanticBuilds = 0;
279
- return {
280
- semantic: () => {
281
- semanticBuilds += 1;
282
- return {
283
- buildAnchor: async () => anchorPayload,
284
- vectorStore: store,
285
- embed: async (text: string) =>
286
- text.includes("reservation") ? ANCHOR_VECTOR : ORTHOGONAL_VECTOR,
287
- embeddingId: CURRENT_EMBEDDING_ID,
288
- };
289
- },
290
- listAccountFilterMessages: async () => ({ items: corpus }),
291
- filterAnchors: {
292
- get: async () => null,
293
- listByAccountConfig: async () => filterAnchorRows,
294
- put: async () => {
295
- throw new Error("matchDeps must not repair an anchor");
296
- },
297
- },
298
- semanticBuilds: () => semanticBuilds,
299
- };
300
- };
301
-
302
- /**
303
- * Deps modelling a deployment that ships no vector pipeline: any attempt to
304
- * build or use the semantic side raises the missing-module shape, and the
305
- * literal corpus is served from plain rows.
306
- */
307
- const vectorlessDeps = (
308
- corpus: OrganizeCandidate[],
309
- ): OrganizeMatchDeps & { semanticUsed: () => boolean } => {
310
- let semanticUsed = false;
311
- return {
312
- semantic: () => {
313
- semanticUsed = true;
314
- return {
315
- buildAnchor: async () => {
316
- throw moduleNotFound();
317
- },
318
- vectorStore: {
319
- query: async () => {
320
- throw moduleNotFound();
321
- },
322
- getByMessage: async () => {
323
- throw moduleNotFound();
324
- },
325
- },
326
- embed: async () => {
327
- throw moduleNotFound();
328
- },
329
- embeddingId: CURRENT_EMBEDDING_ID,
330
- };
331
- },
332
- listAccountFilterMessages: async () => ({ items: corpus }),
333
- filterAnchors: {
334
- get: async () => null,
335
- listByAccountConfig: async () => [],
336
- put: async () => {
337
- throw moduleNotFound();
338
- },
339
- },
340
- semanticUsed: () => semanticUsed,
341
- };
342
- };
343
-
344
- const candidate = (
345
- messageId: string,
346
- over: Partial<OrganizeCandidate["message"]> = {},
347
- ): OrganizeCandidate => ({
348
- messageId,
349
- message: {
350
- from: "",
351
- fromName: "",
352
- subject: "",
353
- text: "",
354
- listId: "",
355
- ...over,
356
- },
357
- });
358
-
359
- describe("matchOrganize", () => {
360
- it("returns every semantically matching message and excludes the misses", async () => {
361
- const store = createMemoryVectorStore();
362
- const matching = ["msg-1", "msg-2", "msg-3", "msg-4", "msg-5"];
363
- await store.upsert([
364
- ...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
365
- bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
366
- ]);
367
-
368
- const { messageIds, semanticUnavailable } = await matchAccepted(
369
- matchDeps(store),
370
- ACCOUNT_CONFIG_ID,
371
- predicate({ actionLabelId: "lbl-1" }),
372
- );
373
-
374
- assert.deepEqual([...messageIds].sort(), matching);
375
- assert.equal(semanticUnavailable, false);
376
- });
377
-
378
- it("matches nothing when the predicate has neither an anchor nor a clause", async () => {
379
- const store = createMemoryVectorStore();
380
- await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
381
-
382
- const { messageIds } = await matchAccepted(
383
- matchDeps(store),
384
- ACCOUNT_CONFIG_ID,
385
- {
386
- ...predicate(),
387
- anchorMessageId: "None",
388
- },
389
- );
390
-
391
- assert.deepEqual(messageIds, []);
392
- });
393
-
394
- it("refines the semantic set by literal clauses", async () => {
395
- const store = createMemoryVectorStore();
396
- await store.upsert([
397
- bodyChunk("msg-1", ANCHOR_VECTOR, {
398
- subject: "Dinner reservation",
399
- textPreview: "your table is booked",
400
- }),
401
- bodyChunk("msg-2", ANCHOR_VECTOR, {
402
- subject: "Newsletter",
403
- textPreview: "weekly digest",
404
- }),
405
- ]);
406
-
407
- const { messageIds } = await matchAccepted(
408
- matchDeps(store),
409
- ACCOUNT_CONFIG_ID,
410
- {
411
- ...predicate(),
412
- literalClauses: [{ field: "Subject", value: "reservation" }],
413
- },
414
- );
415
-
416
- assert.deepEqual(messageIds, ["msg-1"]);
417
- });
418
-
419
- it("never builds the semantic side for a purely-literal predicate", async () => {
420
- const store = createMemoryVectorStore();
421
- const deps = matchDeps(store, [
422
- candidate("msg-1", { subject: "Dinner reservation" }),
423
- candidate("msg-2", { subject: "Newsletter" }),
424
- ]);
425
-
426
- const { messageIds } = await matchAccepted(deps, ACCOUNT_CONFIG_ID, {
427
- ...predicate(),
428
- anchorMessageId: "None",
429
- literalClauses: [{ field: "Subject", value: "reservation" }],
430
- });
431
-
432
- assert.deepEqual(messageIds, ["msg-1"]);
433
- assert.equal(
434
- deps.semanticBuilds(),
435
- 0,
436
- "a literal-only predicate must not construct the vector-backed side",
437
- );
438
- });
439
- });
440
-
441
- describe("matchOrganize honors the persisted FilterAnchor (reader #350)", () => {
442
- it("still matches by reading the persisted anchor after the anchor message is purged", async () => {
443
- const store = createMemoryVectorStore();
444
- const matching = ["msg-1", "msg-2"];
445
- await store.upsert([
446
- ...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
447
- bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
448
- ]);
449
-
450
- const persistedAnchor: FilterAnchorItem = {
451
- accountConfigId: ACCOUNT_CONFIG_ID,
452
- filterId: "filter-a",
453
- anchorEmbedding: ANCHOR_VECTOR,
454
- anchorEmbeddingId: CURRENT_EMBEDDING_ID,
455
- anchorSourceText: ANCHOR_SOURCE_TEXT,
456
- anchorMessageId: "msg-anchor",
457
- createdAt: 0,
458
- updatedAt: 0,
459
- };
460
-
461
- let buildAnchorCalls = 0;
462
- const deps: OrganizeMatchDeps = {
463
- semantic: () => ({
464
- buildAnchor: async () => {
465
- // The live path: the anchor message's chunks are gone (purged),
466
- // exactly what buildMessageAnchor returns in that case today.
467
- buildAnchorCalls += 1;
468
- return null;
469
- },
470
- vectorStore: store,
471
- embed: async () => ANCHOR_VECTOR,
472
- embeddingId: CURRENT_EMBEDDING_ID,
473
- }),
474
- listAccountFilterMessages: async () => ({ items: [] }),
475
- filterAnchors: {
476
- get: async () => null,
477
- listByAccountConfig: async () => [persistedAnchor],
478
- put: async () => {
479
- throw new Error("a current anchor must not be rewritten");
480
- },
481
- },
482
- };
483
-
484
- const { messageIds } = await matchAccepted(
485
- deps,
486
- ACCOUNT_CONFIG_ID,
487
- predicate(),
488
- );
489
-
490
- assert.deepEqual(
491
- [...messageIds].sort(),
492
- matching,
493
- "the persisted anchor vector still finds the same matches the live message would have",
494
- );
495
- assert.equal(
496
- buildAnchorCalls,
497
- 0,
498
- "a persisted anchor must be read instead of re-deriving one from the (now-gone) live message",
499
- );
500
- });
501
-
502
- it("falls back to deriving the anchor from the live message when no filter was ever anchored on it", async () => {
503
- const store = createMemoryVectorStore();
504
- await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
505
-
506
- // No persisted FilterAnchor names this anchorMessageId — an ad hoc "all
507
- // like these" widen over a bare message selection, never tied to a
508
- // standing filter.
509
- const { messageIds } = await matchAccepted(
510
- matchDeps(store),
511
- ACCOUNT_CONFIG_ID,
512
- predicate(),
513
- );
514
-
515
- assert.deepEqual(messageIds, ["msg-1"]);
516
- });
517
- });
518
-
519
- /**
520
- * A zero from the widen has two meanings and the preview could not tell them
521
- * apart (issue #452): no mail resembles the anchor, or the index holds nothing
522
- * to compare it against. Only the second is a reason to keep the rule.
523
- */
524
- describe("matchOrganize reporting an empty semantic index", () => {
525
- it("flags a widen answered by an index with no vectors at all", async () => {
526
- const { messageIds, semanticIndexEmpty, semanticUnavailable } =
527
- await matchAccepted(
528
- matchDeps(createMemoryVectorStore()),
529
- ACCOUNT_CONFIG_ID,
530
- predicate(),
531
- );
532
-
533
- assert.deepEqual(messageIds, []);
534
- assert.equal(semanticIndexEmpty, true);
535
- assert.equal(
536
- semanticUnavailable,
537
- false,
538
- "an empty index is not a missing vector pipeline",
539
- );
540
- });
541
-
542
- it("flags a widen whose anchor message has no chunk vectors to pool", async () => {
543
- const store = createMemoryVectorStore();
544
- await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
545
- const deps: OrganizeMatchDeps = {
546
- ...matchDeps(store),
547
- semantic: () => ({
548
- buildAnchor: async () => null,
549
- vectorStore: store,
550
- embed: async () => ANCHOR_VECTOR,
551
- embeddingId: CURRENT_EMBEDDING_ID,
552
- }),
553
- };
554
-
555
- const { messageIds, semanticIndexEmpty } = await matchAccepted(
556
- deps,
557
- ACCOUNT_CONFIG_ID,
558
- predicate(),
559
- );
560
-
561
- assert.deepEqual(messageIds, []);
562
- assert.equal(semanticIndexEmpty, true);
563
- });
564
-
565
- it("does not flag a populated index that simply holds nothing similar enough", async () => {
566
- const store = createMemoryVectorStore();
567
- await store.upsert([bodyChunk("msg-1", ORTHOGONAL_VECTOR)]);
568
-
569
- const { messageIds, semanticIndexEmpty } = await matchAccepted(
570
- matchDeps(store),
571
- ACCOUNT_CONFIG_ID,
572
- predicate(),
573
- );
574
-
575
- assert.deepEqual(
576
- messageIds,
577
- [],
578
- "the one indexed message is below threshold",
579
- );
580
- assert.equal(
581
- semanticIndexEmpty,
582
- false,
583
- "the index answered; the rule is what matched nothing",
584
- );
585
- });
586
-
587
- it("does not flag a widen that matched", async () => {
588
- const store = createMemoryVectorStore();
589
- await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
590
-
591
- const { messageIds, semanticIndexEmpty } = await matchAccepted(
592
- matchDeps(store),
593
- ACCOUNT_CONFIG_ID,
594
- predicate(),
595
- );
596
-
597
- assert.deepEqual(messageIds, ["msg-1"]);
598
- assert.equal(semanticIndexEmpty, false);
599
- });
600
-
601
- it("does not flag a literal-only predicate, which never consults the index", async () => {
602
- const { semanticIndexEmpty } = await matchAccepted(
603
- matchDeps(createMemoryVectorStore(), [
604
- candidate("msg-1", { subject: "Weekly newsletter" }),
605
- ]),
606
- ACCOUNT_CONFIG_ID,
607
- {
608
- ...predicate(),
609
- anchorMessageId: "None",
610
- literalClauses: [{ field: "Subject", value: "reservation" }],
611
- },
612
- );
613
-
614
- assert.equal(semanticIndexEmpty, false);
615
- });
616
- });
617
-
618
- describe("matchOrganize on a deployment without the vector pipeline", () => {
619
- const ORIGINAL = process.env.DATA_BACKEND;
620
- beforeEach(() => {
621
- process.env.DATA_BACKEND = "sqlite";
622
- _resetSemanticCapabilityForTest();
623
- });
624
- afterEach(() => {
625
- if (ORIGINAL === undefined) delete process.env.DATA_BACKEND;
626
- else process.env.DATA_BACKEND = ORIGINAL;
627
- _resetSemanticCapabilityForTest();
628
- });
629
-
630
- it("matches a literal predicate from the corpus without touching the vector store", async () => {
631
- const deps = vectorlessDeps([
632
- candidate("msg-1", { subject: "Dinner reservation" }),
633
- candidate("msg-2", { subject: "Weekly newsletter" }),
634
- candidate("msg-3", { subject: "Table reservation confirmed" }),
635
- ]);
636
-
637
- const { messageIds, semanticUnavailable } = await matchAccepted(
638
- deps,
639
- ACCOUNT_CONFIG_ID,
640
- {
641
- ...predicate(),
642
- anchorMessageId: "None",
643
- literalClauses: [{ field: "Subject", value: "reservation" }],
644
- },
645
- );
646
-
647
- assert.deepEqual(messageIds, ["msg-1", "msg-3"]);
648
- assert.equal(semanticUnavailable, false);
649
- assert.equal(
650
- deps.semanticUsed(),
651
- false,
652
- "a literal-only predicate must never reach the vector store",
653
- );
654
- });
655
-
656
- it("refuses a body-content (HasWords) clause as a result rather than matching it against a preview", async () => {
657
- const deps = vectorlessDeps([
658
- candidate("msg-1", { subject: "Dinner reservation" }),
659
- ]);
660
-
661
- const result = await matchOrganize(deps, ACCOUNT_CONFIG_ID, {
662
- ...predicate(),
663
- anchorMessageId: "None",
664
- literalClauses: [{ field: "HasWords", value: "invoice" }],
665
- });
666
-
667
- assert.deepEqual(
668
- result.rejected,
669
- {
670
- reason: "BodyContentWithoutVectorPipeline",
671
- message: BODY_CONTENT_REJECTION_MESSAGE,
672
- },
673
- "the vector-free literal path must not silently narrow a body match to a preview",
674
- );
675
- assert.equal(deps.semanticUsed(), false);
676
- });
677
-
678
- // The refusal is the same whichever way the vector-free path is reached, so
679
- // an anchored predicate that degrades onto it cannot land in the corpus scan
680
- // with a body clause it can only mis-evaluate (reader #463).
681
- it("refuses a body-content clause reached through the degraded widen fallback", async () => {
682
- const deps = vectorlessDeps([
683
- candidate("msg-1", { subject: "Dinner reservation" }),
684
- ]);
685
-
686
- const result = await matchOrganize(deps, ACCOUNT_CONFIG_ID, {
687
- ...predicate(),
688
- literalClauses: [{ field: "HasWords", value: "invoice" }],
689
- });
690
-
691
- assert.equal(result.rejected?.reason, "BodyContentWithoutVectorPipeline");
692
- });
693
-
694
- it("degrades an anchor+clauses widen to the literal matches, flagged, instead of crashing", async () => {
695
- const deps = vectorlessDeps([
696
- candidate("msg-1", { subject: "Dinner reservation" }),
697
- candidate("msg-2", { subject: "Weekly newsletter" }),
698
- ]);
699
-
700
- const { messageIds, semanticUnavailable } = await matchAccepted(
701
- deps,
702
- ACCOUNT_CONFIG_ID,
703
- {
704
- ...predicate(),
705
- literalClauses: [{ field: "Subject", value: "reservation" }],
706
- },
707
- );
708
-
709
- assert.deepEqual(messageIds, ["msg-1"]);
710
- assert.equal(semanticUnavailable, true);
711
- });
712
-
713
- it("degrades an anchor-only widen to an empty flagged result instead of crashing", async () => {
714
- const deps = vectorlessDeps([candidate("msg-1"), candidate("msg-2")]);
715
-
716
- const { messageIds, semanticUnavailable, semanticIndexEmpty } =
717
- await matchAccepted(deps, ACCOUNT_CONFIG_ID, predicate());
718
-
719
- assert.deepEqual(messageIds, []);
720
- assert.equal(semanticUnavailable, true);
721
- assert.equal(
722
- semanticIndexEmpty,
723
- false,
724
- "a deployment with no vector pipeline has no index to call empty",
725
- );
726
- });
727
-
728
- it("keeps widening from a drifted persisted anchor when no embedding model is available", async () => {
729
- // Self-host: sqlite-vec is present, `@huggingface/transformers` is not.
730
- // The anchor's stamp will never match — after a model swap, or forever
731
- // for one pooled from pre-#349 chunks and stamped
732
- // UNKNOWN_CHUNK_EMBEDDING_ID — and there is no embedder to repair it
733
- // with, so the widen must query with the vector it has rather than go
734
- // dark and take every later /search/semantic with it.
735
- const store = createMemoryVectorStore();
736
- await store.upsert([
737
- bodyChunk("msg-1", ANCHOR_VECTOR),
738
- bodyChunk("msg-stored-space", ORTHOGONAL_VECTOR),
739
- ]);
740
- const drifted: FilterAnchorItem = {
741
- accountConfigId: ACCOUNT_CONFIG_ID,
742
- filterId: "filter-a",
743
- anchorEmbedding: ORTHOGONAL_VECTOR,
744
- anchorEmbeddingId: STALE_EMBEDDING_ID,
745
- anchorSourceText: ANCHOR_SOURCE_TEXT,
746
- anchorMessageId: "msg-anchor",
747
- createdAt: 0,
748
- updatedAt: 0,
749
- };
750
- const deps: OrganizeMatchDeps = {
751
- semantic: () => ({
752
- buildAnchor: async () => {
753
- throw new Error("a persisted anchor must not be re-derived");
754
- },
755
- vectorStore: store,
756
- embed: async () => {
757
- throw moduleNotFound();
758
- },
759
- embeddingId: CURRENT_EMBEDDING_ID,
760
- }),
761
- listAccountFilterMessages: async () => ({ items: [] }),
762
- filterAnchors: {
763
- get: async () => null,
764
- listByAccountConfig: async () => [drifted],
765
- put: async () => {
766
- throw new Error("an unrepairable anchor must not be rewritten");
767
- },
768
- },
769
- };
770
-
771
- const { messageIds, semanticUnavailable } = await matchAccepted(
772
- deps,
773
- ACCOUNT_CONFIG_ID,
774
- predicate(),
775
- );
776
-
777
- assert.deepEqual(
778
- messageIds,
779
- ["msg-stored-space"],
780
- "the kNN read needs no embedder, so the stored anchor vector still returns real hits",
781
- );
782
- assert.equal(semanticUnavailable, false);
783
- assert.equal(
784
- isSemanticSearchUnavailable(),
785
- false,
786
- "one unrepairable anchor must not disable free-text semantic search for the rest of the process",
787
- );
788
- });
789
-
790
- it("propagates a genuine (non-capability) semantic failure loudly", async () => {
791
- const deps: OrganizeMatchDeps = {
792
- semantic: () => ({
793
- buildAnchor: async () => {
794
- throw new Error("SQLITE_BUSY");
795
- },
796
- vectorStore: {
797
- query: async () => [],
798
- getByMessage: async () => [],
799
- },
800
- embed: async () => [],
801
- embeddingId: CURRENT_EMBEDDING_ID,
802
- }),
803
- listAccountFilterMessages: async () => ({ items: [] }),
804
- filterAnchors: {
805
- get: async () => null,
806
- listByAccountConfig: async () => [],
807
- put: async () => {
808
- throw new Error("unreachable");
809
- },
810
- },
811
- };
812
-
813
- await assert.rejects(
814
- () => matchOrganize(deps, ACCOUNT_CONFIG_ID, predicate()),
815
- /SQLITE_BUSY/,
816
- );
817
- });
818
- });
819
-
820
- describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
821
- it("applies the label to all N matches in one pass and writes zero Filter rows", async () => {
822
- const store = createMemoryVectorStore();
823
- const matching = ["msg-1", "msg-2", "msg-3", "msg-4", "msg-5"];
824
- await store.upsert([
825
- ...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
826
- bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
827
- ]);
828
-
829
- const p = predicate({ actionLabelId: "lbl-1" });
830
-
831
- // The preview and the apply share the same matcher — the previewed set is
832
- // exactly what gets applied.
833
- const previewed = await matchAccepted(
834
- matchDeps(store),
835
- ACCOUNT_CONFIG_ID,
836
- p,
837
- );
838
- const applied = await matchAccepted(matchDeps(store), ACCOUNT_CONFIG_ID, p);
839
- assert.deepEqual(previewed, applied);
840
-
841
- const tracked = trackingClient();
842
- const result = await applyOrganize(
843
- { client: tracked.client },
844
- ACCOUNT_CONFIG_ID,
845
- applied.messageIds,
846
- p,
847
- );
848
-
849
- assert.equal(result.applied, matching.length);
850
- assert.equal(result.failed, 0);
851
- assert.deepEqual(
852
- tracked.labeled.map((row) => row.messageId).sort(),
853
- matching,
854
- );
855
- assert.ok(
856
- tracked.labeled.every((row) => row.labelId === "lbl-1"),
857
- "every matching message gets the requested label",
858
- );
859
- assert.equal(tracked.filterWrites(), 0, "no Filter row is ever created");
860
- assert.equal(
861
- tracked.filterAnchorWrites(),
862
- 0,
863
- "no FilterAnchor row is ever created",
864
- );
865
- });
866
-
867
- it("counts a requested move as failed when no move service is wired", async () => {
868
- const store = createMemoryVectorStore();
869
- await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
870
- const p = predicate({ actionMailboxId: "mbox-target" });
871
-
872
- const { messageIds: matched } = await matchAccepted(
873
- matchDeps(store),
874
- ACCOUNT_CONFIG_ID,
875
- p,
876
- );
877
- const tracked = trackingClient();
878
- const result = await applyOrganize(
879
- { client: tracked.client },
880
- ACCOUNT_CONFIG_ID,
881
- matched,
882
- p,
883
- );
884
-
885
- assert.equal(result.applied, 0);
886
- assert.equal(result.failed, 1);
887
- });
888
-
889
- it("moves every match through the wired move service and writes zero Filter rows", async () => {
890
- const store = createMemoryVectorStore();
891
- const matching = ["msg-1", "msg-2", "msg-3"];
892
- await store.upsert([
893
- ...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
894
- bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
895
- ]);
896
- const p = predicate({ actionMailboxId: "mbox-target" });
897
-
898
- const { messageIds: matched } = await matchAccepted(
899
- matchDeps(store),
900
- ACCOUNT_CONFIG_ID,
901
- p,
902
- );
903
- const tracked = trackingClient();
904
- const mover = trackingMoveService();
905
- const result = await applyOrganize(
906
- {
907
- client: tracked.client,
908
- moveService: mover.moveService,
909
- },
910
- ACCOUNT_CONFIG_ID,
911
- matched,
912
- p,
913
- );
914
-
915
- assert.equal(result.applied, matching.length);
916
- assert.equal(result.failed, 0);
917
- assert.deepEqual(
918
- mover.moves.map((m) => m.messageId).sort(),
919
- matching,
920
- "every matched message is moved once",
921
- );
922
- assert.ok(
923
- mover.moves.every((m) => m.destinationMailboxId === "mbox-target"),
924
- "every move targets the requested mailbox",
925
- );
926
- assert.equal(
927
- tracked.labeled.length,
928
- 0,
929
- "a move-only action applies no label",
930
- );
931
- assert.equal(tracked.filterWrites(), 0, "no Filter row is ever created");
932
- assert.equal(
933
- tracked.filterAnchorWrites(),
934
- 0,
935
- "no FilterAnchor row is ever created",
936
- );
937
- });
938
-
939
- it("applies both a label and a move when both actions are requested", async () => {
940
- const store = createMemoryVectorStore();
941
- await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
942
- const p = predicate({
943
- actionLabelId: "lbl-1",
944
- actionMailboxId: "mbox-target",
945
- });
946
-
947
- const { messageIds: matched } = await matchAccepted(
948
- matchDeps(store),
949
- ACCOUNT_CONFIG_ID,
950
- p,
951
- );
952
- const tracked = trackingClient();
953
- const mover = trackingMoveService();
954
- const result = await applyOrganize(
955
- {
956
- client: tracked.client,
957
- moveService: mover.moveService,
958
- },
959
- ACCOUNT_CONFIG_ID,
960
- matched,
961
- p,
962
- );
963
-
964
- assert.equal(result.applied, 1);
965
- assert.equal(result.failed, 0);
966
- assert.deepEqual(tracked.labeled, [
967
- { messageId: "msg-1", labelId: "lbl-1" },
968
- ]);
969
- assert.deepEqual(
970
- mover.moves.map((m) => m.messageId),
971
- ["msg-1"],
972
- );
973
- });
974
-
975
- it("is idempotent on redelivery: a re-run re-issues the same move without double-enqueuing", async () => {
976
- const store = createMemoryVectorStore();
977
- const matching = ["msg-1", "msg-2"];
978
- await store.upsert(matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)));
979
- const p = predicate({ actionMailboxId: "mbox-target" });
980
-
981
- const { messageIds: matched } = await matchAccepted(
982
- matchDeps(store),
983
- ACCOUNT_CONFIG_ID,
984
- p,
985
- );
986
- const tracked = trackingClient();
987
- const mover = trackingMoveService();
988
-
989
- const first = await applyOrganize(
990
- {
991
- client: tracked.client,
992
- moveService: mover.moveService,
993
- },
994
- ACCOUNT_CONFIG_ID,
995
- matched,
996
- p,
997
- );
998
- const second = await applyOrganize(
999
- {
1000
- client: tracked.client,
1001
- moveService: mover.moveService,
1002
- },
1003
- ACCOUNT_CONFIG_ID,
1004
- matched,
1005
- p,
1006
- );
1007
-
1008
- assert.equal(first.applied, matching.length);
1009
- assert.equal(second.applied, matching.length);
1010
- assert.equal(
1011
- mover.enqueues(),
1012
- matching.length,
1013
- "the second pass drives the marker forward without a fresh enqueue",
1014
- );
1015
- assert.deepEqual(
1016
- [...mover.destinations().entries()].sort(),
1017
- matching.map((id): [string, string] => [id, "mbox-target"]),
1018
- "each message rests in the requested mailbox exactly once",
1019
- );
1020
- });
1021
- });
1022
-
1023
- /**
1024
- * A standing filter acts when a message is first seen and when the user presses
1025
- * "run rules now", and holds no claim afterwards. A later apply is the newer
1026
- * event, so it moves the mail — and because nothing is suppressed, `applied`
1027
- * counts moves that happened rather than moves that were skipped (reader #497).
1028
- */
1029
- describe("a user-initiated apply outranks every standing filter (reader #497)", () => {
1030
- it("moves a hand-picked selection a standing filter claims for a different mailbox", async () => {
1031
- const store = createMemoryVectorStore();
1032
- const picked = ["msg-1", "msg-2", "msg-3"];
1033
- await store.upsert(picked.map((id) => bodyChunk(id, ANCHOR_VECTOR)));
1034
- const p = predicate({ actionMailboxId: "mbox-projects" });
1035
- // The standing rule that filed all three into Archive in the first place —
1036
- // moving them back out is the whole point of the button.
1037
- const standing = filterItem({
1038
- filterId: "filter-github",
1039
- ruleChangedAt: 1_000,
1040
- actionChangedAt: 1_000,
1041
- actionMailboxId: "mbox-archive",
1042
- literalClauses: [{ field: "Subject", value: "reservation" }],
1043
- });
1044
-
1045
- const { messageIds: matched } = await matchAccepted(
1046
- matchDeps(store),
1047
- ACCOUNT_CONFIG_ID,
1048
- p,
1049
- );
1050
- const tracked = trackingClient({
1051
- activeFilters: [standing],
1052
- threadMessages: Object.fromEntries(
1053
- picked.map((id) => [id, { subject: "Dinner reservation" }]),
1054
- ),
1055
- });
1056
- const mover = trackingMoveService();
1057
- const result = await applyOrganize(
1058
- { client: tracked.client, moveService: mover.moveService },
1059
- ACCOUNT_CONFIG_ID,
1060
- matched,
1061
- p,
1062
- );
1063
-
1064
- assert.deepEqual(
1065
- mover.moves.map((move) => move.messageId).sort(),
1066
- picked,
1067
- "every hand-picked message moves",
1068
- );
1069
- assert.ok(
1070
- mover.moves.every(
1071
- (move) => move.destinationMailboxId === "mbox-projects",
1072
- ),
1073
- "the destination is the one the user picked, not the standing filter's",
1074
- );
1075
- assert.equal(
1076
- result.applied,
1077
- mover.moves.length,
1078
- "the reported count equals the moves that actually happened",
1079
- );
1080
- assert.equal(result.failed, 0);
1081
- assert.equal(
1082
- tracked.filterReads(),
1083
- 0,
1084
- "the account's standing filters are not consulted at all",
1085
- );
1086
- });
1087
- });
1088
-
1089
- /**
1090
- * A same-dimension embedding-model swap leaves every persisted FilterAnchor
1091
- * stamped with the old model's id and its vector in the old model's space.
1092
- * Index-time matching repairs that lazily (RFC 039 Decision 1a); the
1093
- * back-apply widen must do the same, or it queries with a stale-model vector
1094
- * and silently disagrees with the pipeline it claims to mirror (reader #399).
1095
- */
1096
- describe("back-apply repairs a drifted anchor the way index-time matching does (reader #399)", () => {
1097
- // Written under the previous model: same dimensions, different space, so
1098
- // nothing but the id stamp distinguishes it from a usable anchor.
1099
- const staleAnchor = (filterId: string): FilterAnchorItem => ({
1100
- accountConfigId: ACCOUNT_CONFIG_ID,
1101
- filterId,
1102
- anchorEmbedding: ORTHOGONAL_VECTOR,
1103
- anchorEmbeddingId: STALE_EMBEDDING_ID,
1104
- anchorSourceText: ANCHOR_SOURCE_TEXT,
1105
- anchorMessageId: "msg-anchor",
1106
- createdAt: 0,
1107
- updatedAt: 0,
1108
- });
1109
-
1110
- /** Every text embeds into the current model's space. */
1111
- const currentModelDeps = (
1112
- store: ReturnType<typeof createMemoryVectorStore>,
1113
- anchors: FilterAnchorItem[],
1114
- embed: (text: string) => Promise<number[]> = async () => ANCHOR_VECTOR,
1115
- ): OrganizeMatchDeps & { anchorPuts: CreateFilterAnchorInput[] } => {
1116
- const anchorPuts: CreateFilterAnchorInput[] = [];
1117
- return {
1118
- semantic: () => ({
1119
- buildAnchor: async () => {
1120
- throw new Error("a persisted anchor must be repaired, not replaced");
1121
- },
1122
- vectorStore: store,
1123
- embed,
1124
- embeddingId: CURRENT_EMBEDDING_ID,
1125
- }),
1126
- listAccountFilterMessages: async () => ({ items: [] }),
1127
- filterAnchors: {
1128
- get: async () => null,
1129
- listByAccountConfig: async () => anchors,
1130
- put: async (input: CreateFilterAnchorInput) => {
1131
- anchorPuts.push(input);
1132
- return { ...input, createdAt: 0, updatedAt: 1 };
1133
- },
1134
- },
1135
- anchorPuts,
1136
- };
1137
- };
1138
-
1139
- it("matchSemantic re-embeds a drifted persisted anchor instead of querying with the stale vector", async () => {
1140
- const store = createMemoryVectorStore();
1141
- await store.upsert([
1142
- bodyChunk("msg-1", ANCHOR_VECTOR),
1143
- bodyChunk("msg-stale-space", ORTHOGONAL_VECTOR),
1144
- ]);
1145
- const deps = currentModelDeps(store, [staleAnchor("filter-a")]);
1146
-
1147
- const { messageIds } = await matchAccepted(
1148
- deps,
1149
- ACCOUNT_CONFIG_ID,
1150
- predicate(),
1151
- );
1152
-
1153
- assert.deepEqual(
1154
- messageIds,
1155
- ["msg-1"],
1156
- "the widen runs on the re-embedded anchor, not the stale-space vector that would have matched msg-stale-space",
1157
- );
1158
- assert.deepEqual(
1159
- deps.anchorPuts.map((put) => [put.filterId, put.anchorEmbeddingId]),
1160
- [["filter-a", CURRENT_EMBEDDING_ID]],
1161
- "the repair is written back in place under the current model's id",
1162
- );
1163
- assert.deepEqual(deps.anchorPuts[0].anchorEmbedding, ANCHOR_VECTOR);
1164
- assert.equal(deps.anchorPuts[0].anchorSourceText, ANCHOR_SOURCE_TEXT);
1165
- });
1166
- });
1167
-
1168
- describe("matchOrganize with ListId and FromDomain clauses", () => {
1169
- const senderChunk = (
1170
- messageId: string,
1171
- fromEmail: string,
1172
- over: Partial<ChunkMetadata> = {},
1173
- ): VectorRecord => ({
1174
- chunkId: `${messageId}#sender`,
1175
- vector: ANCHOR_VECTOR,
1176
- metadata: metadata({
1177
- messageId,
1178
- chunkType: "sender",
1179
- textPreview: fromEmail,
1180
- ...over,
1181
- }),
1182
- });
1183
-
1184
- it("matches a ListId clause exactly off the vector-free projection", async () => {
1185
- const deps = vectorlessDeps([
1186
- candidate("msg-1", { listId: "weekly.news.example.com" }),
1187
- candidate("msg-2", { listId: "news.example.com" }),
1188
- candidate("msg-3", {}),
1189
- ]);
1190
-
1191
- const { messageIds } = await matchAccepted(deps, ACCOUNT_CONFIG_ID, {
1192
- ...predicate(),
1193
- anchorMessageId: "None",
1194
- literalClauses: [{ field: "ListId", value: "weekly.news.example.com" }],
1195
- });
1196
-
1197
- assert.deepEqual(messageIds, ["msg-1"]);
1198
- });
1199
-
1200
- it("matches a FromDomain clause public-suffix aware off the vector-free projection", async () => {
1201
- const deps = vectorlessDeps([
1202
- candidate("msg-1", { from: "notifications@github.com" }),
1203
- candidate("msg-2", { from: "ci@sub.github.com" }),
1204
- candidate("msg-3", { from: "attacker@github.com.evil.example" }),
1205
- ]);
1206
-
1207
- const { messageIds } = await matchAccepted(deps, ACCOUNT_CONFIG_ID, {
1208
- ...predicate(),
1209
- anchorMessageId: "None",
1210
- literalClauses: [{ field: "FromDomain", value: "github.com" }],
1211
- });
1212
-
1213
- assert.deepEqual(messageIds, ["msg-1", "msg-2"]);
1214
- });
1215
-
1216
- it("round-trips ListId and FromDomain through the semantic-arm chunk projection", async () => {
1217
- const store = createMemoryVectorStore();
1218
- await store.upsert([
1219
- senderChunk("msg-1", "ci@github.com", {
1220
- listId: "actions.github.com",
1221
- }),
1222
- bodyChunk("msg-1", ANCHOR_VECTOR, { listId: "actions.github.com" }),
1223
- senderChunk("msg-2", "hi@othersender.example", {
1224
- listId: "other.list.example",
1225
- }),
1226
- bodyChunk("msg-2", ANCHOR_VECTOR, { listId: "other.list.example" }),
1227
- ]);
1228
-
1229
- const byListId = await matchAccepted(matchDeps(store), ACCOUNT_CONFIG_ID, {
1230
- ...predicate(),
1231
- literalClauses: [{ field: "ListId", value: "actions.github.com" }],
1232
- });
1233
- assert.deepEqual(byListId.messageIds, ["msg-1"]);
1234
-
1235
- const byFromDomain = await matchAccepted(
1236
- matchDeps(store),
1237
- ACCOUNT_CONFIG_ID,
1238
- {
1239
- ...predicate(),
1240
- literalClauses: [{ field: "FromDomain", value: "github.com" }],
1241
- },
1242
- );
1243
- assert.deepEqual(byFromDomain.messageIds, ["msg-1"]);
1244
- });
1245
-
1246
- it("applies a ListId predicate to exactly the previewed set (preview == apply)", async () => {
1247
- const deps = vectorlessDeps([
1248
- candidate("msg-1", { listId: "weekly.news.example.com" }),
1249
- candidate("msg-2", { listId: "weekly.news.example.com" }),
1250
- candidate("msg-3", { listId: "other.example.com" }),
1251
- ]);
1252
- const p = predicate({
1253
- anchorMessageId: "None",
1254
- actionLabelId: "lbl-list",
1255
- literalClauses: [{ field: "ListId", value: "weekly.news.example.com" }],
1256
- });
1257
-
1258
- const previewed = await matchAccepted(deps, ACCOUNT_CONFIG_ID, p);
1259
- const applied = await matchAccepted(deps, ACCOUNT_CONFIG_ID, p);
1260
- assert.deepEqual(previewed, applied);
1261
- assert.deepEqual(previewed.messageIds, ["msg-1", "msg-2"]);
1262
-
1263
- const tracked = trackingClient();
1264
- const result = await applyOrganize(
1265
- { client: tracked.client },
1266
- ACCOUNT_CONFIG_ID,
1267
- applied.messageIds,
1268
- p,
1269
- );
1270
-
1271
- assert.equal(result.applied, 2);
1272
- assert.equal(result.failed, 0);
1273
- assert.deepEqual(tracked.labeled.map((row) => row.messageId).sort(), [
1274
- "msg-1",
1275
- "msg-2",
1276
- ]);
1277
- });
1278
- });
1279
-
1280
- /**
1281
- * The corpus read a real deployment gets: the store evaluates the terms and
1282
- * the caller pages the MATCHES. Modelled here rather than stubbed with a fixed
1283
- * array so a matcher that reads only the newest page cannot pass.
1284
- */
1285
- const storeBackedDeps = (
1286
- rows: readonly OrganizeCandidate[],
1287
- ): OrganizeMatchDeps & { pagesRead: () => number } => {
1288
- let pagesRead = 0;
1289
- const termMatches = (
1290
- term: { field: string; contains: string },
1291
- row: OrganizeCandidate,
1292
- ): boolean => {
1293
- const needle = term.contains.toLowerCase();
1294
- if (term.field === "subject") {
1295
- return row.message.subject.toLowerCase().includes(needle);
1296
- }
1297
- if (term.field === "listId") {
1298
- return row.message.listId.toLowerCase().includes(needle);
1299
- }
1300
- return `${row.message.fromName} ${row.message.from}`
1301
- .toLowerCase()
1302
- .includes(needle);
1303
- };
1304
- return {
1305
- semantic: () => {
1306
- throw moduleNotFound();
1307
- },
1308
- listAccountFilterMessages: async (_accountConfigId, query) => {
1309
- pagesRead += 1;
1310
- const narrowed = rows.filter((row) =>
1311
- query.terms.length === 0
1312
- ? true
1313
- : query.operator === "or"
1314
- ? query.terms.some((term) => termMatches(term, row))
1315
- : query.terms.every((term) => termMatches(term, row)),
1316
- );
1317
- const offset = query.continuationToken
1318
- ? Number(query.continuationToken)
1319
- : 0;
1320
- const items = narrowed.slice(offset, offset + query.limit);
1321
- const next = offset + items.length;
1322
- return {
1323
- items,
1324
- continuationToken: next < narrowed.length ? String(next) : undefined,
1325
- };
1326
- },
1327
- filterAnchors: {
1328
- get: async () => null,
1329
- listByAccountConfig: async () => [],
1330
- put: async () => {
1331
- throw new Error("unreachable");
1332
- },
1333
- },
1334
- pagesRead: () => pagesRead,
1335
- };
1336
- };
1337
-
1338
- describe("literal matching bounds the result, not the corpus (reader #459)", () => {
1339
- const quietSender = candidate("quiet-sender", {
1340
- from: "noreply@bank.example",
1341
- fromName: "Statements",
1342
- subject: "Your March statement",
1343
- });
1344
- const newerNoise = Array.from({ length: ORGANIZE_MATCH_LIMIT + 1 }, (_, i) =>
1345
- candidate(`noise-${i}`, {
1346
- from: "digest@other.example",
1347
- fromName: "Digest",
1348
- subject: `Daily digest ${i}`,
1349
- }),
1350
- );
1351
-
1352
- it("matches a domain rule against mail older than a whole page of newer misses", async () => {
1353
- const deps = storeBackedDeps([...newerNoise, quietSender]);
1354
-
1355
- const { messageIds } = await matchAccepted(
1356
- deps,
1357
- ACCOUNT_CONFIG_ID,
1358
- predicate({
1359
- anchorMessageId: "None",
1360
- literalClauses: [{ field: "FromDomain", value: "bank.example" }],
1361
- }),
1362
- );
1363
-
1364
- assert.deepEqual(messageIds, ["quiet-sender"]);
1365
- });
1366
-
1367
- it("reads one narrowed page rather than paging the whole mailbox", async () => {
1368
- const deps = storeBackedDeps([...newerNoise, quietSender]);
1369
-
1370
- await matchAccepted(
1371
- deps,
1372
- ACCOUNT_CONFIG_ID,
1373
- predicate({
1374
- anchorMessageId: "None",
1375
- literalClauses: [{ field: "From", value: "noreply@bank.example" }],
1376
- }),
1377
- );
1378
-
1379
- assert.equal(deps.pagesRead(), 1);
1380
- });
1381
-
1382
- it("still refines what the store returned: a term is a narrowing, not the verdict", async () => {
1383
- const deps = storeBackedDeps([
1384
- candidate("spoofed", { from: "billing@bank.example.evil.test" }),
1385
- quietSender,
1386
- ]);
1387
-
1388
- const { messageIds } = await matchAccepted(
1389
- deps,
1390
- ACCOUNT_CONFIG_ID,
1391
- predicate({
1392
- anchorMessageId: "None",
1393
- literalClauses: [{ field: "FromDomain", value: "bank.example" }],
1394
- }),
1395
- );
1396
-
1397
- assert.deepEqual(messageIds, ["quiet-sender"]);
1398
- });
1399
-
1400
- it("caps the matches at the limit when more of the corpus matches", async () => {
1401
- const matching = Array.from({ length: ORGANIZE_MATCH_LIMIT * 2 }, (_, i) =>
1402
- candidate(`hit-${i}`, { from: "noreply@bank.example" }),
1403
- );
1404
- const deps = storeBackedDeps(matching);
1405
-
1406
- const { messageIds } = await matchAccepted(
1407
- deps,
1408
- ACCOUNT_CONFIG_ID,
1409
- predicate({
1410
- anchorMessageId: "None",
1411
- literalClauses: [{ field: "FromDomain", value: "bank.example" }],
1412
- }),
1413
- );
1414
-
1415
- assert.equal(messageIds.length, ORGANIZE_MATCH_LIMIT);
1416
- });
1417
-
1418
- it("never queries for clauses nothing can satisfy", async () => {
1419
- const deps = storeBackedDeps([quietSender]);
1420
-
1421
- const { messageIds } = await matchAccepted(
1422
- deps,
1423
- ACCOUNT_CONFIG_ID,
1424
- predicate({
1425
- anchorMessageId: "None",
1426
- literalClauses: [{ field: "From", value: " " }],
1427
- }),
1428
- );
1429
-
1430
- assert.deepEqual(messageIds, []);
1431
- assert.equal(deps.pagesRead(), 0);
1432
- });
1433
-
1434
- it("counts one match for mail filed in two folders", async () => {
1435
- const deps = storeBackedDeps([
1436
- candidate("filed-twice", { from: "noreply@bank.example" }),
1437
- candidate("filed-twice", { from: "noreply@bank.example" }),
1438
- ]);
1439
-
1440
- const { messageIds } = await matchAccepted(
1441
- deps,
1442
- ACCOUNT_CONFIG_ID,
1443
- predicate({
1444
- anchorMessageId: "None",
1445
- literalClauses: [{ field: "FromDomain", value: "bank.example" }],
1446
- }),
1447
- );
1448
-
1449
- assert.deepEqual(messageIds, ["filed-twice"]);
1450
- });
1451
- });