@remit/backend 0.0.143 → 0.0.145

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (279) hide show
  1. package/dist/dev-server/content-handler.d.ts +38 -0
  2. package/dist/dev-server/content-handler.js +46 -0
  3. package/dist/src/api.d.ts +5 -0
  4. package/dist/src/api.js +73 -0
  5. package/dist/src/auth.d.ts +24 -0
  6. package/dist/src/auth.js +81 -0
  7. package/dist/src/config/api-config.d.ts +20 -0
  8. package/dist/src/config/api-config.js +1 -0
  9. package/dist/src/config/from-env.d.ts +2 -0
  10. package/dist/src/config/from-env.js +65 -0
  11. package/dist/src/config/msoauth-secrets-manager.d.ts +2 -0
  12. package/dist/src/config/msoauth-secrets-manager.js +28 -0
  13. package/dist/src/config/msoauth.d.ts +19 -0
  14. package/dist/src/config/msoauth.js +6 -0
  15. package/{src/derive/autoMoved.ts → dist/src/derive/autoMoved.d.ts} +1 -29
  16. package/dist/src/derive/autoMoved.js +49 -0
  17. package/dist/src/derive/contentSignature.d.ts +40 -0
  18. package/{src/derive/contentSignature.ts → dist/src/derive/contentSignature.js} +23 -77
  19. package/dist/src/derive/contentUrl.d.ts +33 -0
  20. package/dist/src/derive/contentUrl.js +27 -0
  21. package/{src/derive/deriveMuted.ts → dist/src/derive/deriveMuted.d.ts} +1 -3
  22. package/dist/src/derive/deriveMuted.js +7 -0
  23. package/dist/src/derive/enrichThreadRows.d.ts +118 -0
  24. package/dist/src/derive/enrichThreadRows.js +151 -0
  25. package/dist/src/derive/filterThreadCriteria.d.ts +23 -0
  26. package/dist/src/derive/filterThreadCriteria.js +24 -0
  27. package/{src/derive/pendingMoveCounts.ts → dist/src/derive/pendingMoveCounts.d.ts} +5 -53
  28. package/dist/src/derive/pendingMoveCounts.js +53 -0
  29. package/{src/derive/senderTrust.ts → dist/src/derive/senderTrust.d.ts} +1 -9
  30. package/dist/src/derive/senderTrust.js +19 -0
  31. package/dist/src/error.d.ts +21 -0
  32. package/dist/src/error.js +85 -0
  33. package/dist/src/handlers/account-guards.d.ts +36 -0
  34. package/dist/src/handlers/account-guards.js +107 -0
  35. package/dist/src/handlers/account-oauth.d.ts +15 -0
  36. package/dist/src/handlers/account-oauth.js +334 -0
  37. package/dist/src/handlers/account-overrides.d.ts +53 -0
  38. package/dist/src/handlers/account-overrides.js +233 -0
  39. package/dist/src/handlers/account-signature.d.ts +28 -0
  40. package/dist/src/handlers/account-signature.js +85 -0
  41. package/dist/src/handlers/account.d.ts +7 -0
  42. package/dist/src/handlers/account.js +337 -0
  43. package/dist/src/handlers/address.d.ts +35 -0
  44. package/dist/src/handlers/address.js +133 -0
  45. package/dist/src/handlers/calendar-event.d.ts +70 -0
  46. package/dist/src/handlers/calendar-event.js +417 -0
  47. package/dist/src/handlers/calendar-feed.d.ts +47 -0
  48. package/dist/src/handlers/calendar-feed.js +184 -0
  49. package/dist/src/handlers/calendar-suggestion.d.ts +47 -0
  50. package/dist/src/handlers/calendar-suggestion.js +204 -0
  51. package/dist/src/handlers/calendar.d.ts +116 -0
  52. package/dist/src/handlers/calendar.js +323 -0
  53. package/dist/src/handlers/config.d.ts +43 -0
  54. package/dist/src/handlers/config.js +164 -0
  55. package/dist/src/handlers/ensure-account-config.d.ts +9 -0
  56. package/dist/src/handlers/ensure-account-config.js +27 -0
  57. package/dist/src/handlers/filter.d.ts +3 -0
  58. package/dist/src/handlers/filter.js +88 -0
  59. package/dist/src/handlers/folder-role-appointments.d.ts +68 -0
  60. package/dist/src/handlers/folder-role-appointments.js +168 -0
  61. package/dist/src/handlers/folder-role-labels.d.ts +26 -0
  62. package/dist/src/handlers/folder-role-labels.js +40 -0
  63. package/dist/src/handlers/folder-role.d.ts +9 -0
  64. package/dist/src/handlers/folder-role.js +37 -0
  65. package/dist/src/handlers/index.d.ts +88 -0
  66. package/dist/src/handlers/index.js +62 -0
  67. package/dist/src/handlers/label.d.ts +43 -0
  68. package/dist/src/handlers/label.js +107 -0
  69. package/dist/src/handlers/mailbox-bindings.d.ts +41 -0
  70. package/dist/src/handlers/mailbox-bindings.js +61 -0
  71. package/dist/src/handlers/mailbox.d.ts +23 -0
  72. package/dist/src/handlers/mailbox.js +211 -0
  73. package/dist/src/handlers/me.d.ts +2 -0
  74. package/dist/src/handlers/me.js +117 -0
  75. package/dist/src/handlers/message.d.ts +82 -0
  76. package/dist/src/handlers/message.js +595 -0
  77. package/dist/src/handlers/organize.d.ts +10 -0
  78. package/dist/src/handlers/organize.js +103 -0
  79. package/dist/src/handlers/outbox-attachment.d.ts +25 -0
  80. package/dist/src/handlers/outbox-attachment.js +77 -0
  81. package/dist/src/handlers/outbox.d.ts +4 -0
  82. package/dist/src/handlers/outbox.js +136 -0
  83. package/dist/src/handlers/sync-progress.d.ts +46 -0
  84. package/dist/src/handlers/sync-progress.js +63 -0
  85. package/dist/src/handlers/sync.d.ts +28 -0
  86. package/dist/src/handlers/sync.js +73 -0
  87. package/dist/src/handlers/system-update.d.ts +2 -0
  88. package/dist/src/handlers/system-update.js +159 -0
  89. package/dist/src/handlers/thread.d.ts +148 -0
  90. package/dist/src/handlers/thread.js +228 -0
  91. package/dist/src/handlers/unified-threads.d.ts +247 -0
  92. package/dist/src/handlers/unified-threads.js +406 -0
  93. package/{src/handlers/vip-suggestions.ts → dist/src/handlers/vip-suggestions.d.ts} +1 -13
  94. package/dist/src/handlers/vip-suggestions.js +16 -0
  95. package/dist/src/index.d.ts +5 -0
  96. package/dist/src/index.js +53 -0
  97. package/{src/json.ts → dist/src/json.d.ts} +1 -4
  98. package/dist/src/json.js +10 -0
  99. package/dist/src/jwt-auth.d.ts +15 -0
  100. package/dist/src/jwt-auth.js +45 -0
  101. package/dist/src/lambda.d.ts +13 -0
  102. package/dist/src/lambda.js +33 -0
  103. package/dist/src/request.d.ts +9 -0
  104. package/dist/src/request.js +17 -0
  105. package/dist/src/response.d.ts +44 -0
  106. package/dist/src/response.js +80 -0
  107. package/dist/src/serve.d.ts +31 -0
  108. package/dist/src/serve.js +117 -0
  109. package/dist/src/service/account-deletion.d.ts +20 -0
  110. package/dist/src/service/account-deletion.js +33 -0
  111. package/dist/src/service/compose-sqlite.d.ts +3 -0
  112. package/dist/src/service/compose-sqlite.js +13 -0
  113. package/dist/src/service/create-remit-client.d.ts +109 -0
  114. package/dist/src/service/create-remit-client.js +188 -0
  115. package/dist/src/service/data-client.d.ts +12 -0
  116. package/dist/src/service/data-client.js +42 -0
  117. package/dist/src/service/export-identity.d.ts +4 -0
  118. package/dist/src/service/export-identity.js +33 -0
  119. package/dist/src/service/fire-and-forget.d.ts +54 -0
  120. package/dist/src/service/fire-and-forget.js +37 -0
  121. package/dist/src/service/organize.d.ts +181 -0
  122. package/dist/src/service/organize.js +367 -0
  123. package/dist/src/service/queues.d.ts +12 -0
  124. package/dist/src/service/queues.js +1 -0
  125. package/dist/src/service/refusal.d.ts +4 -0
  126. package/dist/src/service/refusal.js +20 -0
  127. package/dist/src/service/sender-category.d.ts +51 -0
  128. package/{src/service/sender-category.ts → dist/src/service/sender-category.js} +40 -81
  129. package/dist/src/service/shared-deps.d.ts +6 -0
  130. package/dist/src/service/shared-deps.js +24 -0
  131. package/dist/src/service/trigger-sync.d.ts +57 -0
  132. package/dist/src/service/trigger-sync.js +31 -0
  133. package/dist/src/types.d.ts +72 -0
  134. package/dist/src/types.js +1 -0
  135. package/package.json +125 -98
  136. package/.env.test +0 -4
  137. package/dev-server/content-auth.test.ts +0 -113
  138. package/dev-server/content-auth.ts +0 -54
  139. package/dev-server/content-handler.test.ts +0 -105
  140. package/dev-server/content-handler.ts +0 -79
  141. package/dev-server/content-path.test.ts +0 -37
  142. package/dev-server/content-path.ts +0 -27
  143. package/dev-server/cors.test.ts +0 -48
  144. package/dev-server/cors.ts +0 -25
  145. package/dev-server/keep-alive.test.ts +0 -24
  146. package/dev-server/lambda-helpers.ts +0 -69
  147. package/dev-server/log-output.test.ts +0 -173
  148. package/dev-server/oauth-callback-route.test.ts +0 -66
  149. package/dev-server/relational-health.test.ts +0 -83
  150. package/dev-server/relational-health.ts +0 -39
  151. package/dev-server/server.ts +0 -384
  152. package/dev-server/sync-age.test.ts +0 -91
  153. package/dev-server/sync-age.ts +0 -55
  154. package/dev-server/upload-handler.ts +0 -132
  155. package/dev-server/upload-route.test.ts +0 -196
  156. package/scripts/backfill-classification.ts +0 -91
  157. package/scripts/backfill-list-id.ts +0 -83
  158. package/scripts/config-save.ts +0 -128
  159. package/src/auth.test.ts +0 -12
  160. package/src/auth.ts +0 -92
  161. package/src/config/msoauth.test.ts +0 -48
  162. package/src/config/msoauth.ts +0 -48
  163. package/src/data-backend.test.ts +0 -23
  164. package/src/data-backend.ts +0 -14
  165. package/src/derive/autoMoved.test.ts +0 -104
  166. package/src/derive/contentSignature.test.ts +0 -161
  167. package/src/derive/contentUrl.test.ts +0 -172
  168. package/src/derive/contentUrl.ts +0 -86
  169. package/src/derive/deriveMuted.test.ts +0 -34
  170. package/src/derive/enrichThreadRows.test.ts +0 -360
  171. package/src/derive/enrichThreadRows.ts +0 -234
  172. package/src/derive/filterThreadCriteria.test.ts +0 -126
  173. package/src/derive/filterThreadCriteria.ts +0 -51
  174. package/src/derive/senderTrust.test.ts +0 -67
  175. package/src/error.test.ts +0 -201
  176. package/src/error.ts +0 -123
  177. package/src/handlers/account-guards.test.ts +0 -67
  178. package/src/handlers/account-guards.ts +0 -169
  179. package/src/handlers/account-oauth-callback.test.ts +0 -155
  180. package/src/handlers/account-oauth-consent.test.ts +0 -336
  181. package/src/handlers/account-oauth-start.test.ts +0 -191
  182. package/src/handlers/account-oauth.test.ts +0 -439
  183. package/src/handlers/account-oauth.ts +0 -635
  184. package/src/handlers/account-overrides.test.ts +0 -88
  185. package/src/handlers/account-overrides.ts +0 -323
  186. package/src/handlers/account-ownership.ts +0 -25
  187. package/src/handlers/account-signature.ts +0 -117
  188. package/src/handlers/account-synced-services.test.ts +0 -245
  189. package/src/handlers/account.ts +0 -552
  190. package/src/handlers/address.test.ts +0 -381
  191. package/src/handlers/address.ts +0 -200
  192. package/src/handlers/calendar-event.test.ts +0 -685
  193. package/src/handlers/calendar-event.ts +0 -738
  194. package/src/handlers/calendar-feed-validators.test.ts +0 -228
  195. package/src/handlers/calendar-feed.test.ts +0 -439
  196. package/src/handlers/calendar-feed.ts +0 -291
  197. package/src/handlers/calendar-sqlite-fixture.ts +0 -36
  198. package/src/handlers/calendar-suggestion.test.ts +0 -1269
  199. package/src/handlers/calendar-suggestion.ts +0 -425
  200. package/src/handlers/calendar.test.ts +0 -1080
  201. package/src/handlers/calendar.ts +0 -517
  202. package/src/handlers/config.test.ts +0 -245
  203. package/src/handlers/config.ts +0 -332
  204. package/src/handlers/ensure-account-config.ts +0 -30
  205. package/src/handlers/filter.test.ts +0 -644
  206. package/src/handlers/filter.ts +0 -518
  207. package/src/handlers/folder-role-appointments.test.ts +0 -438
  208. package/src/handlers/folder-role-appointments.ts +0 -249
  209. package/src/handlers/folder-role-labels.ts +0 -63
  210. package/src/handlers/folder-role.test.ts +0 -209
  211. package/src/handlers/folder-role.ts +0 -80
  212. package/src/handlers/index.ts +0 -83
  213. package/src/handlers/label.test.ts +0 -112
  214. package/src/handlers/label.ts +0 -210
  215. package/src/handlers/mailbox-bindings.ts +0 -111
  216. package/src/handlers/mailbox-rename-refusal.test.ts +0 -247
  217. package/src/handlers/mailbox-settled-target.test.ts +0 -251
  218. package/src/handlers/mailbox.test.ts +0 -280
  219. package/src/handlers/mailbox.ts +0 -581
  220. package/src/handlers/me.test.ts +0 -81
  221. package/src/handlers/me.ts +0 -203
  222. package/src/handlers/message-read-backfill.test.ts +0 -103
  223. package/src/handlers/message.test.ts +0 -131
  224. package/src/handlers/message.ts +0 -1131
  225. package/src/handlers/organize.test.ts +0 -277
  226. package/src/handlers/organize.ts +0 -197
  227. package/src/handlers/outbox-attachment.test.ts +0 -765
  228. package/src/handlers/outbox-attachment.ts +0 -126
  229. package/src/handlers/outbox-list.test.ts +0 -179
  230. package/src/handlers/outbox.test.ts +0 -772
  231. package/src/handlers/outbox.ts +0 -258
  232. package/src/handlers/search.test.ts +0 -161
  233. package/src/handlers/search.ts +0 -105
  234. package/src/handlers/sync-progress.test.ts +0 -205
  235. package/src/handlers/sync-progress.ts +0 -99
  236. package/src/handlers/sync.test.ts +0 -168
  237. package/src/handlers/sync.ts +0 -118
  238. package/src/handlers/system-update.test.ts +0 -497
  239. package/src/handlers/system-update.ts +0 -230
  240. package/src/handlers/thread.test.ts +0 -396
  241. package/src/handlers/thread.ts +0 -427
  242. package/src/handlers/unified-threads-mail-sources.sqlite.test.ts +0 -197
  243. package/src/handlers/unified-threads.test.ts +0 -785
  244. package/src/handlers/unified-threads.ts +0 -669
  245. package/src/index.test.ts +0 -83
  246. package/src/index.ts +0 -228
  247. package/src/jwt-auth.test.ts +0 -152
  248. package/src/jwt-auth.ts +0 -132
  249. package/src/log-redaction.test.ts +0 -144
  250. package/src/openapi-spec.test.ts +0 -118
  251. package/src/request-context.test.ts +0 -61
  252. package/src/request-context.ts +0 -33
  253. package/src/request-pipeline.test.ts +0 -88
  254. package/src/request.ts +0 -10
  255. package/src/response.test.ts +0 -130
  256. package/src/response.ts +0 -144
  257. package/src/service/compose-sqlite.ts +0 -98
  258. package/src/service/config-import.ts +0 -39
  259. package/src/service/create-remit-client.ts +0 -522
  260. package/src/service/data-client.test.ts +0 -97
  261. package/src/service/data-client.ts +0 -58
  262. package/src/service/export-identity.ts +0 -39
  263. package/src/service/filter.ts +0 -40
  264. package/src/service/fire-and-forget.test.ts +0 -126
  265. package/src/service/fire-and-forget.ts +0 -79
  266. package/src/service/localhost-env-config.test.ts +0 -36
  267. package/src/service/organize-queue.ts +0 -98
  268. package/src/service/organize.test.ts +0 -1451
  269. package/src/service/organize.ts +0 -1025
  270. package/src/service/read-path-filter-config.test.ts +0 -243
  271. package/src/service/semantic-capability.test.ts +0 -78
  272. package/src/service/semantic-capability.ts +0 -95
  273. package/src/service/sender-category.test.ts +0 -235
  274. package/src/service/sqs.ts +0 -4
  275. package/src/service/trigger-sync.test.ts +0 -205
  276. package/src/service/trigger-sync.ts +0 -100
  277. package/src/stdout-capture-fixture.ts +0 -40
  278. package/src/types.ts +0 -271
  279. package/tsconfig.json +0 -7
@@ -1,1451 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { afterEach, beforeEach, describe, it } from "node:test";
3
- import type {
4
- CreateFilterAnchorInput,
5
- FilterAnchorItem,
6
- FilterItem,
7
- } from "@remit/data-ports";
8
- import { NotFoundError } from "@remit/data-ports/errors";
9
- import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
10
- import type {
11
- AnchorPayload,
12
- ChunkMetadata,
13
- VectorRecord,
14
- } from "@remit/search-service";
15
- import { createMemoryVectorStore } from "@remit/search-service";
16
- import type { RemitClient } from "./data-client.js";
17
- import {
18
- applyOrganize,
19
- BODY_CONTENT_REJECTION_MESSAGE,
20
- matchOrganize,
21
- ORGANIZE_MATCH_LIMIT,
22
- type OrganizeCandidate,
23
- type OrganizeMatchDeps,
24
- type OrganizeMatched,
25
- type OrganizePredicate,
26
- } from "./organize.js";
27
- import {
28
- _resetSemanticCapabilityForTest,
29
- isSemanticSearchUnavailable,
30
- } from "./semantic-capability.js";
31
-
32
- const moduleNotFound = (): Error => {
33
- const error = new Error("Cannot find package 'sqlite-vec' imported from …");
34
- (error as Error & { code: string }).code = "ERR_MODULE_NOT_FOUND";
35
- return error;
36
- };
37
-
38
- const ACCOUNT_CONFIG_ID = "cfg-1";
39
- const ANCHOR_VECTOR = [1, 0, 0, 0];
40
- const ORTHOGONAL_VECTOR = [0, 1, 0, 0];
41
- /** The model the fixtures' embedder is currently configured with. */
42
- const CURRENT_EMBEDDING_ID = "test-model@4";
43
- /** The model a persisted anchor was written under before a same-dimension swap. */
44
- const STALE_EMBEDDING_ID = "older-model@4";
45
- const ANCHOR_SOURCE_TEXT = "book me a table";
46
-
47
- const anchorPayload: AnchorPayload = {
48
- anchorEmbedding: ANCHOR_VECTOR,
49
- anchorEmbeddingId: CURRENT_EMBEDDING_ID,
50
- anchorSourceText: ANCHOR_SOURCE_TEXT,
51
- };
52
-
53
- const metadata = (over: Partial<ChunkMetadata>): ChunkMetadata => ({
54
- messageId: "msg-x",
55
- threadId: "thread-x",
56
- accountConfigId: ACCOUNT_CONFIG_ID,
57
- mailboxIds: ["mbox-1"],
58
- chunkType: "body",
59
- sentDate: 1_700_000_000,
60
- isRead: false,
61
- hasAttachment: false,
62
- hasStars: false,
63
- ...over,
64
- });
65
-
66
- const bodyChunk = (
67
- messageId: string,
68
- vector: number[],
69
- over: Partial<ChunkMetadata> = {},
70
- ): VectorRecord => ({
71
- chunkId: `${messageId}#body`,
72
- vector,
73
- metadata: metadata({ messageId, ...over }),
74
- });
75
-
76
- const predicate = (
77
- over: Partial<OrganizePredicate> = {},
78
- ): OrganizePredicate => ({
79
- anchorMessageId: "msg-anchor",
80
- matchOperator: FilterMatchOperator.And,
81
- literalClauses: [],
82
- similarityThreshold: 0.75,
83
- actionLabelId: "None",
84
- actionMailboxId: "None",
85
- ...over,
86
- });
87
-
88
- /**
89
- * Run the matcher and assert it accepted the predicate, so every test that only
90
- * cares about the matched set reads the accepted arm directly. A rejection is a
91
- * result, not a throw (reader #463), so without this the two would be silently
92
- * interchangeable at the call site.
93
- */
94
- const matchAccepted = async (
95
- ...args: Parameters<typeof matchOrganize>
96
- ): Promise<OrganizeMatched> => {
97
- const result = await matchOrganize(...args);
98
- assert.ok(
99
- result.rejected === null,
100
- `expected an accepted match, got: ${result.rejected?.message}`,
101
- );
102
- return result;
103
- };
104
-
105
- /** A standing filter fixture — the account's already-existing rules. */
106
- const filterItem = (over: Partial<FilterItem> = {}): FilterItem => ({
107
- filterId: "filter-other",
108
- accountConfigId: ACCOUNT_CONFIG_ID,
109
- name: "other filter",
110
- scope: "Standing",
111
- state: FilterState.Active,
112
- disabledReason: "None",
113
- hasAnchor: false,
114
- ruleChangedAt: 0,
115
- actionChangedAt: 0,
116
- matchOperator: FilterMatchOperator.And,
117
- literalClauses: [],
118
- actionLabelId: "None",
119
- actionMailboxId: "None",
120
- createdAt: 0,
121
- updatedAt: 0,
122
- ...over,
123
- });
124
-
125
- /**
126
- * A client that records MessageLabel writes and blows up if the back-apply path
127
- * ever touches Filter/FilterAnchor — the RFC 034 guardrail: this scope never
128
- * persists a standing rule.
129
- *
130
- * `activeFilters`/`filterAnchorRows`/`threadMessages` model the account's
131
- * *other*, already-existing standing filters and message rows. A back-apply
132
- * never arbitrates against them (reader #497), so `filterReads` pins that the
133
- * Active filter set is not even consulted.
134
- */
135
- const trackingClient = (
136
- seed: {
137
- activeFilters?: FilterItem[];
138
- filterAnchorRows?: FilterAnchorItem[];
139
- threadMessages?: Record<
140
- string,
141
- { fromEmail?: string; fromName?: string; subject?: string }
142
- >;
143
- } = {},
144
- ) => {
145
- const labeled: Array<{ messageId: string; labelId: string }> = [];
146
- let filterWrites = 0;
147
- let filterReads = 0;
148
- let filterAnchorWrites = 0;
149
- const anchorPuts: CreateFilterAnchorInput[] = [];
150
- const activeFilters = seed.activeFilters ?? [];
151
- const filterAnchorRows = seed.filterAnchorRows ?? [];
152
- const threadMessages = seed.threadMessages ?? {};
153
- const client = {
154
- messageLabel: {
155
- apply: async (input: {
156
- messageId: string;
157
- labelId: string;
158
- accountConfigId: string;
159
- appliedByFilterId?: string;
160
- }) => {
161
- assert.equal(
162
- input.appliedByFilterId,
163
- undefined,
164
- "back-apply must never attribute a filter",
165
- );
166
- labeled.push({ messageId: input.messageId, labelId: input.labelId });
167
- return {} as never;
168
- },
169
- },
170
- message: {
171
- get: async (messageId: string) => ({ messageId, mailboxId: "mbox-src" }),
172
- },
173
- mailbox: {
174
- resolveAccountId: async () => "acct-1",
175
- },
176
- threadMessage: {
177
- get: async (_accountConfigId: string, messageId: string) => {
178
- const row = threadMessages[messageId];
179
- if (!row) {
180
- throw new NotFoundError("ThreadMessage not found");
181
- }
182
- return {
183
- threadMessageId: messageId,
184
- fromEmail: row.fromEmail,
185
- fromName: row.fromName,
186
- subject: row.subject,
187
- } as never;
188
- },
189
- },
190
- filter: {
191
- create: async () => {
192
- filterWrites += 1;
193
- return {} as never;
194
- },
195
- listByAccountAndState: async () => {
196
- filterReads += 1;
197
- return activeFilters;
198
- },
199
- refreshExpiry: async (filter: FilterItem) => filter,
200
- },
201
- filterAnchor: {
202
- put: async (input: CreateFilterAnchorInput) => {
203
- filterAnchorWrites += 1;
204
- anchorPuts.push(input);
205
- const row: FilterAnchorItem = { ...input, createdAt: 0, updatedAt: 1 };
206
- const at = filterAnchorRows.findIndex(
207
- (existing) => existing.filterId === input.filterId,
208
- );
209
- if (at === -1) filterAnchorRows.push(row);
210
- else filterAnchorRows[at] = row;
211
- return row;
212
- },
213
- get: async (_accountConfigId: string, filterId: string) =>
214
- filterAnchorRows.find((row) => row.filterId === filterId) ?? null,
215
- listByAccountConfig: async () => filterAnchorRows,
216
- },
217
- } as unknown as RemitClient;
218
- return {
219
- client,
220
- labeled,
221
- filterWrites: () => filterWrites,
222
- filterReads: () => filterReads,
223
- filterAnchorWrites: () => filterAnchorWrites,
224
- anchorPuts,
225
- };
226
- };
227
-
228
- /**
229
- * A stand-in for the local-first placement mover. `moveMessage` is idempotent
230
- * on the (messageId, destination) pair — mirroring the real
231
- * `PlacementMoveService` marker engine (#1297) — so a redelivered back-apply
232
- * re-issues the same move without double-enqueuing. Records every call and the
233
- * final resting mailbox per message.
234
- */
235
- const trackingMoveService = () => {
236
- const moves: Array<{
237
- messageId: string;
238
- destinationMailboxId: string;
239
- accountId: string;
240
- }> = [];
241
- const destinationOf = new Map<string, string>();
242
- let enqueues = 0;
243
- return {
244
- moveService: {
245
- moveMessage: async (
246
- _accountConfigId: string,
247
- messageId: string,
248
- destinationMailboxId: string,
249
- accountId: string,
250
- ): Promise<void> => {
251
- moves.push({ messageId, destinationMailboxId, accountId });
252
- if (destinationOf.get(messageId) === destinationMailboxId) return;
253
- destinationOf.set(messageId, destinationMailboxId);
254
- enqueues += 1;
255
- },
256
- } as unknown as import("@remit/mailbox-service").PlacementMoveService,
257
- moves,
258
- enqueues: () => enqueues,
259
- destinations: () => destinationOf,
260
- };
261
- };
262
-
263
- /**
264
- * Deps whose semantic side is the in-memory vector store — the vector-backed
265
- * deployment. `listAccountFilterMessages` returns the given corpus (empty by
266
- * default), and constructing the semantic side is tracked so a literal-only
267
- * predicate can be shown never to build it. `filterAnchorRows` seeds the
268
- * account's *persisted* FilterAnchor rows (empty by default) — the reverse
269
- * lookup back-apply's anchor consultation reads (reader #350) — and `embed`
270
- * is a deterministic stand-in for the configured embedding model, used only
271
- * by the exclusive-move precedence check.
272
- */
273
- const matchDeps = (
274
- store: ReturnType<typeof createMemoryVectorStore>,
275
- corpus: OrganizeCandidate[] = [],
276
- filterAnchorRows: FilterAnchorItem[] = [],
277
- ): OrganizeMatchDeps & { semanticBuilds: () => number } => {
278
- let semanticBuilds = 0;
279
- return {
280
- semantic: () => {
281
- semanticBuilds += 1;
282
- return {
283
- buildAnchor: async () => anchorPayload,
284
- vectorStore: store,
285
- embed: async (text: string) =>
286
- text.includes("reservation") ? ANCHOR_VECTOR : ORTHOGONAL_VECTOR,
287
- embeddingId: CURRENT_EMBEDDING_ID,
288
- };
289
- },
290
- listAccountFilterMessages: async () => ({ items: corpus }),
291
- filterAnchors: {
292
- get: async () => null,
293
- listByAccountConfig: async () => filterAnchorRows,
294
- put: async () => {
295
- throw new Error("matchDeps must not repair an anchor");
296
- },
297
- },
298
- semanticBuilds: () => semanticBuilds,
299
- };
300
- };
301
-
302
- /**
303
- * Deps modelling a deployment that ships no vector pipeline: any attempt to
304
- * build or use the semantic side raises the missing-module shape, and the
305
- * literal corpus is served from plain rows.
306
- */
307
- const vectorlessDeps = (
308
- corpus: OrganizeCandidate[],
309
- ): OrganizeMatchDeps & { semanticUsed: () => boolean } => {
310
- let semanticUsed = false;
311
- return {
312
- semantic: () => {
313
- semanticUsed = true;
314
- return {
315
- buildAnchor: async () => {
316
- throw moduleNotFound();
317
- },
318
- vectorStore: {
319
- query: async () => {
320
- throw moduleNotFound();
321
- },
322
- getByMessage: async () => {
323
- throw moduleNotFound();
324
- },
325
- },
326
- embed: async () => {
327
- throw moduleNotFound();
328
- },
329
- embeddingId: CURRENT_EMBEDDING_ID,
330
- };
331
- },
332
- listAccountFilterMessages: async () => ({ items: corpus }),
333
- filterAnchors: {
334
- get: async () => null,
335
- listByAccountConfig: async () => [],
336
- put: async () => {
337
- throw moduleNotFound();
338
- },
339
- },
340
- semanticUsed: () => semanticUsed,
341
- };
342
- };
343
-
344
- const candidate = (
345
- messageId: string,
346
- over: Partial<OrganizeCandidate["message"]> = {},
347
- ): OrganizeCandidate => ({
348
- messageId,
349
- message: {
350
- from: "",
351
- fromName: "",
352
- subject: "",
353
- text: "",
354
- listId: "",
355
- ...over,
356
- },
357
- });
358
-
359
- describe("matchOrganize", () => {
360
- it("returns every semantically matching message and excludes the misses", async () => {
361
- const store = createMemoryVectorStore();
362
- const matching = ["msg-1", "msg-2", "msg-3", "msg-4", "msg-5"];
363
- await store.upsert([
364
- ...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
365
- bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
366
- ]);
367
-
368
- const { messageIds, semanticUnavailable } = await matchAccepted(
369
- matchDeps(store),
370
- ACCOUNT_CONFIG_ID,
371
- predicate({ actionLabelId: "lbl-1" }),
372
- );
373
-
374
- assert.deepEqual([...messageIds].sort(), matching);
375
- assert.equal(semanticUnavailable, false);
376
- });
377
-
378
- it("matches nothing when the predicate has neither an anchor nor a clause", async () => {
379
- const store = createMemoryVectorStore();
380
- await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
381
-
382
- const { messageIds } = await matchAccepted(
383
- matchDeps(store),
384
- ACCOUNT_CONFIG_ID,
385
- {
386
- ...predicate(),
387
- anchorMessageId: "None",
388
- },
389
- );
390
-
391
- assert.deepEqual(messageIds, []);
392
- });
393
-
394
- it("refines the semantic set by literal clauses", async () => {
395
- const store = createMemoryVectorStore();
396
- await store.upsert([
397
- bodyChunk("msg-1", ANCHOR_VECTOR, {
398
- subject: "Dinner reservation",
399
- textPreview: "your table is booked",
400
- }),
401
- bodyChunk("msg-2", ANCHOR_VECTOR, {
402
- subject: "Newsletter",
403
- textPreview: "weekly digest",
404
- }),
405
- ]);
406
-
407
- const { messageIds } = await matchAccepted(
408
- matchDeps(store),
409
- ACCOUNT_CONFIG_ID,
410
- {
411
- ...predicate(),
412
- literalClauses: [{ field: "Subject", value: "reservation" }],
413
- },
414
- );
415
-
416
- assert.deepEqual(messageIds, ["msg-1"]);
417
- });
418
-
419
- it("never builds the semantic side for a purely-literal predicate", async () => {
420
- const store = createMemoryVectorStore();
421
- const deps = matchDeps(store, [
422
- candidate("msg-1", { subject: "Dinner reservation" }),
423
- candidate("msg-2", { subject: "Newsletter" }),
424
- ]);
425
-
426
- const { messageIds } = await matchAccepted(deps, ACCOUNT_CONFIG_ID, {
427
- ...predicate(),
428
- anchorMessageId: "None",
429
- literalClauses: [{ field: "Subject", value: "reservation" }],
430
- });
431
-
432
- assert.deepEqual(messageIds, ["msg-1"]);
433
- assert.equal(
434
- deps.semanticBuilds(),
435
- 0,
436
- "a literal-only predicate must not construct the vector-backed side",
437
- );
438
- });
439
- });
440
-
441
- describe("matchOrganize honors the persisted FilterAnchor (reader #350)", () => {
442
- it("still matches by reading the persisted anchor after the anchor message is purged", async () => {
443
- const store = createMemoryVectorStore();
444
- const matching = ["msg-1", "msg-2"];
445
- await store.upsert([
446
- ...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
447
- bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
448
- ]);
449
-
450
- const persistedAnchor: FilterAnchorItem = {
451
- accountConfigId: ACCOUNT_CONFIG_ID,
452
- filterId: "filter-a",
453
- anchorEmbedding: ANCHOR_VECTOR,
454
- anchorEmbeddingId: CURRENT_EMBEDDING_ID,
455
- anchorSourceText: ANCHOR_SOURCE_TEXT,
456
- anchorMessageId: "msg-anchor",
457
- createdAt: 0,
458
- updatedAt: 0,
459
- };
460
-
461
- let buildAnchorCalls = 0;
462
- const deps: OrganizeMatchDeps = {
463
- semantic: () => ({
464
- buildAnchor: async () => {
465
- // The live path: the anchor message's chunks are gone (purged),
466
- // exactly what buildMessageAnchor returns in that case today.
467
- buildAnchorCalls += 1;
468
- return null;
469
- },
470
- vectorStore: store,
471
- embed: async () => ANCHOR_VECTOR,
472
- embeddingId: CURRENT_EMBEDDING_ID,
473
- }),
474
- listAccountFilterMessages: async () => ({ items: [] }),
475
- filterAnchors: {
476
- get: async () => null,
477
- listByAccountConfig: async () => [persistedAnchor],
478
- put: async () => {
479
- throw new Error("a current anchor must not be rewritten");
480
- },
481
- },
482
- };
483
-
484
- const { messageIds } = await matchAccepted(
485
- deps,
486
- ACCOUNT_CONFIG_ID,
487
- predicate(),
488
- );
489
-
490
- assert.deepEqual(
491
- [...messageIds].sort(),
492
- matching,
493
- "the persisted anchor vector still finds the same matches the live message would have",
494
- );
495
- assert.equal(
496
- buildAnchorCalls,
497
- 0,
498
- "a persisted anchor must be read instead of re-deriving one from the (now-gone) live message",
499
- );
500
- });
501
-
502
- it("falls back to deriving the anchor from the live message when no filter was ever anchored on it", async () => {
503
- const store = createMemoryVectorStore();
504
- await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
505
-
506
- // No persisted FilterAnchor names this anchorMessageId — an ad hoc "all
507
- // like these" widen over a bare message selection, never tied to a
508
- // standing filter.
509
- const { messageIds } = await matchAccepted(
510
- matchDeps(store),
511
- ACCOUNT_CONFIG_ID,
512
- predicate(),
513
- );
514
-
515
- assert.deepEqual(messageIds, ["msg-1"]);
516
- });
517
- });
518
-
519
- /**
520
- * A zero from the widen has two meanings and the preview could not tell them
521
- * apart (issue #452): no mail resembles the anchor, or the index holds nothing
522
- * to compare it against. Only the second is a reason to keep the rule.
523
- */
524
- describe("matchOrganize reporting an empty semantic index", () => {
525
- it("flags a widen answered by an index with no vectors at all", async () => {
526
- const { messageIds, semanticIndexEmpty, semanticUnavailable } =
527
- await matchAccepted(
528
- matchDeps(createMemoryVectorStore()),
529
- ACCOUNT_CONFIG_ID,
530
- predicate(),
531
- );
532
-
533
- assert.deepEqual(messageIds, []);
534
- assert.equal(semanticIndexEmpty, true);
535
- assert.equal(
536
- semanticUnavailable,
537
- false,
538
- "an empty index is not a missing vector pipeline",
539
- );
540
- });
541
-
542
- it("flags a widen whose anchor message has no chunk vectors to pool", async () => {
543
- const store = createMemoryVectorStore();
544
- await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
545
- const deps: OrganizeMatchDeps = {
546
- ...matchDeps(store),
547
- semantic: () => ({
548
- buildAnchor: async () => null,
549
- vectorStore: store,
550
- embed: async () => ANCHOR_VECTOR,
551
- embeddingId: CURRENT_EMBEDDING_ID,
552
- }),
553
- };
554
-
555
- const { messageIds, semanticIndexEmpty } = await matchAccepted(
556
- deps,
557
- ACCOUNT_CONFIG_ID,
558
- predicate(),
559
- );
560
-
561
- assert.deepEqual(messageIds, []);
562
- assert.equal(semanticIndexEmpty, true);
563
- });
564
-
565
- it("does not flag a populated index that simply holds nothing similar enough", async () => {
566
- const store = createMemoryVectorStore();
567
- await store.upsert([bodyChunk("msg-1", ORTHOGONAL_VECTOR)]);
568
-
569
- const { messageIds, semanticIndexEmpty } = await matchAccepted(
570
- matchDeps(store),
571
- ACCOUNT_CONFIG_ID,
572
- predicate(),
573
- );
574
-
575
- assert.deepEqual(
576
- messageIds,
577
- [],
578
- "the one indexed message is below threshold",
579
- );
580
- assert.equal(
581
- semanticIndexEmpty,
582
- false,
583
- "the index answered; the rule is what matched nothing",
584
- );
585
- });
586
-
587
- it("does not flag a widen that matched", async () => {
588
- const store = createMemoryVectorStore();
589
- await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
590
-
591
- const { messageIds, semanticIndexEmpty } = await matchAccepted(
592
- matchDeps(store),
593
- ACCOUNT_CONFIG_ID,
594
- predicate(),
595
- );
596
-
597
- assert.deepEqual(messageIds, ["msg-1"]);
598
- assert.equal(semanticIndexEmpty, false);
599
- });
600
-
601
- it("does not flag a literal-only predicate, which never consults the index", async () => {
602
- const { semanticIndexEmpty } = await matchAccepted(
603
- matchDeps(createMemoryVectorStore(), [
604
- candidate("msg-1", { subject: "Weekly newsletter" }),
605
- ]),
606
- ACCOUNT_CONFIG_ID,
607
- {
608
- ...predicate(),
609
- anchorMessageId: "None",
610
- literalClauses: [{ field: "Subject", value: "reservation" }],
611
- },
612
- );
613
-
614
- assert.equal(semanticIndexEmpty, false);
615
- });
616
- });
617
-
618
- describe("matchOrganize on a deployment without the vector pipeline", () => {
619
- const ORIGINAL = process.env.DATA_BACKEND;
620
- beforeEach(() => {
621
- process.env.DATA_BACKEND = "sqlite";
622
- _resetSemanticCapabilityForTest();
623
- });
624
- afterEach(() => {
625
- if (ORIGINAL === undefined) delete process.env.DATA_BACKEND;
626
- else process.env.DATA_BACKEND = ORIGINAL;
627
- _resetSemanticCapabilityForTest();
628
- });
629
-
630
- it("matches a literal predicate from the corpus without touching the vector store", async () => {
631
- const deps = vectorlessDeps([
632
- candidate("msg-1", { subject: "Dinner reservation" }),
633
- candidate("msg-2", { subject: "Weekly newsletter" }),
634
- candidate("msg-3", { subject: "Table reservation confirmed" }),
635
- ]);
636
-
637
- const { messageIds, semanticUnavailable } = await matchAccepted(
638
- deps,
639
- ACCOUNT_CONFIG_ID,
640
- {
641
- ...predicate(),
642
- anchorMessageId: "None",
643
- literalClauses: [{ field: "Subject", value: "reservation" }],
644
- },
645
- );
646
-
647
- assert.deepEqual(messageIds, ["msg-1", "msg-3"]);
648
- assert.equal(semanticUnavailable, false);
649
- assert.equal(
650
- deps.semanticUsed(),
651
- false,
652
- "a literal-only predicate must never reach the vector store",
653
- );
654
- });
655
-
656
- it("refuses a body-content (HasWords) clause as a result rather than matching it against a preview", async () => {
657
- const deps = vectorlessDeps([
658
- candidate("msg-1", { subject: "Dinner reservation" }),
659
- ]);
660
-
661
- const result = await matchOrganize(deps, ACCOUNT_CONFIG_ID, {
662
- ...predicate(),
663
- anchorMessageId: "None",
664
- literalClauses: [{ field: "HasWords", value: "invoice" }],
665
- });
666
-
667
- assert.deepEqual(
668
- result.rejected,
669
- {
670
- reason: "BodyContentWithoutVectorPipeline",
671
- message: BODY_CONTENT_REJECTION_MESSAGE,
672
- },
673
- "the vector-free literal path must not silently narrow a body match to a preview",
674
- );
675
- assert.equal(deps.semanticUsed(), false);
676
- });
677
-
678
- // The refusal is the same whichever way the vector-free path is reached, so
679
- // an anchored predicate that degrades onto it cannot land in the corpus scan
680
- // with a body clause it can only mis-evaluate (reader #463).
681
- it("refuses a body-content clause reached through the degraded widen fallback", async () => {
682
- const deps = vectorlessDeps([
683
- candidate("msg-1", { subject: "Dinner reservation" }),
684
- ]);
685
-
686
- const result = await matchOrganize(deps, ACCOUNT_CONFIG_ID, {
687
- ...predicate(),
688
- literalClauses: [{ field: "HasWords", value: "invoice" }],
689
- });
690
-
691
- assert.equal(result.rejected?.reason, "BodyContentWithoutVectorPipeline");
692
- });
693
-
694
- it("degrades an anchor+clauses widen to the literal matches, flagged, instead of crashing", async () => {
695
- const deps = vectorlessDeps([
696
- candidate("msg-1", { subject: "Dinner reservation" }),
697
- candidate("msg-2", { subject: "Weekly newsletter" }),
698
- ]);
699
-
700
- const { messageIds, semanticUnavailable } = await matchAccepted(
701
- deps,
702
- ACCOUNT_CONFIG_ID,
703
- {
704
- ...predicate(),
705
- literalClauses: [{ field: "Subject", value: "reservation" }],
706
- },
707
- );
708
-
709
- assert.deepEqual(messageIds, ["msg-1"]);
710
- assert.equal(semanticUnavailable, true);
711
- });
712
-
713
- it("degrades an anchor-only widen to an empty flagged result instead of crashing", async () => {
714
- const deps = vectorlessDeps([candidate("msg-1"), candidate("msg-2")]);
715
-
716
- const { messageIds, semanticUnavailable, semanticIndexEmpty } =
717
- await matchAccepted(deps, ACCOUNT_CONFIG_ID, predicate());
718
-
719
- assert.deepEqual(messageIds, []);
720
- assert.equal(semanticUnavailable, true);
721
- assert.equal(
722
- semanticIndexEmpty,
723
- false,
724
- "a deployment with no vector pipeline has no index to call empty",
725
- );
726
- });
727
-
728
- it("keeps widening from a drifted persisted anchor when no embedding model is available", async () => {
729
- // Self-host: sqlite-vec is present, `@huggingface/transformers` is not.
730
- // The anchor's stamp will never match — after a model swap, or forever
731
- // for one pooled from pre-#349 chunks and stamped
732
- // UNKNOWN_CHUNK_EMBEDDING_ID — and there is no embedder to repair it
733
- // with, so the widen must query with the vector it has rather than go
734
- // dark and take every later /search/semantic with it.
735
- const store = createMemoryVectorStore();
736
- await store.upsert([
737
- bodyChunk("msg-1", ANCHOR_VECTOR),
738
- bodyChunk("msg-stored-space", ORTHOGONAL_VECTOR),
739
- ]);
740
- const drifted: FilterAnchorItem = {
741
- accountConfigId: ACCOUNT_CONFIG_ID,
742
- filterId: "filter-a",
743
- anchorEmbedding: ORTHOGONAL_VECTOR,
744
- anchorEmbeddingId: STALE_EMBEDDING_ID,
745
- anchorSourceText: ANCHOR_SOURCE_TEXT,
746
- anchorMessageId: "msg-anchor",
747
- createdAt: 0,
748
- updatedAt: 0,
749
- };
750
- const deps: OrganizeMatchDeps = {
751
- semantic: () => ({
752
- buildAnchor: async () => {
753
- throw new Error("a persisted anchor must not be re-derived");
754
- },
755
- vectorStore: store,
756
- embed: async () => {
757
- throw moduleNotFound();
758
- },
759
- embeddingId: CURRENT_EMBEDDING_ID,
760
- }),
761
- listAccountFilterMessages: async () => ({ items: [] }),
762
- filterAnchors: {
763
- get: async () => null,
764
- listByAccountConfig: async () => [drifted],
765
- put: async () => {
766
- throw new Error("an unrepairable anchor must not be rewritten");
767
- },
768
- },
769
- };
770
-
771
- const { messageIds, semanticUnavailable } = await matchAccepted(
772
- deps,
773
- ACCOUNT_CONFIG_ID,
774
- predicate(),
775
- );
776
-
777
- assert.deepEqual(
778
- messageIds,
779
- ["msg-stored-space"],
780
- "the kNN read needs no embedder, so the stored anchor vector still returns real hits",
781
- );
782
- assert.equal(semanticUnavailable, false);
783
- assert.equal(
784
- isSemanticSearchUnavailable(),
785
- false,
786
- "one unrepairable anchor must not disable free-text semantic search for the rest of the process",
787
- );
788
- });
789
-
790
- it("propagates a genuine (non-capability) semantic failure loudly", async () => {
791
- const deps: OrganizeMatchDeps = {
792
- semantic: () => ({
793
- buildAnchor: async () => {
794
- throw new Error("SQLITE_BUSY");
795
- },
796
- vectorStore: {
797
- query: async () => [],
798
- getByMessage: async () => [],
799
- },
800
- embed: async () => [],
801
- embeddingId: CURRENT_EMBEDDING_ID,
802
- }),
803
- listAccountFilterMessages: async () => ({ items: [] }),
804
- filterAnchors: {
805
- get: async () => null,
806
- listByAccountConfig: async () => [],
807
- put: async () => {
808
- throw new Error("unreachable");
809
- },
810
- },
811
- };
812
-
813
- await assert.rejects(
814
- () => matchOrganize(deps, ACCOUNT_CONFIG_ID, predicate()),
815
- /SQLITE_BUSY/,
816
- );
817
- });
818
- });
819
-
820
- describe("back-apply pipeline (matchOrganize -> applyOrganize)", () => {
821
- it("applies the label to all N matches in one pass and writes zero Filter rows", async () => {
822
- const store = createMemoryVectorStore();
823
- const matching = ["msg-1", "msg-2", "msg-3", "msg-4", "msg-5"];
824
- await store.upsert([
825
- ...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
826
- bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
827
- ]);
828
-
829
- const p = predicate({ actionLabelId: "lbl-1" });
830
-
831
- // The preview and the apply share the same matcher — the previewed set is
832
- // exactly what gets applied.
833
- const previewed = await matchAccepted(
834
- matchDeps(store),
835
- ACCOUNT_CONFIG_ID,
836
- p,
837
- );
838
- const applied = await matchAccepted(matchDeps(store), ACCOUNT_CONFIG_ID, p);
839
- assert.deepEqual(previewed, applied);
840
-
841
- const tracked = trackingClient();
842
- const result = await applyOrganize(
843
- { client: tracked.client },
844
- ACCOUNT_CONFIG_ID,
845
- applied.messageIds,
846
- p,
847
- );
848
-
849
- assert.equal(result.applied, matching.length);
850
- assert.equal(result.failed, 0);
851
- assert.deepEqual(
852
- tracked.labeled.map((row) => row.messageId).sort(),
853
- matching,
854
- );
855
- assert.ok(
856
- tracked.labeled.every((row) => row.labelId === "lbl-1"),
857
- "every matching message gets the requested label",
858
- );
859
- assert.equal(tracked.filterWrites(), 0, "no Filter row is ever created");
860
- assert.equal(
861
- tracked.filterAnchorWrites(),
862
- 0,
863
- "no FilterAnchor row is ever created",
864
- );
865
- });
866
-
867
- it("counts a requested move as failed when no move service is wired", async () => {
868
- const store = createMemoryVectorStore();
869
- await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
870
- const p = predicate({ actionMailboxId: "mbox-target" });
871
-
872
- const { messageIds: matched } = await matchAccepted(
873
- matchDeps(store),
874
- ACCOUNT_CONFIG_ID,
875
- p,
876
- );
877
- const tracked = trackingClient();
878
- const result = await applyOrganize(
879
- { client: tracked.client },
880
- ACCOUNT_CONFIG_ID,
881
- matched,
882
- p,
883
- );
884
-
885
- assert.equal(result.applied, 0);
886
- assert.equal(result.failed, 1);
887
- });
888
-
889
- it("moves every match through the wired move service and writes zero Filter rows", async () => {
890
- const store = createMemoryVectorStore();
891
- const matching = ["msg-1", "msg-2", "msg-3"];
892
- await store.upsert([
893
- ...matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)),
894
- bodyChunk("msg-miss", ORTHOGONAL_VECTOR),
895
- ]);
896
- const p = predicate({ actionMailboxId: "mbox-target" });
897
-
898
- const { messageIds: matched } = await matchAccepted(
899
- matchDeps(store),
900
- ACCOUNT_CONFIG_ID,
901
- p,
902
- );
903
- const tracked = trackingClient();
904
- const mover = trackingMoveService();
905
- const result = await applyOrganize(
906
- {
907
- client: tracked.client,
908
- moveService: mover.moveService,
909
- },
910
- ACCOUNT_CONFIG_ID,
911
- matched,
912
- p,
913
- );
914
-
915
- assert.equal(result.applied, matching.length);
916
- assert.equal(result.failed, 0);
917
- assert.deepEqual(
918
- mover.moves.map((m) => m.messageId).sort(),
919
- matching,
920
- "every matched message is moved once",
921
- );
922
- assert.ok(
923
- mover.moves.every((m) => m.destinationMailboxId === "mbox-target"),
924
- "every move targets the requested mailbox",
925
- );
926
- assert.equal(
927
- tracked.labeled.length,
928
- 0,
929
- "a move-only action applies no label",
930
- );
931
- assert.equal(tracked.filterWrites(), 0, "no Filter row is ever created");
932
- assert.equal(
933
- tracked.filterAnchorWrites(),
934
- 0,
935
- "no FilterAnchor row is ever created",
936
- );
937
- });
938
-
939
- it("applies both a label and a move when both actions are requested", async () => {
940
- const store = createMemoryVectorStore();
941
- await store.upsert([bodyChunk("msg-1", ANCHOR_VECTOR)]);
942
- const p = predicate({
943
- actionLabelId: "lbl-1",
944
- actionMailboxId: "mbox-target",
945
- });
946
-
947
- const { messageIds: matched } = await matchAccepted(
948
- matchDeps(store),
949
- ACCOUNT_CONFIG_ID,
950
- p,
951
- );
952
- const tracked = trackingClient();
953
- const mover = trackingMoveService();
954
- const result = await applyOrganize(
955
- {
956
- client: tracked.client,
957
- moveService: mover.moveService,
958
- },
959
- ACCOUNT_CONFIG_ID,
960
- matched,
961
- p,
962
- );
963
-
964
- assert.equal(result.applied, 1);
965
- assert.equal(result.failed, 0);
966
- assert.deepEqual(tracked.labeled, [
967
- { messageId: "msg-1", labelId: "lbl-1" },
968
- ]);
969
- assert.deepEqual(
970
- mover.moves.map((m) => m.messageId),
971
- ["msg-1"],
972
- );
973
- });
974
-
975
- it("is idempotent on redelivery: a re-run re-issues the same move without double-enqueuing", async () => {
976
- const store = createMemoryVectorStore();
977
- const matching = ["msg-1", "msg-2"];
978
- await store.upsert(matching.map((id) => bodyChunk(id, ANCHOR_VECTOR)));
979
- const p = predicate({ actionMailboxId: "mbox-target" });
980
-
981
- const { messageIds: matched } = await matchAccepted(
982
- matchDeps(store),
983
- ACCOUNT_CONFIG_ID,
984
- p,
985
- );
986
- const tracked = trackingClient();
987
- const mover = trackingMoveService();
988
-
989
- const first = await applyOrganize(
990
- {
991
- client: tracked.client,
992
- moveService: mover.moveService,
993
- },
994
- ACCOUNT_CONFIG_ID,
995
- matched,
996
- p,
997
- );
998
- const second = await applyOrganize(
999
- {
1000
- client: tracked.client,
1001
- moveService: mover.moveService,
1002
- },
1003
- ACCOUNT_CONFIG_ID,
1004
- matched,
1005
- p,
1006
- );
1007
-
1008
- assert.equal(first.applied, matching.length);
1009
- assert.equal(second.applied, matching.length);
1010
- assert.equal(
1011
- mover.enqueues(),
1012
- matching.length,
1013
- "the second pass drives the marker forward without a fresh enqueue",
1014
- );
1015
- assert.deepEqual(
1016
- [...mover.destinations().entries()].sort(),
1017
- matching.map((id): [string, string] => [id, "mbox-target"]),
1018
- "each message rests in the requested mailbox exactly once",
1019
- );
1020
- });
1021
- });
1022
-
1023
- /**
1024
- * A standing filter acts when a message is first seen and when the user presses
1025
- * "run rules now", and holds no claim afterwards. A later apply is the newer
1026
- * event, so it moves the mail — and because nothing is suppressed, `applied`
1027
- * counts moves that happened rather than moves that were skipped (reader #497).
1028
- */
1029
- describe("a user-initiated apply outranks every standing filter (reader #497)", () => {
1030
- it("moves a hand-picked selection a standing filter claims for a different mailbox", async () => {
1031
- const store = createMemoryVectorStore();
1032
- const picked = ["msg-1", "msg-2", "msg-3"];
1033
- await store.upsert(picked.map((id) => bodyChunk(id, ANCHOR_VECTOR)));
1034
- const p = predicate({ actionMailboxId: "mbox-projects" });
1035
- // The standing rule that filed all three into Archive in the first place —
1036
- // moving them back out is the whole point of the button.
1037
- const standing = filterItem({
1038
- filterId: "filter-github",
1039
- ruleChangedAt: 1_000,
1040
- actionChangedAt: 1_000,
1041
- actionMailboxId: "mbox-archive",
1042
- literalClauses: [{ field: "Subject", value: "reservation" }],
1043
- });
1044
-
1045
- const { messageIds: matched } = await matchAccepted(
1046
- matchDeps(store),
1047
- ACCOUNT_CONFIG_ID,
1048
- p,
1049
- );
1050
- const tracked = trackingClient({
1051
- activeFilters: [standing],
1052
- threadMessages: Object.fromEntries(
1053
- picked.map((id) => [id, { subject: "Dinner reservation" }]),
1054
- ),
1055
- });
1056
- const mover = trackingMoveService();
1057
- const result = await applyOrganize(
1058
- { client: tracked.client, moveService: mover.moveService },
1059
- ACCOUNT_CONFIG_ID,
1060
- matched,
1061
- p,
1062
- );
1063
-
1064
- assert.deepEqual(
1065
- mover.moves.map((move) => move.messageId).sort(),
1066
- picked,
1067
- "every hand-picked message moves",
1068
- );
1069
- assert.ok(
1070
- mover.moves.every(
1071
- (move) => move.destinationMailboxId === "mbox-projects",
1072
- ),
1073
- "the destination is the one the user picked, not the standing filter's",
1074
- );
1075
- assert.equal(
1076
- result.applied,
1077
- mover.moves.length,
1078
- "the reported count equals the moves that actually happened",
1079
- );
1080
- assert.equal(result.failed, 0);
1081
- assert.equal(
1082
- tracked.filterReads(),
1083
- 0,
1084
- "the account's standing filters are not consulted at all",
1085
- );
1086
- });
1087
- });
1088
-
1089
- /**
1090
- * A same-dimension embedding-model swap leaves every persisted FilterAnchor
1091
- * stamped with the old model's id and its vector in the old model's space.
1092
- * Index-time matching repairs that lazily (RFC 039 Decision 1a); the
1093
- * back-apply widen must do the same, or it queries with a stale-model vector
1094
- * and silently disagrees with the pipeline it claims to mirror (reader #399).
1095
- */
1096
- describe("back-apply repairs a drifted anchor the way index-time matching does (reader #399)", () => {
1097
- // Written under the previous model: same dimensions, different space, so
1098
- // nothing but the id stamp distinguishes it from a usable anchor.
1099
- const staleAnchor = (filterId: string): FilterAnchorItem => ({
1100
- accountConfigId: ACCOUNT_CONFIG_ID,
1101
- filterId,
1102
- anchorEmbedding: ORTHOGONAL_VECTOR,
1103
- anchorEmbeddingId: STALE_EMBEDDING_ID,
1104
- anchorSourceText: ANCHOR_SOURCE_TEXT,
1105
- anchorMessageId: "msg-anchor",
1106
- createdAt: 0,
1107
- updatedAt: 0,
1108
- });
1109
-
1110
- /** Every text embeds into the current model's space. */
1111
- const currentModelDeps = (
1112
- store: ReturnType<typeof createMemoryVectorStore>,
1113
- anchors: FilterAnchorItem[],
1114
- embed: (text: string) => Promise<number[]> = async () => ANCHOR_VECTOR,
1115
- ): OrganizeMatchDeps & { anchorPuts: CreateFilterAnchorInput[] } => {
1116
- const anchorPuts: CreateFilterAnchorInput[] = [];
1117
- return {
1118
- semantic: () => ({
1119
- buildAnchor: async () => {
1120
- throw new Error("a persisted anchor must be repaired, not replaced");
1121
- },
1122
- vectorStore: store,
1123
- embed,
1124
- embeddingId: CURRENT_EMBEDDING_ID,
1125
- }),
1126
- listAccountFilterMessages: async () => ({ items: [] }),
1127
- filterAnchors: {
1128
- get: async () => null,
1129
- listByAccountConfig: async () => anchors,
1130
- put: async (input: CreateFilterAnchorInput) => {
1131
- anchorPuts.push(input);
1132
- return { ...input, createdAt: 0, updatedAt: 1 };
1133
- },
1134
- },
1135
- anchorPuts,
1136
- };
1137
- };
1138
-
1139
- it("matchSemantic re-embeds a drifted persisted anchor instead of querying with the stale vector", async () => {
1140
- const store = createMemoryVectorStore();
1141
- await store.upsert([
1142
- bodyChunk("msg-1", ANCHOR_VECTOR),
1143
- bodyChunk("msg-stale-space", ORTHOGONAL_VECTOR),
1144
- ]);
1145
- const deps = currentModelDeps(store, [staleAnchor("filter-a")]);
1146
-
1147
- const { messageIds } = await matchAccepted(
1148
- deps,
1149
- ACCOUNT_CONFIG_ID,
1150
- predicate(),
1151
- );
1152
-
1153
- assert.deepEqual(
1154
- messageIds,
1155
- ["msg-1"],
1156
- "the widen runs on the re-embedded anchor, not the stale-space vector that would have matched msg-stale-space",
1157
- );
1158
- assert.deepEqual(
1159
- deps.anchorPuts.map((put) => [put.filterId, put.anchorEmbeddingId]),
1160
- [["filter-a", CURRENT_EMBEDDING_ID]],
1161
- "the repair is written back in place under the current model's id",
1162
- );
1163
- assert.deepEqual(deps.anchorPuts[0].anchorEmbedding, ANCHOR_VECTOR);
1164
- assert.equal(deps.anchorPuts[0].anchorSourceText, ANCHOR_SOURCE_TEXT);
1165
- });
1166
- });
1167
-
1168
- describe("matchOrganize with ListId and FromDomain clauses", () => {
1169
- const senderChunk = (
1170
- messageId: string,
1171
- fromEmail: string,
1172
- over: Partial<ChunkMetadata> = {},
1173
- ): VectorRecord => ({
1174
- chunkId: `${messageId}#sender`,
1175
- vector: ANCHOR_VECTOR,
1176
- metadata: metadata({
1177
- messageId,
1178
- chunkType: "sender",
1179
- textPreview: fromEmail,
1180
- ...over,
1181
- }),
1182
- });
1183
-
1184
- it("matches a ListId clause exactly off the vector-free projection", async () => {
1185
- const deps = vectorlessDeps([
1186
- candidate("msg-1", { listId: "weekly.news.example.com" }),
1187
- candidate("msg-2", { listId: "news.example.com" }),
1188
- candidate("msg-3", {}),
1189
- ]);
1190
-
1191
- const { messageIds } = await matchAccepted(deps, ACCOUNT_CONFIG_ID, {
1192
- ...predicate(),
1193
- anchorMessageId: "None",
1194
- literalClauses: [{ field: "ListId", value: "weekly.news.example.com" }],
1195
- });
1196
-
1197
- assert.deepEqual(messageIds, ["msg-1"]);
1198
- });
1199
-
1200
- it("matches a FromDomain clause public-suffix aware off the vector-free projection", async () => {
1201
- const deps = vectorlessDeps([
1202
- candidate("msg-1", { from: "notifications@github.com" }),
1203
- candidate("msg-2", { from: "ci@sub.github.com" }),
1204
- candidate("msg-3", { from: "attacker@github.com.evil.example" }),
1205
- ]);
1206
-
1207
- const { messageIds } = await matchAccepted(deps, ACCOUNT_CONFIG_ID, {
1208
- ...predicate(),
1209
- anchorMessageId: "None",
1210
- literalClauses: [{ field: "FromDomain", value: "github.com" }],
1211
- });
1212
-
1213
- assert.deepEqual(messageIds, ["msg-1", "msg-2"]);
1214
- });
1215
-
1216
- it("round-trips ListId and FromDomain through the semantic-arm chunk projection", async () => {
1217
- const store = createMemoryVectorStore();
1218
- await store.upsert([
1219
- senderChunk("msg-1", "ci@github.com", {
1220
- listId: "actions.github.com",
1221
- }),
1222
- bodyChunk("msg-1", ANCHOR_VECTOR, { listId: "actions.github.com" }),
1223
- senderChunk("msg-2", "hi@othersender.example", {
1224
- listId: "other.list.example",
1225
- }),
1226
- bodyChunk("msg-2", ANCHOR_VECTOR, { listId: "other.list.example" }),
1227
- ]);
1228
-
1229
- const byListId = await matchAccepted(matchDeps(store), ACCOUNT_CONFIG_ID, {
1230
- ...predicate(),
1231
- literalClauses: [{ field: "ListId", value: "actions.github.com" }],
1232
- });
1233
- assert.deepEqual(byListId.messageIds, ["msg-1"]);
1234
-
1235
- const byFromDomain = await matchAccepted(
1236
- matchDeps(store),
1237
- ACCOUNT_CONFIG_ID,
1238
- {
1239
- ...predicate(),
1240
- literalClauses: [{ field: "FromDomain", value: "github.com" }],
1241
- },
1242
- );
1243
- assert.deepEqual(byFromDomain.messageIds, ["msg-1"]);
1244
- });
1245
-
1246
- it("applies a ListId predicate to exactly the previewed set (preview == apply)", async () => {
1247
- const deps = vectorlessDeps([
1248
- candidate("msg-1", { listId: "weekly.news.example.com" }),
1249
- candidate("msg-2", { listId: "weekly.news.example.com" }),
1250
- candidate("msg-3", { listId: "other.example.com" }),
1251
- ]);
1252
- const p = predicate({
1253
- anchorMessageId: "None",
1254
- actionLabelId: "lbl-list",
1255
- literalClauses: [{ field: "ListId", value: "weekly.news.example.com" }],
1256
- });
1257
-
1258
- const previewed = await matchAccepted(deps, ACCOUNT_CONFIG_ID, p);
1259
- const applied = await matchAccepted(deps, ACCOUNT_CONFIG_ID, p);
1260
- assert.deepEqual(previewed, applied);
1261
- assert.deepEqual(previewed.messageIds, ["msg-1", "msg-2"]);
1262
-
1263
- const tracked = trackingClient();
1264
- const result = await applyOrganize(
1265
- { client: tracked.client },
1266
- ACCOUNT_CONFIG_ID,
1267
- applied.messageIds,
1268
- p,
1269
- );
1270
-
1271
- assert.equal(result.applied, 2);
1272
- assert.equal(result.failed, 0);
1273
- assert.deepEqual(tracked.labeled.map((row) => row.messageId).sort(), [
1274
- "msg-1",
1275
- "msg-2",
1276
- ]);
1277
- });
1278
- });
1279
-
1280
- /**
1281
- * The corpus read a real deployment gets: the store evaluates the terms and
1282
- * the caller pages the MATCHES. Modelled here rather than stubbed with a fixed
1283
- * array so a matcher that reads only the newest page cannot pass.
1284
- */
1285
- const storeBackedDeps = (
1286
- rows: readonly OrganizeCandidate[],
1287
- ): OrganizeMatchDeps & { pagesRead: () => number } => {
1288
- let pagesRead = 0;
1289
- const termMatches = (
1290
- term: { field: string; contains: string },
1291
- row: OrganizeCandidate,
1292
- ): boolean => {
1293
- const needle = term.contains.toLowerCase();
1294
- if (term.field === "subject") {
1295
- return row.message.subject.toLowerCase().includes(needle);
1296
- }
1297
- if (term.field === "listId") {
1298
- return row.message.listId.toLowerCase().includes(needle);
1299
- }
1300
- return `${row.message.fromName} ${row.message.from}`
1301
- .toLowerCase()
1302
- .includes(needle);
1303
- };
1304
- return {
1305
- semantic: () => {
1306
- throw moduleNotFound();
1307
- },
1308
- listAccountFilterMessages: async (_accountConfigId, query) => {
1309
- pagesRead += 1;
1310
- const narrowed = rows.filter((row) =>
1311
- query.terms.length === 0
1312
- ? true
1313
- : query.operator === "or"
1314
- ? query.terms.some((term) => termMatches(term, row))
1315
- : query.terms.every((term) => termMatches(term, row)),
1316
- );
1317
- const offset = query.continuationToken
1318
- ? Number(query.continuationToken)
1319
- : 0;
1320
- const items = narrowed.slice(offset, offset + query.limit);
1321
- const next = offset + items.length;
1322
- return {
1323
- items,
1324
- continuationToken: next < narrowed.length ? String(next) : undefined,
1325
- };
1326
- },
1327
- filterAnchors: {
1328
- get: async () => null,
1329
- listByAccountConfig: async () => [],
1330
- put: async () => {
1331
- throw new Error("unreachable");
1332
- },
1333
- },
1334
- pagesRead: () => pagesRead,
1335
- };
1336
- };
1337
-
1338
- describe("literal matching bounds the result, not the corpus (reader #459)", () => {
1339
- const quietSender = candidate("quiet-sender", {
1340
- from: "noreply@bank.example",
1341
- fromName: "Statements",
1342
- subject: "Your March statement",
1343
- });
1344
- const newerNoise = Array.from({ length: ORGANIZE_MATCH_LIMIT + 1 }, (_, i) =>
1345
- candidate(`noise-${i}`, {
1346
- from: "digest@other.example",
1347
- fromName: "Digest",
1348
- subject: `Daily digest ${i}`,
1349
- }),
1350
- );
1351
-
1352
- it("matches a domain rule against mail older than a whole page of newer misses", async () => {
1353
- const deps = storeBackedDeps([...newerNoise, quietSender]);
1354
-
1355
- const { messageIds } = await matchAccepted(
1356
- deps,
1357
- ACCOUNT_CONFIG_ID,
1358
- predicate({
1359
- anchorMessageId: "None",
1360
- literalClauses: [{ field: "FromDomain", value: "bank.example" }],
1361
- }),
1362
- );
1363
-
1364
- assert.deepEqual(messageIds, ["quiet-sender"]);
1365
- });
1366
-
1367
- it("reads one narrowed page rather than paging the whole mailbox", async () => {
1368
- const deps = storeBackedDeps([...newerNoise, quietSender]);
1369
-
1370
- await matchAccepted(
1371
- deps,
1372
- ACCOUNT_CONFIG_ID,
1373
- predicate({
1374
- anchorMessageId: "None",
1375
- literalClauses: [{ field: "From", value: "noreply@bank.example" }],
1376
- }),
1377
- );
1378
-
1379
- assert.equal(deps.pagesRead(), 1);
1380
- });
1381
-
1382
- it("still refines what the store returned: a term is a narrowing, not the verdict", async () => {
1383
- const deps = storeBackedDeps([
1384
- candidate("spoofed", { from: "billing@bank.example.evil.test" }),
1385
- quietSender,
1386
- ]);
1387
-
1388
- const { messageIds } = await matchAccepted(
1389
- deps,
1390
- ACCOUNT_CONFIG_ID,
1391
- predicate({
1392
- anchorMessageId: "None",
1393
- literalClauses: [{ field: "FromDomain", value: "bank.example" }],
1394
- }),
1395
- );
1396
-
1397
- assert.deepEqual(messageIds, ["quiet-sender"]);
1398
- });
1399
-
1400
- it("caps the matches at the limit when more of the corpus matches", async () => {
1401
- const matching = Array.from({ length: ORGANIZE_MATCH_LIMIT * 2 }, (_, i) =>
1402
- candidate(`hit-${i}`, { from: "noreply@bank.example" }),
1403
- );
1404
- const deps = storeBackedDeps(matching);
1405
-
1406
- const { messageIds } = await matchAccepted(
1407
- deps,
1408
- ACCOUNT_CONFIG_ID,
1409
- predicate({
1410
- anchorMessageId: "None",
1411
- literalClauses: [{ field: "FromDomain", value: "bank.example" }],
1412
- }),
1413
- );
1414
-
1415
- assert.equal(messageIds.length, ORGANIZE_MATCH_LIMIT);
1416
- });
1417
-
1418
- it("never queries for clauses nothing can satisfy", async () => {
1419
- const deps = storeBackedDeps([quietSender]);
1420
-
1421
- const { messageIds } = await matchAccepted(
1422
- deps,
1423
- ACCOUNT_CONFIG_ID,
1424
- predicate({
1425
- anchorMessageId: "None",
1426
- literalClauses: [{ field: "From", value: " " }],
1427
- }),
1428
- );
1429
-
1430
- assert.deepEqual(messageIds, []);
1431
- assert.equal(deps.pagesRead(), 0);
1432
- });
1433
-
1434
- it("counts one match for mail filed in two folders", async () => {
1435
- const deps = storeBackedDeps([
1436
- candidate("filed-twice", { from: "noreply@bank.example" }),
1437
- candidate("filed-twice", { from: "noreply@bank.example" }),
1438
- ]);
1439
-
1440
- const { messageIds } = await matchAccepted(
1441
- deps,
1442
- ACCOUNT_CONFIG_ID,
1443
- predicate({
1444
- anchorMessageId: "None",
1445
- literalClauses: [{ field: "FromDomain", value: "bank.example" }],
1446
- }),
1447
- );
1448
-
1449
- assert.deepEqual(messageIds, ["filed-twice"]);
1450
- });
1451
- });