@remit/backend 0.0.143 → 0.0.145

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (279) hide show
  1. package/dist/dev-server/content-handler.d.ts +38 -0
  2. package/dist/dev-server/content-handler.js +46 -0
  3. package/dist/src/api.d.ts +5 -0
  4. package/dist/src/api.js +73 -0
  5. package/dist/src/auth.d.ts +24 -0
  6. package/dist/src/auth.js +81 -0
  7. package/dist/src/config/api-config.d.ts +20 -0
  8. package/dist/src/config/api-config.js +1 -0
  9. package/dist/src/config/from-env.d.ts +2 -0
  10. package/dist/src/config/from-env.js +65 -0
  11. package/dist/src/config/msoauth-secrets-manager.d.ts +2 -0
  12. package/dist/src/config/msoauth-secrets-manager.js +28 -0
  13. package/dist/src/config/msoauth.d.ts +19 -0
  14. package/dist/src/config/msoauth.js +6 -0
  15. package/{src/derive/autoMoved.ts → dist/src/derive/autoMoved.d.ts} +1 -29
  16. package/dist/src/derive/autoMoved.js +49 -0
  17. package/dist/src/derive/contentSignature.d.ts +40 -0
  18. package/{src/derive/contentSignature.ts → dist/src/derive/contentSignature.js} +23 -77
  19. package/dist/src/derive/contentUrl.d.ts +33 -0
  20. package/dist/src/derive/contentUrl.js +27 -0
  21. package/{src/derive/deriveMuted.ts → dist/src/derive/deriveMuted.d.ts} +1 -3
  22. package/dist/src/derive/deriveMuted.js +7 -0
  23. package/dist/src/derive/enrichThreadRows.d.ts +118 -0
  24. package/dist/src/derive/enrichThreadRows.js +151 -0
  25. package/dist/src/derive/filterThreadCriteria.d.ts +23 -0
  26. package/dist/src/derive/filterThreadCriteria.js +24 -0
  27. package/{src/derive/pendingMoveCounts.ts → dist/src/derive/pendingMoveCounts.d.ts} +5 -53
  28. package/dist/src/derive/pendingMoveCounts.js +53 -0
  29. package/{src/derive/senderTrust.ts → dist/src/derive/senderTrust.d.ts} +1 -9
  30. package/dist/src/derive/senderTrust.js +19 -0
  31. package/dist/src/error.d.ts +21 -0
  32. package/dist/src/error.js +85 -0
  33. package/dist/src/handlers/account-guards.d.ts +36 -0
  34. package/dist/src/handlers/account-guards.js +107 -0
  35. package/dist/src/handlers/account-oauth.d.ts +15 -0
  36. package/dist/src/handlers/account-oauth.js +334 -0
  37. package/dist/src/handlers/account-overrides.d.ts +53 -0
  38. package/dist/src/handlers/account-overrides.js +233 -0
  39. package/dist/src/handlers/account-signature.d.ts +28 -0
  40. package/dist/src/handlers/account-signature.js +85 -0
  41. package/dist/src/handlers/account.d.ts +7 -0
  42. package/dist/src/handlers/account.js +337 -0
  43. package/dist/src/handlers/address.d.ts +35 -0
  44. package/dist/src/handlers/address.js +133 -0
  45. package/dist/src/handlers/calendar-event.d.ts +70 -0
  46. package/dist/src/handlers/calendar-event.js +417 -0
  47. package/dist/src/handlers/calendar-feed.d.ts +47 -0
  48. package/dist/src/handlers/calendar-feed.js +184 -0
  49. package/dist/src/handlers/calendar-suggestion.d.ts +47 -0
  50. package/dist/src/handlers/calendar-suggestion.js +204 -0
  51. package/dist/src/handlers/calendar.d.ts +116 -0
  52. package/dist/src/handlers/calendar.js +323 -0
  53. package/dist/src/handlers/config.d.ts +43 -0
  54. package/dist/src/handlers/config.js +164 -0
  55. package/dist/src/handlers/ensure-account-config.d.ts +9 -0
  56. package/dist/src/handlers/ensure-account-config.js +27 -0
  57. package/dist/src/handlers/filter.d.ts +3 -0
  58. package/dist/src/handlers/filter.js +88 -0
  59. package/dist/src/handlers/folder-role-appointments.d.ts +68 -0
  60. package/dist/src/handlers/folder-role-appointments.js +168 -0
  61. package/dist/src/handlers/folder-role-labels.d.ts +26 -0
  62. package/dist/src/handlers/folder-role-labels.js +40 -0
  63. package/dist/src/handlers/folder-role.d.ts +9 -0
  64. package/dist/src/handlers/folder-role.js +37 -0
  65. package/dist/src/handlers/index.d.ts +88 -0
  66. package/dist/src/handlers/index.js +62 -0
  67. package/dist/src/handlers/label.d.ts +43 -0
  68. package/dist/src/handlers/label.js +107 -0
  69. package/dist/src/handlers/mailbox-bindings.d.ts +41 -0
  70. package/dist/src/handlers/mailbox-bindings.js +61 -0
  71. package/dist/src/handlers/mailbox.d.ts +23 -0
  72. package/dist/src/handlers/mailbox.js +211 -0
  73. package/dist/src/handlers/me.d.ts +2 -0
  74. package/dist/src/handlers/me.js +117 -0
  75. package/dist/src/handlers/message.d.ts +82 -0
  76. package/dist/src/handlers/message.js +595 -0
  77. package/dist/src/handlers/organize.d.ts +10 -0
  78. package/dist/src/handlers/organize.js +103 -0
  79. package/dist/src/handlers/outbox-attachment.d.ts +25 -0
  80. package/dist/src/handlers/outbox-attachment.js +77 -0
  81. package/dist/src/handlers/outbox.d.ts +4 -0
  82. package/dist/src/handlers/outbox.js +136 -0
  83. package/dist/src/handlers/sync-progress.d.ts +46 -0
  84. package/dist/src/handlers/sync-progress.js +63 -0
  85. package/dist/src/handlers/sync.d.ts +28 -0
  86. package/dist/src/handlers/sync.js +73 -0
  87. package/dist/src/handlers/system-update.d.ts +2 -0
  88. package/dist/src/handlers/system-update.js +159 -0
  89. package/dist/src/handlers/thread.d.ts +148 -0
  90. package/dist/src/handlers/thread.js +228 -0
  91. package/dist/src/handlers/unified-threads.d.ts +247 -0
  92. package/dist/src/handlers/unified-threads.js +406 -0
  93. package/{src/handlers/vip-suggestions.ts → dist/src/handlers/vip-suggestions.d.ts} +1 -13
  94. package/dist/src/handlers/vip-suggestions.js +16 -0
  95. package/dist/src/index.d.ts +5 -0
  96. package/dist/src/index.js +53 -0
  97. package/{src/json.ts → dist/src/json.d.ts} +1 -4
  98. package/dist/src/json.js +10 -0
  99. package/dist/src/jwt-auth.d.ts +15 -0
  100. package/dist/src/jwt-auth.js +45 -0
  101. package/dist/src/lambda.d.ts +13 -0
  102. package/dist/src/lambda.js +33 -0
  103. package/dist/src/request.d.ts +9 -0
  104. package/dist/src/request.js +17 -0
  105. package/dist/src/response.d.ts +44 -0
  106. package/dist/src/response.js +80 -0
  107. package/dist/src/serve.d.ts +31 -0
  108. package/dist/src/serve.js +117 -0
  109. package/dist/src/service/account-deletion.d.ts +20 -0
  110. package/dist/src/service/account-deletion.js +33 -0
  111. package/dist/src/service/compose-sqlite.d.ts +3 -0
  112. package/dist/src/service/compose-sqlite.js +13 -0
  113. package/dist/src/service/create-remit-client.d.ts +109 -0
  114. package/dist/src/service/create-remit-client.js +188 -0
  115. package/dist/src/service/data-client.d.ts +12 -0
  116. package/dist/src/service/data-client.js +42 -0
  117. package/dist/src/service/export-identity.d.ts +4 -0
  118. package/dist/src/service/export-identity.js +33 -0
  119. package/dist/src/service/fire-and-forget.d.ts +54 -0
  120. package/dist/src/service/fire-and-forget.js +37 -0
  121. package/dist/src/service/organize.d.ts +181 -0
  122. package/dist/src/service/organize.js +367 -0
  123. package/dist/src/service/queues.d.ts +12 -0
  124. package/dist/src/service/queues.js +1 -0
  125. package/dist/src/service/refusal.d.ts +4 -0
  126. package/dist/src/service/refusal.js +20 -0
  127. package/dist/src/service/sender-category.d.ts +51 -0
  128. package/{src/service/sender-category.ts → dist/src/service/sender-category.js} +40 -81
  129. package/dist/src/service/shared-deps.d.ts +6 -0
  130. package/dist/src/service/shared-deps.js +24 -0
  131. package/dist/src/service/trigger-sync.d.ts +57 -0
  132. package/dist/src/service/trigger-sync.js +31 -0
  133. package/dist/src/types.d.ts +72 -0
  134. package/dist/src/types.js +1 -0
  135. package/package.json +125 -98
  136. package/.env.test +0 -4
  137. package/dev-server/content-auth.test.ts +0 -113
  138. package/dev-server/content-auth.ts +0 -54
  139. package/dev-server/content-handler.test.ts +0 -105
  140. package/dev-server/content-handler.ts +0 -79
  141. package/dev-server/content-path.test.ts +0 -37
  142. package/dev-server/content-path.ts +0 -27
  143. package/dev-server/cors.test.ts +0 -48
  144. package/dev-server/cors.ts +0 -25
  145. package/dev-server/keep-alive.test.ts +0 -24
  146. package/dev-server/lambda-helpers.ts +0 -69
  147. package/dev-server/log-output.test.ts +0 -173
  148. package/dev-server/oauth-callback-route.test.ts +0 -66
  149. package/dev-server/relational-health.test.ts +0 -83
  150. package/dev-server/relational-health.ts +0 -39
  151. package/dev-server/server.ts +0 -384
  152. package/dev-server/sync-age.test.ts +0 -91
  153. package/dev-server/sync-age.ts +0 -55
  154. package/dev-server/upload-handler.ts +0 -132
  155. package/dev-server/upload-route.test.ts +0 -196
  156. package/scripts/backfill-classification.ts +0 -91
  157. package/scripts/backfill-list-id.ts +0 -83
  158. package/scripts/config-save.ts +0 -128
  159. package/src/auth.test.ts +0 -12
  160. package/src/auth.ts +0 -92
  161. package/src/config/msoauth.test.ts +0 -48
  162. package/src/config/msoauth.ts +0 -48
  163. package/src/data-backend.test.ts +0 -23
  164. package/src/data-backend.ts +0 -14
  165. package/src/derive/autoMoved.test.ts +0 -104
  166. package/src/derive/contentSignature.test.ts +0 -161
  167. package/src/derive/contentUrl.test.ts +0 -172
  168. package/src/derive/contentUrl.ts +0 -86
  169. package/src/derive/deriveMuted.test.ts +0 -34
  170. package/src/derive/enrichThreadRows.test.ts +0 -360
  171. package/src/derive/enrichThreadRows.ts +0 -234
  172. package/src/derive/filterThreadCriteria.test.ts +0 -126
  173. package/src/derive/filterThreadCriteria.ts +0 -51
  174. package/src/derive/senderTrust.test.ts +0 -67
  175. package/src/error.test.ts +0 -201
  176. package/src/error.ts +0 -123
  177. package/src/handlers/account-guards.test.ts +0 -67
  178. package/src/handlers/account-guards.ts +0 -169
  179. package/src/handlers/account-oauth-callback.test.ts +0 -155
  180. package/src/handlers/account-oauth-consent.test.ts +0 -336
  181. package/src/handlers/account-oauth-start.test.ts +0 -191
  182. package/src/handlers/account-oauth.test.ts +0 -439
  183. package/src/handlers/account-oauth.ts +0 -635
  184. package/src/handlers/account-overrides.test.ts +0 -88
  185. package/src/handlers/account-overrides.ts +0 -323
  186. package/src/handlers/account-ownership.ts +0 -25
  187. package/src/handlers/account-signature.ts +0 -117
  188. package/src/handlers/account-synced-services.test.ts +0 -245
  189. package/src/handlers/account.ts +0 -552
  190. package/src/handlers/address.test.ts +0 -381
  191. package/src/handlers/address.ts +0 -200
  192. package/src/handlers/calendar-event.test.ts +0 -685
  193. package/src/handlers/calendar-event.ts +0 -738
  194. package/src/handlers/calendar-feed-validators.test.ts +0 -228
  195. package/src/handlers/calendar-feed.test.ts +0 -439
  196. package/src/handlers/calendar-feed.ts +0 -291
  197. package/src/handlers/calendar-sqlite-fixture.ts +0 -36
  198. package/src/handlers/calendar-suggestion.test.ts +0 -1269
  199. package/src/handlers/calendar-suggestion.ts +0 -425
  200. package/src/handlers/calendar.test.ts +0 -1080
  201. package/src/handlers/calendar.ts +0 -517
  202. package/src/handlers/config.test.ts +0 -245
  203. package/src/handlers/config.ts +0 -332
  204. package/src/handlers/ensure-account-config.ts +0 -30
  205. package/src/handlers/filter.test.ts +0 -644
  206. package/src/handlers/filter.ts +0 -518
  207. package/src/handlers/folder-role-appointments.test.ts +0 -438
  208. package/src/handlers/folder-role-appointments.ts +0 -249
  209. package/src/handlers/folder-role-labels.ts +0 -63
  210. package/src/handlers/folder-role.test.ts +0 -209
  211. package/src/handlers/folder-role.ts +0 -80
  212. package/src/handlers/index.ts +0 -83
  213. package/src/handlers/label.test.ts +0 -112
  214. package/src/handlers/label.ts +0 -210
  215. package/src/handlers/mailbox-bindings.ts +0 -111
  216. package/src/handlers/mailbox-rename-refusal.test.ts +0 -247
  217. package/src/handlers/mailbox-settled-target.test.ts +0 -251
  218. package/src/handlers/mailbox.test.ts +0 -280
  219. package/src/handlers/mailbox.ts +0 -581
  220. package/src/handlers/me.test.ts +0 -81
  221. package/src/handlers/me.ts +0 -203
  222. package/src/handlers/message-read-backfill.test.ts +0 -103
  223. package/src/handlers/message.test.ts +0 -131
  224. package/src/handlers/message.ts +0 -1131
  225. package/src/handlers/organize.test.ts +0 -277
  226. package/src/handlers/organize.ts +0 -197
  227. package/src/handlers/outbox-attachment.test.ts +0 -765
  228. package/src/handlers/outbox-attachment.ts +0 -126
  229. package/src/handlers/outbox-list.test.ts +0 -179
  230. package/src/handlers/outbox.test.ts +0 -772
  231. package/src/handlers/outbox.ts +0 -258
  232. package/src/handlers/search.test.ts +0 -161
  233. package/src/handlers/search.ts +0 -105
  234. package/src/handlers/sync-progress.test.ts +0 -205
  235. package/src/handlers/sync-progress.ts +0 -99
  236. package/src/handlers/sync.test.ts +0 -168
  237. package/src/handlers/sync.ts +0 -118
  238. package/src/handlers/system-update.test.ts +0 -497
  239. package/src/handlers/system-update.ts +0 -230
  240. package/src/handlers/thread.test.ts +0 -396
  241. package/src/handlers/thread.ts +0 -427
  242. package/src/handlers/unified-threads-mail-sources.sqlite.test.ts +0 -197
  243. package/src/handlers/unified-threads.test.ts +0 -785
  244. package/src/handlers/unified-threads.ts +0 -669
  245. package/src/index.test.ts +0 -83
  246. package/src/index.ts +0 -228
  247. package/src/jwt-auth.test.ts +0 -152
  248. package/src/jwt-auth.ts +0 -132
  249. package/src/log-redaction.test.ts +0 -144
  250. package/src/openapi-spec.test.ts +0 -118
  251. package/src/request-context.test.ts +0 -61
  252. package/src/request-context.ts +0 -33
  253. package/src/request-pipeline.test.ts +0 -88
  254. package/src/request.ts +0 -10
  255. package/src/response.test.ts +0 -130
  256. package/src/response.ts +0 -144
  257. package/src/service/compose-sqlite.ts +0 -98
  258. package/src/service/config-import.ts +0 -39
  259. package/src/service/create-remit-client.ts +0 -522
  260. package/src/service/data-client.test.ts +0 -97
  261. package/src/service/data-client.ts +0 -58
  262. package/src/service/export-identity.ts +0 -39
  263. package/src/service/filter.ts +0 -40
  264. package/src/service/fire-and-forget.test.ts +0 -126
  265. package/src/service/fire-and-forget.ts +0 -79
  266. package/src/service/localhost-env-config.test.ts +0 -36
  267. package/src/service/organize-queue.ts +0 -98
  268. package/src/service/organize.test.ts +0 -1451
  269. package/src/service/organize.ts +0 -1025
  270. package/src/service/read-path-filter-config.test.ts +0 -243
  271. package/src/service/semantic-capability.test.ts +0 -78
  272. package/src/service/semantic-capability.ts +0 -95
  273. package/src/service/sender-category.test.ts +0 -235
  274. package/src/service/sqs.ts +0 -4
  275. package/src/service/trigger-sync.test.ts +0 -205
  276. package/src/service/trigger-sync.ts +0 -100
  277. package/src/stdout-capture-fixture.ts +0 -40
  278. package/src/types.ts +0 -271
  279. package/tsconfig.json +0 -7
@@ -1,243 +0,0 @@
1
- /**
2
- * The read-path body backfill must run a semantic (anchor-only) standing filter,
3
- * which needs the env-selected embedder wired into the filter config
4
- * `createRemitClient` hands its BodySyncService (issue #298).
5
- *
6
- * These drive the composed client — `createRemitClient(...).bodySync` — rather
7
- * than `buildReadPathFilterConfig` alone, so replacing that call with an inline
8
- * filter-config literal (which carries no embedder) fails here.
9
- */
10
-
11
- import assert from "node:assert/strict";
12
- import { afterEach, beforeEach, describe, it } from "node:test";
13
- import type {
14
- CreateMessageLabelInput,
15
- FilterAnchorItem,
16
- FilterItem,
17
- UpdateMessageInput,
18
- } from "@remit/data-ports";
19
- import { NotFoundError } from "@remit/data-ports/errors";
20
- import { FilterMatchOperator, FilterState } from "@remit/domain-enums";
21
- import {
22
- buildMatchText,
23
- type FilterMessage,
24
- type IImapConnection,
25
- NO_ACTION,
26
- } from "@remit/mailbox-service";
27
- import type { SearchService } from "@remit/search-service";
28
- import { buildEmbeddingServiceFromEnv } from "@remit/search-service/from-env";
29
- import type { SecretsService } from "@remit/secrets-service";
30
- import type { StorageService } from "@remit/storage-service";
31
- import {
32
- buildReadPathFilterConfig,
33
- createRemitClient,
34
- type RemitClientRepositories,
35
- } from "./create-remit-client.js";
36
-
37
- const MESSAGE: FilterMessage = {
38
- from: "billing@stripe.com",
39
- fromName: "Stripe",
40
- subject: "Your receipt",
41
- text: "Thanks for your payment",
42
- listId: "",
43
- };
44
-
45
- const eml = (message: FilterMessage): Buffer =>
46
- Buffer.from(
47
- [
48
- `From: ${message.fromName} <${message.from}>`,
49
- "To: me@example.com",
50
- `Subject: ${message.subject}`,
51
- "Content-Type: text/plain",
52
- "",
53
- message.text,
54
- ].join("\r\n"),
55
- );
56
-
57
- /**
58
- * Anchor-only: no literal clauses, so it matches on the embedding alone. Its
59
- * action is a label, which the fake MessageLabel repository below records —
60
- * a move would run the real PlacementMoveService against SQS.
61
- */
62
- const anchorOnlyFilter: FilterItem = {
63
- filterId: "flt-semantic",
64
- accountConfigId: "cfg-1",
65
- name: "receipts",
66
- scope: "Standing",
67
- state: FilterState.Active,
68
- hasAnchor: true,
69
- ruleChangedAt: 1,
70
- matchOperator: FilterMatchOperator.Or,
71
- literalClauses: [],
72
- actionLabelId: "lbl-receipts",
73
- actionMailboxId: NO_ACTION,
74
- createdAt: 1,
75
- updatedAt: 1,
76
- } as unknown as FilterItem;
77
-
78
- // The read path must embed a candidate message under the same model that wrote
79
- // the anchors, so an anchor written from this text matches it exactly.
80
- const anchorFor = async (message: FilterMessage): Promise<FilterAnchorItem> => {
81
- const embedding = buildEmbeddingServiceFromEnv();
82
- const [anchorEmbedding] = await embedding.embed([buildMatchText(message)]);
83
- return {
84
- accountConfigId: "cfg-1",
85
- filterId: "flt-semantic",
86
- anchorEmbedding,
87
- anchorEmbeddingId: embedding.embeddingId,
88
- anchorSourceText: buildMatchText(message),
89
- anchorMessageId: "m-anchor",
90
- } as unknown as FilterAnchorItem;
91
- };
92
-
93
- const repositoriesWithAnchor = (
94
- anchor: FilterAnchorItem,
95
- applied: CreateMessageLabelInput[] = [],
96
- ): RemitClientRepositories =>
97
- ({
98
- filter: {
99
- listByAccountAndState: async () => [anchorOnlyFilter],
100
- refreshExpiry: async (filter: FilterItem) => filter,
101
- },
102
- filterAnchor: { get: async () => anchor },
103
- messageLabel: {
104
- apply: async (input: CreateMessageLabelInput) => {
105
- applied.push(input);
106
- },
107
- },
108
- message: {
109
- get: async (messageId: string) => ({
110
- messageId,
111
- mailboxId: "mb-inbox",
112
- uid: 1,
113
- }),
114
- update: async (_messageId: string, _input: UpdateMessageInput) => {},
115
- },
116
- threadMessage: {
117
- findAllByMessageId: async () => [
118
- {
119
- threadMessageId: "tm-1",
120
- sentDate: 1,
121
- mailboxId: "mb-inbox",
122
- isRead: false,
123
- isDeleted: false,
124
- hasStars: false,
125
- hasAttachment: false,
126
- },
127
- ],
128
- update: async () => {},
129
- },
130
- address: {
131
- getAddress: async () => {
132
- throw new NotFoundError("Address not found");
133
- },
134
- incrementInboundCount: async () => {},
135
- },
136
- envelope: { listBodyParts: async () => [] },
137
- placementMove: {},
138
- mailboxSpecialUse: {},
139
- }) as unknown as RemitClientRepositories;
140
-
141
- const storage = {
142
- storeMessageBody: async () => ({ uri: "s3://bodies/m-1" }),
143
- storeParsedBody: async () => {},
144
- listBodyParts: async () => [],
145
- } as unknown as StorageService;
146
-
147
- const readBodyThroughClient = async (
148
- repositories: RemitClientRepositories,
149
- message: FilterMessage,
150
- ): Promise<void> => {
151
- const client = createRemitClient({
152
- repositories,
153
- storage,
154
- search: {} as unknown as SearchService,
155
- secrets: {} as unknown as SecretsService,
156
- sqsQueueUrl: "https://sqs.eu-west-1.amazonaws.com/0/test",
157
- sqsSmtpQueueUrl: "https://sqs.eu-west-1.amazonaws.com/0/test-smtp",
158
- });
159
- const connection = {
160
- openBox: async () => {},
161
- fetchMessageBody: async () => eml(message),
162
- } as unknown as IImapConnection;
163
-
164
- await client.bodySync.fetchAndGetBody(
165
- "m-1",
166
- "acc-1",
167
- "cfg-1",
168
- { fullPath: "INBOX", placement: "inbox" },
169
- async () => connection,
170
- );
171
- };
172
-
173
- describe("createRemitClient read-path filter config", () => {
174
- let previousQueueUrl: string | undefined;
175
-
176
- beforeEach(() => {
177
- previousQueueUrl = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
178
- process.env.SQS_QUEUE_URL_MESSAGE_MGMT =
179
- "https://sqs.eu-west-1.amazonaws.com/0/test-message-mgmt";
180
- });
181
-
182
- afterEach(() => {
183
- if (previousQueueUrl === undefined) {
184
- delete process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
185
- return;
186
- }
187
- process.env.SQS_QUEUE_URL_MESSAGE_MGMT = previousQueueUrl;
188
- });
189
-
190
- it("labels a message an anchor-only filter matches on a read-path backfill", async () => {
191
- const applied: CreateMessageLabelInput[] = [];
192
-
193
- await readBodyThroughClient(
194
- repositoriesWithAnchor(await anchorFor(MESSAGE), applied),
195
- MESSAGE,
196
- );
197
-
198
- assert.deepEqual(
199
- applied.map(({ labelId, appliedByFilterId }) => ({
200
- labelId,
201
- appliedByFilterId,
202
- })),
203
- [{ labelId: "lbl-receipts", appliedByFilterId: "flt-semantic" }],
204
- );
205
- });
206
-
207
- it("leaves a message the anchor does not match unlabelled", async () => {
208
- const applied: CreateMessageLabelInput[] = [];
209
-
210
- await readBodyThroughClient(
211
- repositoriesWithAnchor(
212
- await anchorFor({
213
- ...MESSAGE,
214
- subject: "Scheduled maintenance window",
215
- text: "Our service will be unavailable on Sunday",
216
- }),
217
- applied,
218
- ),
219
- MESSAGE,
220
- );
221
-
222
- assert.deepEqual(applied, []);
223
- });
224
- });
225
-
226
- describe("buildReadPathFilterConfig", () => {
227
- it("keeps filters off when the message-management queue is unset", async () => {
228
- const previous = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
229
- delete process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
230
- try {
231
- assert.equal(
232
- buildReadPathFilterConfig(
233
- repositoriesWithAnchor(await anchorFor(MESSAGE)),
234
- ),
235
- undefined,
236
- );
237
- } finally {
238
- if (previous !== undefined) {
239
- process.env.SQS_QUEUE_URL_MESSAGE_MGMT = previous;
240
- }
241
- }
242
- });
243
- });
@@ -1,78 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { afterEach, beforeEach, describe, it } from "node:test";
3
- import {
4
- _resetSemanticCapabilityForTest,
5
- isSemanticSearchUnavailable,
6
- noteSemanticCapabilityAbsence,
7
- } from "./semantic-capability.js";
8
-
9
- const moduleNotFound = (): Error => {
10
- const error = new Error(
11
- "Cannot find package '@huggingface/transformers' imported from /app/server.mjs",
12
- );
13
- (error as Error & { code: string }).code = "ERR_MODULE_NOT_FOUND";
14
- return error;
15
- };
16
-
17
- const modelUnavailable = (): Error => {
18
- const error = new Error(
19
- 'Embedding model "Xenova/paraphrase-multilingual-MiniLM-L12-v2" could not be loaded',
20
- );
21
- (error as Error & { code: string }).code = "ERR_EMBEDDING_MODEL_UNAVAILABLE";
22
- return error;
23
- };
24
-
25
- describe("noteSemanticCapabilityAbsence", () => {
26
- const ORIGINAL = process.env.DATA_BACKEND;
27
- beforeEach(() => {
28
- _resetSemanticCapabilityForTest();
29
- });
30
- afterEach(() => {
31
- if (ORIGINAL === undefined) delete process.env.DATA_BACKEND;
32
- else process.env.DATA_BACKEND = ORIGINAL;
33
- _resetSemanticCapabilityForTest();
34
- });
35
-
36
- it("absorbs a missing-module failure on the self-host SQL backend and remembers it", () => {
37
- process.env.DATA_BACKEND = "sqlite";
38
- assert.equal(isSemanticSearchUnavailable(), false);
39
- assert.equal(noteSemanticCapabilityAbsence(moduleNotFound()), true);
40
- assert.equal(isSemanticSearchUnavailable(), true);
41
- });
42
-
43
- it("absorbs a dlopen failure (musl loading a glibc extension)", () => {
44
- process.env.DATA_BACKEND = "sqlite";
45
- const error = new Error(
46
- "Error loading shared library ld-linux-x86-64.so.2: No such file or directory",
47
- );
48
- (error as Error & { code: string }).code = "ERR_DLOPEN_FAILED";
49
- assert.equal(noteSemanticCapabilityAbsence(error), true);
50
- });
51
-
52
- it("absorbs an embedding-model load failure (e2e-dev from source, HuggingFace fetch failed) and remembers it", () => {
53
- process.env.DATA_BACKEND = "sqlite";
54
- assert.equal(isSemanticSearchUnavailable(), false);
55
- assert.equal(noteSemanticCapabilityAbsence(modelUnavailable()), true);
56
- assert.equal(isSemanticSearchUnavailable(), true);
57
- });
58
-
59
- it("rethrows genuine query errors on the self-host SQL backend", () => {
60
- process.env.DATA_BACKEND = "sqlite";
61
- assert.equal(
62
- noteSemanticCapabilityAbsence(new Error("SQLITE_BUSY")),
63
- false,
64
- );
65
- assert.equal(isSemanticSearchUnavailable(), false);
66
- assert.equal(noteSemanticCapabilityAbsence(undefined), false);
67
- });
68
-
69
- it("never engages on the AWS DynamoDB path — a missing module there is a broken deploy", () => {
70
- delete process.env.DATA_BACKEND;
71
- assert.equal(noteSemanticCapabilityAbsence(moduleNotFound()), false);
72
- assert.equal(noteSemanticCapabilityAbsence(modelUnavailable()), false);
73
- process.env.DATA_BACKEND = "dynamodb";
74
- assert.equal(noteSemanticCapabilityAbsence(moduleNotFound()), false);
75
- assert.equal(noteSemanticCapabilityAbsence(modelUnavailable()), false);
76
- assert.equal(isSemanticSearchUnavailable(), false);
77
- });
78
- });
@@ -1,95 +0,0 @@
1
- import { logger } from "@remit/logger-lambda";
2
- import {
3
- _resetEmbeddingCapabilityForTest,
4
- isEmbeddingCapabilityAbsence,
5
- isEmbeddingCapabilityUnavailable,
6
- recordEmbeddingCapabilityAbsence,
7
- } from "@remit/search-service/capability";
8
- import { isSelfHostSqlBackend } from "../data-backend.js";
9
-
10
- /**
11
- * Semantic-search capability gate for the self-host compose profiles.
12
- *
13
- * This gates the free-text `/search/semantic` path only. Answering that query
14
- * means embedding it in-process, and the backend container image ships without
15
- * `@huggingface/transformers` (the model plus its dependencies would roughly
16
- * quadruple the image — see npm-scripts/docker-bundle.mjs and
17
- * deploy/vps/README.md). The embedder is reached lazily through `runtimeImport`,
18
- * so the first semantic query — not startup — hits ERR_MODULE_NOT_FOUND. Without
19
- * this gate every search the web client issues fires a `/search/semantic`
20
- * request that 500s (the client fetches semantic hits alongside every literal
21
- * search), retried by the client on top.
22
- *
23
- * The vector STORE is a separate capability and is NOT gated here. The Organize
24
- * anchor widen (organize.ts matchSemantic) pools an anchor message's already
25
- * stored chunk vectors and runs a KNN read over the vector store — no embedding
26
- * model involved — so it needs only `sqlite-vec`, which the backend image now
27
- * carries as a musl build (Dockerfile sqlite-vec-musl stage,
28
- * SQLITE_VEC_EXTENSION_PATH). matchOrganize never consults the memoized flag
29
- * below; a missing embedder therefore never disables the widen. The one place
30
- * the widen would reach for an embedder is repairing a persisted FilterAnchor
31
- * whose model has drifted (organize.ts matchSemantic): with no embedder there
32
- * is no repair to be had, so it queries with the stored vector and classifies
33
- * the failure through {@link isSemanticCapabilityAbsence}, which deliberately
34
- * does not record the absence. Recording it there would take every later
35
- * free-text semantic query down on the strength of one filter's stale stamp.
36
- *
37
- * The e2e-dev lane runs this backend from source rather than the container, so
38
- * `@huggingface/transformers` IS present and the local embedder instead tries to
39
- * lazily download the model from HuggingFace; a transient `fetch failed` there
40
- * surfaces the same way. The search-service raises it as a typed
41
- * `EmbeddingModelUnavailableError` (code ERR_EMBEDDING_MODEL_UNAVAILABLE), which
42
- * this gate treats as the same capability absence — the model is not available,
43
- * empty semantic results are the truthful answer, and the lane never depends on
44
- * a live external download.
45
- *
46
- * When the pipeline is genuinely absent, empty hits are the truthful answer:
47
- * the FTS/literal engine is the primary search surface on these profiles and is
48
- * unaffected. The absence is remembered so subsequent requests short-circuit.
49
- *
50
- * Scoped to the self-host SQL backend: on AWS the pipeline (Bedrock +
51
- * S3 Vectors) is bundled, so a module-resolution failure there is a broken
52
- * deploy and must keep failing loud. Falling back to the FTS engine here
53
- * instead was considered and rejected — it would fabricate relevance scores and
54
- * matched-chunk labels for the "Related" UI, and cannot serve the cross-account
55
- * queries the daily brief issues.
56
- *
57
- * The error vocabulary and the memoized absence itself live in
58
- * `@remit/search-service/capability`, shared with the filter pipeline's semantic
59
- * skip: whichever path embeds first is the probe, and the other stops trying.
60
- * This module adds the self-host scoping and the operator-facing log.
61
- */
62
-
63
- /** Test-only reset for the memoized absence. */
64
- export const _resetSemanticCapabilityForTest = (): void => {
65
- _resetEmbeddingCapabilityForTest();
66
- };
67
-
68
- export const isSemanticSearchUnavailable = (): boolean =>
69
- isEmbeddingCapabilityUnavailable();
70
-
71
- /**
72
- * Whether a failure is this deployment simply not carrying the semantic
73
- * pipeline — the missing-module/extension shape on a self-host SQL backend.
74
- * Records nothing, for the caller that must degrade one operation without
75
- * disabling `/search/semantic` process-wide.
76
- */
77
- export const isSemanticCapabilityAbsence = (error: unknown): boolean =>
78
- isSelfHostSqlBackend() && isEmbeddingCapabilityAbsence(error);
79
-
80
- /**
81
- * Classify a semantic-search failure. Returns true — and remembers the
82
- * absence — when it is the missing-module/extension shape on a self-host SQL
83
- * backend; any other error is the caller's to rethrow.
84
- */
85
- export const noteSemanticCapabilityAbsence = (error: unknown): boolean => {
86
- if (!isSemanticCapabilityAbsence(error)) return false;
87
- if (!isEmbeddingCapabilityUnavailable()) {
88
- logger.warn(
89
- { error: error instanceof Error ? error.message : String(error) },
90
- "Semantic search pipeline unavailable in this deployment — serving empty semantic results (see deploy/vps/README.md)",
91
- );
92
- }
93
- recordEmbeddingCapabilityAbsence(error);
94
- return true;
95
- };
@@ -1,235 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import { describe, it } from "node:test";
3
- import type {
4
- ThreadMessageFieldTerm,
5
- ThreadMessageItem,
6
- UpdateMessageInput,
7
- UpdateThreadMessageInput,
8
- } from "@remit/data-ports";
9
- import {
10
- backApplySenderCategory,
11
- SENDER_CATEGORY_BACKAPPLY_LIMIT,
12
- type SenderCategoryBackApplyDeps,
13
- } from "./sender-category.js";
14
-
15
- const ACCOUNT_CONFIG_ID = "cfg-415";
16
- const SENDER = "post@ns.nl";
17
-
18
- const row = (n: number, over: Partial<ThreadMessageItem> = {}) =>
19
- ({
20
- threadMessageId: `tm-${n}`,
21
- threadId: `th-${n}`,
22
- messageId: `msg-${n}`,
23
- accountConfigId: ACCOUNT_CONFIG_ID,
24
- mailboxId: "mbx-inbox",
25
- fromEmail: SENDER,
26
- fromName: "NS Reizigers",
27
- subject: `reisoverzicht ${n}`,
28
- category: "personal",
29
- sentDate: 2_000 - n,
30
- isRead: false,
31
- isDeleted: false,
32
- hasStars: false,
33
- hasAttachment: false,
34
- ...over,
35
- }) as unknown as ThreadMessageItem;
36
-
37
- interface Writes {
38
- messages: Array<{ messageId: string; input: UpdateMessageInput }>;
39
- threadRows: Array<{
40
- threadMessageId: string;
41
- input: UpdateThreadMessageInput;
42
- }>;
43
- terms: ThreadMessageFieldTerm[][];
44
- pageSizes: Array<number | undefined>;
45
- }
46
-
47
- /**
48
- * A store that pages its rows the way the real listing does — newest first,
49
- * `limit` rows per page, a continuation token while more remain — and that
50
- * answers `findAllByMessageId` from the same rows, so a re-label is provably
51
- * written to both halves of the pair.
52
- */
53
- const storeOf = (
54
- rows: ThreadMessageItem[],
55
- writes: Writes,
56
- ): SenderCategoryBackApplyDeps => ({
57
- client: {
58
- message: {
59
- update: async (messageId: string, input: UpdateMessageInput) => {
60
- writes.messages.push({ messageId, input });
61
- return {} as never;
62
- },
63
- },
64
- threadMessage: {
65
- listByFieldTerms: async (
66
- _accountConfigId: string,
67
- terms: readonly ThreadMessageFieldTerm[],
68
- options?: { limit?: number; continuationToken?: string },
69
- ) => {
70
- writes.terms.push([...terms]);
71
- writes.pageSizes.push(options?.limit);
72
- const from = Number(options?.continuationToken ?? 0);
73
- const size = options?.limit ?? rows.length;
74
- const page = rows.slice(from, from + size);
75
- const next = from + page.length;
76
- return {
77
- items: page,
78
- continuationToken: next < rows.length ? String(next) : undefined,
79
- };
80
- },
81
- findAllByMessageId: async (_accountConfigId: string, messageId: string) =>
82
- rows.filter((r) => r.messageId === messageId),
83
- update: async (
84
- _accountConfigId: string,
85
- threadMessageId: string,
86
- input: UpdateThreadMessageInput,
87
- ) => {
88
- writes.threadRows.push({ threadMessageId, input });
89
- return {} as never;
90
- },
91
- },
92
- } as unknown as SenderCategoryBackApplyDeps["client"],
93
- });
94
-
95
- const emptyWrites = (): Writes => ({
96
- messages: [],
97
- threadRows: [],
98
- terms: [],
99
- pageSizes: [],
100
- });
101
-
102
- describe("backApplySenderCategory (#415)", () => {
103
- it("re-labels mail the classifier already decided, on both halves of the pair", async () => {
104
- const writes = emptyWrites();
105
- const rows = [row(1), row(2), row(3)];
106
-
107
- const result = await backApplySenderCategory(
108
- storeOf(rows, writes),
109
- ACCOUNT_CONFIG_ID,
110
- SENDER,
111
- "newsletter",
112
- );
113
-
114
- assert.deepEqual(result, { matched: 3, applied: 3, failed: 0 });
115
- assert.deepEqual(
116
- writes.messages.map((w) => w.messageId),
117
- ["msg-1", "msg-2", "msg-3"],
118
- );
119
- assert.deepEqual(
120
- writes.messages.map((w) => w.input.category),
121
- ["newsletter", "newsletter", "newsletter"],
122
- );
123
- assert.deepEqual(
124
- writes.threadRows.map((w) => w.threadMessageId),
125
- ["tm-1", "tm-2", "tm-3"],
126
- );
127
- assert.deepEqual(
128
- writes.threadRows.map((w) => w.input.category),
129
- ["newsletter", "newsletter", "newsletter"],
130
- );
131
- });
132
-
133
- it("stops at the bound, keeping the newest of the sender's mail", async () => {
134
- const writes = emptyWrites();
135
- const rows = Array.from(
136
- { length: SENDER_CATEGORY_BACKAPPLY_LIMIT + 25 },
137
- (_, i) => row(i + 1),
138
- );
139
-
140
- const result = await backApplySenderCategory(
141
- storeOf(rows, writes),
142
- ACCOUNT_CONFIG_ID,
143
- SENDER,
144
- "newsletter",
145
- );
146
-
147
- assert.equal(result.matched, SENDER_CATEGORY_BACKAPPLY_LIMIT);
148
- assert.equal(result.applied, SENDER_CATEGORY_BACKAPPLY_LIMIT);
149
- assert.equal(writes.messages.length, SENDER_CATEGORY_BACKAPPLY_LIMIT);
150
- assert.equal(writes.messages[0]?.messageId, "msg-1");
151
- assert.equal(
152
- writes.messages.at(-1)?.messageId,
153
- `msg-${SENDER_CATEGORY_BACKAPPLY_LIMIT}`,
154
- );
155
- });
156
-
157
- it("refines the store's substring narrowing to an exact sender match", async () => {
158
- const writes = emptyWrites();
159
- const rows = [
160
- row(1),
161
- row(2, { messageId: "msg-lookalike", fromEmail: "no-post@ns.nl" }),
162
- row(3, { messageId: "msg-name-only", fromEmail: "spoof@example.com" }),
163
- ];
164
-
165
- const result = await backApplySenderCategory(
166
- storeOf(rows, writes),
167
- ACCOUNT_CONFIG_ID,
168
- SENDER,
169
- "newsletter",
170
- );
171
-
172
- assert.equal(result.matched, 1);
173
- assert.deepEqual(
174
- writes.messages.map((w) => w.messageId),
175
- ["msg-1"],
176
- );
177
- assert.deepEqual(writes.terms[0], [{ field: "sender", contains: SENDER }]);
178
- });
179
-
180
- it("pages the narrowed listing until the bound fills", async () => {
181
- const writes = emptyWrites();
182
- const rows = Array.from({ length: 12 }, (_, i) =>
183
- i % 2 === 0
184
- ? row(i + 1)
185
- : row(i + 1, {
186
- messageId: `msg-other-${i}`,
187
- fromEmail: "someone@example.com",
188
- }),
189
- );
190
-
191
- const result = await backApplySenderCategory(
192
- storeOf(rows, writes),
193
- ACCOUNT_CONFIG_ID,
194
- SENDER,
195
- "newsletter",
196
- 4,
197
- );
198
-
199
- assert.equal(result.matched, 4);
200
- assert.ok(
201
- writes.terms.length > 1,
202
- "a page of narrowed rows that yields fewer matches than the bound must be followed by the next page",
203
- );
204
- assert.deepEqual(writes.pageSizes, Array(writes.terms.length).fill(4));
205
- });
206
-
207
- it("counts a poisoned message as failed and re-labels the rest", async () => {
208
- const writes = emptyWrites();
209
- const rows = [row(1), row(2), row(3)];
210
- const store = storeOf(rows, writes);
211
- const threadMessage = store.client.threadMessage as unknown as Record<
212
- string,
213
- unknown
214
- >;
215
- const findAll = threadMessage.findAllByMessageId as (
216
- a: string,
217
- b: string,
218
- ) => Promise<ThreadMessageItem[]>;
219
- threadMessage.findAllByMessageId = async (a: string, b: string) =>
220
- b === "msg-2" ? [] : findAll(a, b);
221
-
222
- const result = await backApplySenderCategory(
223
- store,
224
- ACCOUNT_CONFIG_ID,
225
- SENDER,
226
- "newsletter",
227
- );
228
-
229
- assert.deepEqual(result, { matched: 3, applied: 2, failed: 1 });
230
- assert.deepEqual(
231
- writes.messages.map((w) => w.messageId),
232
- ["msg-1", "msg-3"],
233
- );
234
- });
235
- });
@@ -1,4 +0,0 @@
1
- import { createQueueProducer } from "@remit/sqs-client/producer";
2
- import { env } from "expect-env";
3
-
4
- export const sqsClient = createQueueProducer({ queueUrl: env.SQS_QUEUE_URL });