@remit/backend 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/.env.test +7 -0
  2. package/dev-server/content-auth.test.ts +113 -0
  3. package/dev-server/content-auth.ts +54 -0
  4. package/dev-server/content-handler.test.ts +105 -0
  5. package/dev-server/content-handler.ts +79 -0
  6. package/dev-server/content-path.test.ts +37 -0
  7. package/dev-server/content-path.ts +27 -0
  8. package/dev-server/cors.test.ts +48 -0
  9. package/dev-server/cors.ts +25 -0
  10. package/dev-server/lambda-helpers.ts +69 -0
  11. package/dev-server/server.ts +289 -0
  12. package/package.json +82 -0
  13. package/src/auth.ts +92 -0
  14. package/src/config/msoauth.ts +60 -0
  15. package/src/data-backend.test.ts +25 -0
  16. package/src/data-backend.ts +20 -0
  17. package/src/derive/autoMoved.ts +28 -0
  18. package/src/derive/contentSignature.test.ts +153 -0
  19. package/src/derive/contentSignature.ts +139 -0
  20. package/src/derive/contentUrl.test.ts +156 -0
  21. package/src/derive/contentUrl.ts +70 -0
  22. package/src/derive/enrichThreadRows.ts +155 -0
  23. package/src/derive/filterThreadCriteria.test.ts +119 -0
  24. package/src/derive/filterThreadCriteria.ts +60 -0
  25. package/src/derive/pendingMoveCounts.ts +90 -0
  26. package/src/derive/senderTrust.test.ts +67 -0
  27. package/src/derive/senderTrust.ts +23 -0
  28. package/src/error.ts +55 -0
  29. package/src/handlers/account-guards.ts +137 -0
  30. package/src/handlers/account-oauth.test.ts +383 -0
  31. package/src/handlers/account-oauth.ts +452 -0
  32. package/src/handlers/account-overrides.test.ts +69 -0
  33. package/src/handlers/account-overrides.ts +286 -0
  34. package/src/handlers/account-ownership.ts +25 -0
  35. package/src/handlers/account-signature.ts +129 -0
  36. package/src/handlers/account.ts +524 -0
  37. package/src/handlers/address.ts +136 -0
  38. package/src/handlers/config.test.ts +184 -0
  39. package/src/handlers/config.ts +219 -0
  40. package/src/handlers/ensure-account-config.ts +30 -0
  41. package/src/handlers/filter.ts +294 -0
  42. package/src/handlers/folder-role-appointments.test.ts +250 -0
  43. package/src/handlers/folder-role-appointments.ts +272 -0
  44. package/src/handlers/folder-role.ts +76 -0
  45. package/src/handlers/index.ts +48 -0
  46. package/src/handlers/mailbox.ts +411 -0
  47. package/src/handlers/me.ts +190 -0
  48. package/src/handlers/message.ts +886 -0
  49. package/src/handlers/organize.test.ts +43 -0
  50. package/src/handlers/organize.ts +196 -0
  51. package/src/handlers/outbox.ts +218 -0
  52. package/src/handlers/search.test.ts +182 -0
  53. package/src/handlers/search.ts +105 -0
  54. package/src/handlers/sync-progress.test.ts +158 -0
  55. package/src/handlers/sync-progress.ts +64 -0
  56. package/src/handlers/sync.test.ts +146 -0
  57. package/src/handlers/sync.ts +119 -0
  58. package/src/handlers/thread.ts +361 -0
  59. package/src/handlers/unified-threads.ts +196 -0
  60. package/src/handlers/vip-suggestions.ts +21 -0
  61. package/src/index.ts +170 -0
  62. package/src/json.ts +11 -0
  63. package/src/jwt-auth.test.ts +89 -0
  64. package/src/jwt-auth.ts +95 -0
  65. package/src/request-context.test.ts +61 -0
  66. package/src/request-context.ts +29 -0
  67. package/src/request.ts +10 -0
  68. package/src/response.test.ts +107 -0
  69. package/src/response.ts +80 -0
  70. package/src/service/compose-postgres.ts +72 -0
  71. package/src/service/compose-sqlite.ts +80 -0
  72. package/src/service/create-remit-client.ts +358 -0
  73. package/src/service/dynamodb.test.ts +100 -0
  74. package/src/service/dynamodb.ts +58 -0
  75. package/src/service/filter.ts +55 -0
  76. package/src/service/fire-and-forget.test.ts +126 -0
  77. package/src/service/fire-and-forget.ts +79 -0
  78. package/src/service/localhost-env-config.test.ts +40 -0
  79. package/src/service/organize.test.ts +384 -0
  80. package/src/service/organize.ts +354 -0
  81. package/src/service/semantic-capability.test.ts +64 -0
  82. package/src/service/semantic-capability.ts +62 -0
  83. package/src/service/sqs.ts +4 -0
  84. package/src/service/trigger-sync.test.ts +211 -0
  85. package/src/service/trigger-sync.ts +102 -0
  86. package/src/types.ts +161 -0
  87. package/tsconfig.json +7 -0
@@ -0,0 +1,354 @@
1
+ import type { FilterItem, OrganizeJobRequestItem } from "@remit/data-ports";
2
+ import {
3
+ DEFAULT_SEMANTIC_MATCH_THRESHOLD,
4
+ type FilterMessage,
5
+ literalClausesMatch,
6
+ NO_ACTION,
7
+ PlacementMoveService,
8
+ } from "@remit/mailbox-service";
9
+ import {
10
+ type AnchorPayload,
11
+ buildMessageAnchor,
12
+ type VectorRecord,
13
+ type VectorStoreService,
14
+ } from "@remit/search-service";
15
+ import {
16
+ buildEmbeddingServiceFromEnv,
17
+ buildVectorStoreFromEnv,
18
+ } from "@remit/search-service/from-env";
19
+ import type { RemitClient } from "./dynamodb.js";
20
+
21
+ /**
22
+ * Hard cap on both the previewed and the applied set. A back-apply is a
23
+ * data-heavy corpus pass (remit-data-heavy: frugal): the semantic side is
24
+ * bounded by the vector query's `topK`, the literal-only side by a paginated
25
+ * message scan, and the final match set never exceeds this.
26
+ */
27
+ export const ORGANIZE_MATCH_LIMIT = 500;
28
+
29
+ /**
30
+ * How many chunk matches to pull per requested message. A message contributes
31
+ * several chunk vectors (sender / subject / body / …); over-fetching chunks
32
+ * keeps enough distinct messages after de-duplication to fill the cap.
33
+ */
34
+ const VECTOR_CHUNK_FACTOR = 8;
35
+
36
+ /**
37
+ * The predicate + action for one back-apply pass — the OrganizeInput fields,
38
+ * flattened. `anchorMessageId` uses the `"None"` sentinel (never optional) to
39
+ * mean "purely literal", matching how the job row and the action fields carry
40
+ * absence (RFC 034 Decision 3.1).
41
+ */
42
+ export interface OrganizePredicate {
43
+ anchorMessageId: string;
44
+ matchOperator: FilterItem["matchOperator"];
45
+ literalClauses: FilterItem["literalClauses"];
46
+ similarityThreshold: number;
47
+ actionLabelId: string;
48
+ actionMailboxId: string;
49
+ }
50
+
51
+ /**
52
+ * The predicate a back-apply job snapshotted onto its row, read back verbatim so
53
+ * the worker runs exactly what the request asked for.
54
+ */
55
+ export const predicateFromJob = (
56
+ job: OrganizeJobRequestItem,
57
+ ): OrganizePredicate => ({
58
+ anchorMessageId: job.anchorMessageId,
59
+ matchOperator: job.matchOperator,
60
+ literalClauses: job.literalClauses,
61
+ similarityThreshold: job.similarityThreshold,
62
+ actionLabelId: job.actionLabelId,
63
+ actionMailboxId: job.actionMailboxId,
64
+ });
65
+
66
+ export interface OrganizeMatchDeps {
67
+ buildAnchor: (
68
+ accountConfigId: string,
69
+ anchorMessageId: string,
70
+ ) => Promise<AnchorPayload | null>;
71
+ vectorStore: Pick<VectorStoreService, "query" | "getByMessage">;
72
+ listAccountMessageIds: (
73
+ accountConfigId: string,
74
+ limit: number,
75
+ ) => Promise<string[]>;
76
+ }
77
+
78
+ const hasAnchor = (predicate: OrganizePredicate): boolean =>
79
+ predicate.anchorMessageId !== NO_ACTION && predicate.anchorMessageId !== "";
80
+
81
+ /**
82
+ * Reconstruct the literal-match view of a message from its already-indexed chunk
83
+ * vectors — no body fetch, no re-embedding (remit-data-heavy: frugal). The
84
+ * sender chunk carries the from address, the subject/body chunks the text; each
85
+ * is the same 512-char preview the semantic side is derived from.
86
+ */
87
+ const filterMessageFromChunks = (
88
+ records: VectorRecord[],
89
+ ): FilterMessage | null => {
90
+ if (records.length === 0) return null;
91
+ let subject = "";
92
+ let fromName = "";
93
+ let from = "";
94
+ const textParts: string[] = [];
95
+ for (const record of records) {
96
+ const meta = record.metadata;
97
+ if (meta.subject && !subject) subject = meta.subject;
98
+ if (meta.fromName && !fromName) fromName = meta.fromName;
99
+ if (meta.chunkType === "sender" && meta.textPreview && !from) {
100
+ from = meta.textPreview;
101
+ }
102
+ if (
103
+ (meta.chunkType === "body" || meta.chunkType === "subject") &&
104
+ meta.textPreview
105
+ ) {
106
+ textParts.push(meta.textPreview);
107
+ }
108
+ }
109
+ return { from, fromName, subject, text: textParts.join("\n") };
110
+ };
111
+
112
+ /**
113
+ * The matcher shared by preview and apply — the previewed set equals the applied
114
+ * set for the same input. Read-only. Returns the matching message ids, bounded.
115
+ *
116
+ * - Semantic anchor: the anchor vector is pooled transiently from the anchor
117
+ * message's existing chunk vectors (never a FilterAnchor row), then a k-NN
118
+ * query fans out to candidate messages gated on the cosine threshold.
119
+ * - Literal clauses (RFC 031) refine the candidate set; a purely-literal
120
+ * back-apply scans a bounded slice of the corpus instead.
121
+ * - A predicate with neither an anchor nor a clause matches nothing, mirroring
122
+ * the index-time filter matcher.
123
+ */
124
+ export const matchOrganize = async (
125
+ deps: OrganizeMatchDeps,
126
+ accountConfigId: string,
127
+ predicate: OrganizePredicate,
128
+ limit: number = ORGANIZE_MATCH_LIMIT,
129
+ ): Promise<string[]> => {
130
+ const anchored = hasAnchor(predicate);
131
+ const clauses = predicate.literalClauses;
132
+ if (!anchored && clauses.length === 0) return [];
133
+
134
+ let base: string[];
135
+ if (anchored) {
136
+ const anchor = await deps.buildAnchor(
137
+ accountConfigId,
138
+ predicate.anchorMessageId,
139
+ );
140
+ if (!anchor) return [];
141
+ const threshold =
142
+ predicate.similarityThreshold ?? DEFAULT_SEMANTIC_MATCH_THRESHOLD;
143
+ const matches = await deps.vectorStore.query({
144
+ vector: anchor.anchorEmbedding,
145
+ topK: limit * VECTOR_CHUNK_FACTOR,
146
+ filter: { accountConfigId },
147
+ });
148
+ const bestScore = new Map<string, number>();
149
+ for (const match of matches) {
150
+ const messageId = match.metadata.messageId;
151
+ const prev = bestScore.get(messageId);
152
+ if (prev === undefined || match.score > prev) {
153
+ bestScore.set(messageId, match.score);
154
+ }
155
+ }
156
+ base = [...bestScore.entries()]
157
+ .filter(([, score]) => score >= threshold)
158
+ .map(([messageId]) => messageId);
159
+ } else {
160
+ base = await deps.listAccountMessageIds(accountConfigId, limit);
161
+ }
162
+
163
+ if (clauses.length === 0) return base.slice(0, limit);
164
+
165
+ const matched: string[] = [];
166
+ for (const messageId of base) {
167
+ if (matched.length >= limit) break;
168
+ const records = await deps.vectorStore.getByMessage(messageId);
169
+ const message = filterMessageFromChunks(records);
170
+ if (!message) continue;
171
+ if (literalClausesMatch(clauses, predicate.matchOperator, message)) {
172
+ matched.push(messageId);
173
+ }
174
+ }
175
+ return matched;
176
+ };
177
+
178
+ let cachedVectorStore: Pick<
179
+ VectorStoreService,
180
+ "query" | "getByMessage"
181
+ > | null = null;
182
+
183
+ const getVectorStore = (): Pick<
184
+ VectorStoreService,
185
+ "query" | "getByMessage"
186
+ > => {
187
+ if (!cachedVectorStore) {
188
+ const embedder = buildEmbeddingServiceFromEnv();
189
+ cachedVectorStore = buildVectorStoreFromEnv(embedder.dimensions);
190
+ }
191
+ return cachedVectorStore;
192
+ };
193
+
194
+ const buildAnchorFromEnv = (): OrganizeMatchDeps["buildAnchor"] => {
195
+ const embedder = buildEmbeddingServiceFromEnv();
196
+ const store = buildVectorStoreFromEnv(embedder.dimensions);
197
+ return (accountConfigId, anchorMessageId) =>
198
+ buildMessageAnchor(
199
+ { store, embedder },
200
+ { accountConfigId, anchorMessageId },
201
+ );
202
+ };
203
+
204
+ /**
205
+ * A bounded corpus slice for a purely-literal back-apply: the account's message
206
+ * ids, gathered mailbox by mailbox and capped. Paginated per mailbox so a large
207
+ * corpus never loads at once.
208
+ */
209
+ const listAccountMessageIdsFromClient =
210
+ (client: RemitClient): OrganizeMatchDeps["listAccountMessageIds"] =>
211
+ async (accountConfigId, limit) => {
212
+ const accounts =
213
+ await client.account.listAllByAccountConfig(accountConfigId);
214
+ const ids: string[] = [];
215
+ for (const account of accounts) {
216
+ const mailboxes = await client.mailbox.listAllByAccount(
217
+ account.accountId,
218
+ );
219
+ for (const mailbox of mailboxes) {
220
+ let continuationToken: string | undefined;
221
+ do {
222
+ const page = await client.message.listByMailbox(mailbox.mailboxId, {
223
+ limit,
224
+ continuationToken,
225
+ });
226
+ for (const message of page.items) {
227
+ ids.push(message.messageId);
228
+ if (ids.length >= limit) return ids;
229
+ }
230
+ continuationToken = page.continuationToken;
231
+ } while (continuationToken);
232
+ }
233
+ }
234
+ return ids;
235
+ };
236
+
237
+ /**
238
+ * The env-wired matcher deps for the running backend/worker. The vector store
239
+ * and embedder are selected by env, independent of `DATA_BACKEND`.
240
+ */
241
+ export const buildOrganizeMatchDeps = (
242
+ client: RemitClient,
243
+ ): OrganizeMatchDeps => ({
244
+ buildAnchor: buildAnchorFromEnv(),
245
+ vectorStore: getVectorStore(),
246
+ listAccountMessageIds: listAccountMessageIdsFromClient(client),
247
+ });
248
+
249
+ /**
250
+ * The exclusive-move arm of a back-apply, wired exactly like the body-sync
251
+ * placement mover (`sync-message-body.ts`): the local-first
252
+ * {@link PlacementMoveService} over the shared message-management queue. A move
253
+ * commits locally (marker + ThreadMessage + Message row) and enqueues a
254
+ * `PLACEMENT_MOVE_PUSH` for the reconciler; its marker state engine makes a
255
+ * redelivered job idempotent (mirrors #1297). Returns `undefined` when no
256
+ * message-management queue is wired — the same gate body sync uses to keep the
257
+ * move path off — so a move requested in that environment is counted as failed
258
+ * rather than silently dropped (see {@link applyOrganize}).
259
+ */
260
+ export const buildOrganizeMoveService = (
261
+ client: RemitClient,
262
+ ): PlacementMoveService | undefined => {
263
+ const queueUrl = process.env.SQS_QUEUE_URL_MESSAGE_MGMT;
264
+ if (!queueUrl) return undefined;
265
+ return new PlacementMoveService({
266
+ messageService: client.message,
267
+ threadMessageService: client.threadMessage,
268
+ markerService: client.placementMove,
269
+ sqsQueueUrl: queueUrl,
270
+ });
271
+ };
272
+
273
+ export interface ApplyOrganizeDeps {
274
+ client: RemitClient;
275
+ moveService?: PlacementMoveService;
276
+ }
277
+
278
+ export interface ApplyOrganizeResult {
279
+ applied: number;
280
+ failed: number;
281
+ }
282
+
283
+ /**
284
+ * Apply the back-apply action to every matched message, reusing the index-time
285
+ * apply plumbing: an idempotent MessageLabel upsert (additive) with
286
+ * `appliedByFilterId` deliberately ABSENT — this path attributes to no filter,
287
+ * exactly like a hand-applied label (RFC 034 Decision 3.3) — and an idempotent
288
+ * folder move (exclusive). One poisoned message never fails the batch; it is
289
+ * counted as failed and the pass continues.
290
+ */
291
+ export const applyOrganize = async (
292
+ deps: ApplyOrganizeDeps,
293
+ accountConfigId: string,
294
+ messageIds: readonly string[],
295
+ predicate: OrganizePredicate,
296
+ ): Promise<ApplyOrganizeResult> => {
297
+ const { client, moveService } = deps;
298
+ const applyLabel =
299
+ predicate.actionLabelId !== NO_ACTION && predicate.actionLabelId !== "";
300
+ const applyMove =
301
+ predicate.actionMailboxId !== NO_ACTION && predicate.actionMailboxId !== "";
302
+
303
+ const applyToMessage = async (messageId: string): Promise<void> => {
304
+ if (applyLabel) {
305
+ await client.messageLabel.apply({
306
+ messageId,
307
+ labelId: predicate.actionLabelId,
308
+ accountConfigId,
309
+ });
310
+ }
311
+ if (applyMove) {
312
+ if (!moveService) {
313
+ // An exclusive move was requested but this caller wired no move
314
+ // service. Never silently pretend it applied — surface it as a
315
+ // failed message so the job's failedCount reflects reality.
316
+ throw new Error(
317
+ "Organize move action requested but no move service is wired",
318
+ );
319
+ }
320
+ const message = await client.message.get(messageId);
321
+ const accountId = await client.mailbox.resolveAccountId(
322
+ message.mailboxId,
323
+ );
324
+ if (!accountId) {
325
+ throw new Error(
326
+ `Cannot resolve owning account for mailbox ${message.mailboxId}`,
327
+ );
328
+ }
329
+ await moveService.moveMessage(
330
+ accountConfigId,
331
+ messageId,
332
+ predicate.actionMailboxId,
333
+ accountId,
334
+ );
335
+ }
336
+ };
337
+
338
+ let applied = 0;
339
+ let failed = 0;
340
+ for (const messageId of messageIds) {
341
+ // Per-message isolation: one poisoned message is counted and skipped, the
342
+ // pass continues. `.catch()` (not a block try/catch) keeps this inside the
343
+ // no-silent-catch ban.
344
+ const ok = await applyToMessage(messageId)
345
+ .then(() => true)
346
+ .catch(() => false);
347
+ if (ok) {
348
+ applied += 1;
349
+ } else {
350
+ failed += 1;
351
+ }
352
+ }
353
+ return { applied, failed };
354
+ };
@@ -0,0 +1,64 @@
1
+ import assert from "node:assert/strict";
2
+ import { afterEach, beforeEach, describe, it } from "node:test";
3
+ import {
4
+ _resetSemanticCapabilityForTest,
5
+ isSemanticSearchUnavailable,
6
+ noteSemanticCapabilityAbsence,
7
+ } from "./semantic-capability.js";
8
+
9
+ const moduleNotFound = (): Error => {
10
+ const error = new Error(
11
+ "Cannot find package '@huggingface/transformers' imported from /app/server.mjs",
12
+ );
13
+ (error as Error & { code: string }).code = "ERR_MODULE_NOT_FOUND";
14
+ return error;
15
+ };
16
+
17
+ describe("noteSemanticCapabilityAbsence", () => {
18
+ const ORIGINAL = process.env.DATA_BACKEND;
19
+ beforeEach(() => {
20
+ _resetSemanticCapabilityForTest();
21
+ });
22
+ afterEach(() => {
23
+ if (ORIGINAL === undefined) delete process.env.DATA_BACKEND;
24
+ else process.env.DATA_BACKEND = ORIGINAL;
25
+ _resetSemanticCapabilityForTest();
26
+ });
27
+
28
+ it("absorbs a missing-module failure on the self-host SQL backends and remembers it", () => {
29
+ for (const backend of ["sqlite", "postgres"]) {
30
+ _resetSemanticCapabilityForTest();
31
+ process.env.DATA_BACKEND = backend;
32
+ assert.equal(isSemanticSearchUnavailable(), false);
33
+ assert.equal(noteSemanticCapabilityAbsence(moduleNotFound()), true);
34
+ assert.equal(isSemanticSearchUnavailable(), true);
35
+ }
36
+ });
37
+
38
+ it("absorbs a dlopen failure (musl loading a glibc extension)", () => {
39
+ process.env.DATA_BACKEND = "sqlite";
40
+ const error = new Error(
41
+ "Error loading shared library ld-linux-x86-64.so.2: No such file or directory",
42
+ );
43
+ (error as Error & { code: string }).code = "ERR_DLOPEN_FAILED";
44
+ assert.equal(noteSemanticCapabilityAbsence(error), true);
45
+ });
46
+
47
+ it("rethrows genuine query errors on the self-host SQL backends", () => {
48
+ process.env.DATA_BACKEND = "sqlite";
49
+ assert.equal(
50
+ noteSemanticCapabilityAbsence(new Error("SQLITE_BUSY")),
51
+ false,
52
+ );
53
+ assert.equal(isSemanticSearchUnavailable(), false);
54
+ assert.equal(noteSemanticCapabilityAbsence(undefined), false);
55
+ });
56
+
57
+ it("never engages on the AWS DynamoDB path — a missing module there is a broken deploy", () => {
58
+ delete process.env.DATA_BACKEND;
59
+ assert.equal(noteSemanticCapabilityAbsence(moduleNotFound()), false);
60
+ process.env.DATA_BACKEND = "dynamodb";
61
+ assert.equal(noteSemanticCapabilityAbsence(moduleNotFound()), false);
62
+ assert.equal(isSemanticSearchUnavailable(), false);
63
+ });
64
+ });
@@ -0,0 +1,62 @@
1
+ import { logger } from "@remit/logger-lambda";
2
+ import { isSelfHostSqlBackend } from "../data-backend.js";
3
+
4
+ /**
5
+ * Semantic-search capability gate for the self-host compose profiles.
6
+ *
7
+ * The backend container image ships neither `@huggingface/transformers` (query
8
+ * embedding) nor `sqlite-vec` (the vector store's loadable extension, glibc-only
9
+ * on top of that — see npm-scripts/docker-bundle.mjs and deploy/vps/README.md).
10
+ * Both are reached lazily through `runtimeImport`, so the first semantic query —
11
+ * not startup — hits ERR_MODULE_NOT_FOUND. Without this gate every search the
12
+ * web client issues fires a `/search/semantic` request that 500s (the client
13
+ * fetches semantic hits alongside every literal search), retried by the client
14
+ * on top.
15
+ *
16
+ * When the pipeline is genuinely absent, empty hits are the truthful answer:
17
+ * the FTS/literal engine is the primary search surface on these profiles and is
18
+ * unaffected. The absence is remembered so subsequent requests short-circuit.
19
+ *
20
+ * Scoped to the self-host SQL backends: on AWS the pipeline (Bedrock +
21
+ * S3 Vectors) is bundled, so a module-resolution failure there is a broken
22
+ * deploy and must keep failing loud. Falling back to the FTS engine here
23
+ * instead was considered and rejected — it would fabricate relevance scores and
24
+ * matched-chunk labels for the "Related" UI, and cannot serve the cross-account
25
+ * queries the daily brief issues.
26
+ */
27
+
28
+ const CAPABILITY_ABSENCE_CODES = new Set([
29
+ "ERR_MODULE_NOT_FOUND",
30
+ "MODULE_NOT_FOUND",
31
+ "ERR_DLOPEN_FAILED",
32
+ ]);
33
+
34
+ let semanticUnavailable = false;
35
+
36
+ /** Test-only reset for the memoized absence. */
37
+ export const _resetSemanticCapabilityForTest = (): void => {
38
+ semanticUnavailable = false;
39
+ };
40
+
41
+ export const isSemanticSearchUnavailable = (): boolean => semanticUnavailable;
42
+
43
+ /**
44
+ * Classify a semantic-search failure. Returns true — and remembers the
45
+ * absence — when it is the missing-module/extension shape on a self-host SQL
46
+ * backend; any other error is the caller's to rethrow.
47
+ */
48
+ export const noteSemanticCapabilityAbsence = (error: unknown): boolean => {
49
+ if (!isSelfHostSqlBackend()) return false;
50
+ const code = (error as { code?: unknown } | null)?.code;
51
+ if (typeof code !== "string" || !CAPABILITY_ABSENCE_CODES.has(code)) {
52
+ return false;
53
+ }
54
+ if (!semanticUnavailable) {
55
+ logger.warn(
56
+ { error: error instanceof Error ? error.message : String(error) },
57
+ "Semantic search pipeline unavailable in this deployment — serving empty semantic results (see deploy/vps/README.md)",
58
+ );
59
+ }
60
+ semanticUnavailable = true;
61
+ return true;
62
+ };
@@ -0,0 +1,4 @@
1
+ import { createQueueProducer } from "@remit/sqs-client/producer";
2
+ import { env } from "expect-env";
3
+
4
+ export const sqsClient = createQueueProducer({ queueUrl: env.SQS_QUEUE_URL });
@@ -0,0 +1,211 @@
1
+ import assert from "node:assert/strict";
2
+ import { describe, it } from "node:test";
3
+ import { SendMessageCommand, type SQSClient } from "@aws-sdk/client-sqs";
4
+ import {
5
+ buildManualSyncDedupId,
6
+ buildScheduledSyncDedupId,
7
+ buildSyncMailboxesCommand,
8
+ triggerAccountSync,
9
+ } from "./trigger-sync.js";
10
+
11
+ const FIFO_QUEUE_URL =
12
+ "https://sqs.eu-west-1.amazonaws.com/123456789012/remit-dev-mailboxes.fifo";
13
+ const STANDARD_QUEUE_URL =
14
+ "http://localhost:9324/000000000000/remit-dev-mailboxes";
15
+
16
+ const parseBody = (cmd: SendMessageCommand): Record<string, unknown> => {
17
+ const body = cmd.input.MessageBody ?? "{}";
18
+ return JSON.parse(body) as Record<string, unknown>;
19
+ };
20
+
21
+ describe("buildSyncMailboxesCommand", () => {
22
+ it("sets MessageGroupId to accountId for FIFO queues", () => {
23
+ const cmd = buildSyncMailboxesCommand({
24
+ sqsClient: {} as SQSClient,
25
+ queueUrl: FIFO_QUEUE_URL,
26
+ accountId: "account-abc",
27
+ });
28
+
29
+ assert.equal(cmd.input.MessageGroupId, "account-abc");
30
+ });
31
+
32
+ it("defaults MessageDeduplicationId to the time-bucketed manual dedup id for FIFO queues", () => {
33
+ const cmd = buildSyncMailboxesCommand({
34
+ sqsClient: {} as SQSClient,
35
+ queueUrl: FIFO_QUEUE_URL,
36
+ accountId: "account-abc",
37
+ });
38
+
39
+ assert.ok(
40
+ cmd.input.MessageDeduplicationId?.startsWith(
41
+ "SYNC_MAILBOXES:manual:account-abc:",
42
+ ),
43
+ );
44
+ });
45
+
46
+ it("does not set FIFO params for standard queues", () => {
47
+ const cmd = buildSyncMailboxesCommand({
48
+ sqsClient: {} as SQSClient,
49
+ queueUrl: STANDARD_QUEUE_URL,
50
+ accountId: "account-abc",
51
+ });
52
+
53
+ assert.equal(cmd.input.MessageGroupId, undefined);
54
+ assert.equal(cmd.input.MessageDeduplicationId, undefined);
55
+ });
56
+
57
+ it("builds a SYNC_MAILBOXES event body with accountId and eventId", () => {
58
+ const cmd = buildSyncMailboxesCommand({
59
+ sqsClient: {} as SQSClient,
60
+ queueUrl: FIFO_QUEUE_URL,
61
+ accountId: "account-abc",
62
+ });
63
+
64
+ const event = parseBody(cmd);
65
+ assert.equal(event.type, "SYNC_MAILBOXES");
66
+ assert.equal(event.accountId, "account-abc");
67
+ assert.equal(typeof event.eventId, "string");
68
+ assert.equal(typeof event.timestamp, "number");
69
+ });
70
+
71
+ it("targets the configured queue url", () => {
72
+ const cmd = buildSyncMailboxesCommand({
73
+ sqsClient: {} as SQSClient,
74
+ queueUrl: FIFO_QUEUE_URL,
75
+ accountId: "account-abc",
76
+ });
77
+
78
+ assert.equal(cmd.input.QueueUrl, FIFO_QUEUE_URL);
79
+ });
80
+
81
+ it("uses the caller-supplied dedupId instead of the manual default", () => {
82
+ const cmd = buildSyncMailboxesCommand({
83
+ sqsClient: {} as SQSClient,
84
+ queueUrl: FIFO_QUEUE_URL,
85
+ accountId: "account-abc",
86
+ dedupId: "SYNC_MAILBOXES:scheduled:account-abc:12345",
87
+ });
88
+
89
+ assert.equal(
90
+ cmd.input.MessageDeduplicationId,
91
+ "SYNC_MAILBOXES:scheduled:account-abc:12345",
92
+ );
93
+ });
94
+ });
95
+
96
+ describe("buildManualSyncDedupId", () => {
97
+ const SIXTY_SECONDS_MS = 60 * 1000;
98
+
99
+ it("is stable within the same time bucket (dedupes a rapid double-tap)", () => {
100
+ const bucketStart = 10 * SIXTY_SECONDS_MS;
101
+
102
+ const first = buildManualSyncDedupId("account-abc", bucketStart);
103
+ const second = buildManualSyncDedupId("account-abc", bucketStart + 1_000);
104
+
105
+ assert.equal(first, second);
106
+ });
107
+
108
+ it("changes across a bucket boundary (never dedupes two polls a minute apart)", () => {
109
+ const bucketStart = 10 * SIXTY_SECONDS_MS;
110
+
111
+ const thisTrigger = buildManualSyncDedupId("account-abc", bucketStart);
112
+ const nextTrigger = buildManualSyncDedupId(
113
+ "account-abc",
114
+ bucketStart + SIXTY_SECONDS_MS,
115
+ );
116
+
117
+ assert.notEqual(thisTrigger, nextTrigger);
118
+ });
119
+
120
+ it("never collides with the scheduler's dedup namespace", () => {
121
+ const now = 10 * SIXTY_SECONDS_MS;
122
+ const manual = buildManualSyncDedupId("account-abc", now);
123
+ const scheduled = buildScheduledSyncDedupId(
124
+ "account-abc",
125
+ now,
126
+ SIXTY_SECONDS_MS,
127
+ );
128
+
129
+ assert.notEqual(manual, scheduled);
130
+ });
131
+ });
132
+
133
+ describe("buildScheduledSyncDedupId", () => {
134
+ const FIVE_MINUTES_MS = 5 * 60 * 1000;
135
+
136
+ it("is stable within the same time bucket (dedupes a retried tick)", () => {
137
+ const bucketStart = 10 * FIVE_MINUTES_MS;
138
+
139
+ const first = buildScheduledSyncDedupId(
140
+ "account-abc",
141
+ bucketStart,
142
+ FIVE_MINUTES_MS,
143
+ );
144
+ const second = buildScheduledSyncDedupId(
145
+ "account-abc",
146
+ bucketStart + 1_000,
147
+ FIVE_MINUTES_MS,
148
+ );
149
+
150
+ assert.equal(first, second);
151
+ });
152
+
153
+ it("changes across a bucket boundary (never dedupes two ticks apart)", () => {
154
+ const bucketStart = 10 * FIVE_MINUTES_MS;
155
+
156
+ const thisTick = buildScheduledSyncDedupId(
157
+ "account-abc",
158
+ bucketStart,
159
+ FIVE_MINUTES_MS,
160
+ );
161
+ const nextTick = buildScheduledSyncDedupId(
162
+ "account-abc",
163
+ bucketStart + FIVE_MINUTES_MS,
164
+ FIVE_MINUTES_MS,
165
+ );
166
+
167
+ assert.notEqual(thisTick, nextTick);
168
+ });
169
+ });
170
+
171
+ describe("triggerAccountSync", () => {
172
+ it("sends a SendMessageCommand to the client", async () => {
173
+ const sent: SendMessageCommand[] = [];
174
+ const sqsClient = {
175
+ send: async (cmd: SendMessageCommand) => {
176
+ sent.push(cmd);
177
+ return {};
178
+ },
179
+ } as unknown as SQSClient;
180
+
181
+ const result = await triggerAccountSync({
182
+ sqsClient,
183
+ queueUrl: FIFO_QUEUE_URL,
184
+ accountId: "account-xyz",
185
+ });
186
+
187
+ assert.equal(sent.length, 1);
188
+ const cmd = sent[0];
189
+ if (!cmd) throw new Error("expected command");
190
+ assert.ok(cmd instanceof SendMessageCommand);
191
+ assert.equal(cmd.input.MessageGroupId, "account-xyz");
192
+ assert.equal(typeof result.eventId, "string");
193
+ });
194
+
195
+ it("propagates SQS send errors to the caller", async () => {
196
+ const sqsClient = {
197
+ send: async () => {
198
+ throw new Error("MissingRequiredParameterException");
199
+ },
200
+ } as unknown as SQSClient;
201
+
202
+ await assert.rejects(
203
+ triggerAccountSync({
204
+ sqsClient,
205
+ queueUrl: FIFO_QUEUE_URL,
206
+ accountId: "account-xyz",
207
+ }),
208
+ /MissingRequiredParameterException/,
209
+ );
210
+ });
211
+ });