@remit/search-service 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/package.json +66 -0
  2. package/src/anchor.test.ts +164 -0
  3. package/src/anchor.ts +127 -0
  4. package/src/backends/bedrock.test.ts +148 -0
  5. package/src/backends/bedrock.ts +105 -0
  6. package/src/backends/memory.test.ts +168 -0
  7. package/src/backends/memory.ts +152 -0
  8. package/src/backends/pgvector.integ.test.ts +174 -0
  9. package/src/backends/pgvector.ts +306 -0
  10. package/src/backends/runtime-import.ts +16 -0
  11. package/src/backends/s3-vectors.test.ts +929 -0
  12. package/src/backends/s3-vectors.ts +383 -0
  13. package/src/backends/sqlite-vec.integ.test.ts +144 -0
  14. package/src/backends/sqlite-vec.ts +250 -0
  15. package/src/bedrock.ts +4 -0
  16. package/src/chunking/chunker.test.ts +79 -0
  17. package/src/chunking/chunker.ts +56 -0
  18. package/src/chunking/entities.test.ts +82 -0
  19. package/src/chunking/entities.ts +74 -0
  20. package/src/chunking/entropy.test.ts +98 -0
  21. package/src/chunking/entropy.ts +161 -0
  22. package/src/chunking/keys.ts +22 -0
  23. package/src/chunking/structured.test.ts +120 -0
  24. package/src/chunking/structured.ts +79 -0
  25. package/src/content-hash.test.ts +27 -0
  26. package/src/content-hash.ts +10 -0
  27. package/src/embeddings.test.ts +28 -0
  28. package/src/embeddings.ts +149 -0
  29. package/src/from-env.test.ts +62 -0
  30. package/src/from-env.ts +130 -0
  31. package/src/index.ts +71 -0
  32. package/src/pgvector.ts +4 -0
  33. package/src/s3-vectors.ts +5 -0
  34. package/src/search.test.ts +772 -0
  35. package/src/search.ts +395 -0
  36. package/src/semantic-search.integ.test.ts +130 -0
  37. package/src/sqlite-vec.ts +4 -0
  38. package/src/types.ts +155 -0
  39. package/tsconfig.json +8 -0
@@ -0,0 +1,161 @@
1
+ import type { Chunk } from "../types.js";
2
+ import { MAX_CHUNKS_PER_TYPE } from "./keys.js";
3
+
4
+ const SIGNATURE_MARKERS: RegExp[] = [
5
+ /^--\s*$/m,
6
+ /^Best regards/im,
7
+ /^Best,/im,
8
+ /^Regards,/im,
9
+ /^Kind regards/im,
10
+ /^Thanks,/im,
11
+ /^Cheers,/im,
12
+ /^Sent from my (iPhone|iPad|Android|Samsung)/im,
13
+ ];
14
+
15
+ const QUOTE_MARKERS: RegExp[] = [
16
+ /^>/m,
17
+ /^On .+ wrote:$/im,
18
+ /^-{3,}\s*Original Message\s*-{3,}$/im,
19
+ /^_{3,}$/m,
20
+ /^From:.*Sent:.*To:/im,
21
+ ];
22
+
23
+ const stripQuoted = (text: string): string => {
24
+ const lines = text.split("\n");
25
+ const result: string[] = [];
26
+ for (const line of lines) {
27
+ let isQuoted = false;
28
+ for (const re of QUOTE_MARKERS) {
29
+ if (re.test(line)) {
30
+ isQuoted = true;
31
+ break;
32
+ }
33
+ }
34
+ if (isQuoted) break;
35
+ result.push(line);
36
+ }
37
+ return result.join("\n");
38
+ };
39
+
40
+ const stripSignature = (text: string): string => {
41
+ let earliest = text.length;
42
+ for (const re of SIGNATURE_MARKERS) {
43
+ const match = re.exec(text);
44
+ if (match && match.index < earliest) {
45
+ earliest = match.index;
46
+ }
47
+ }
48
+ return text.slice(0, earliest).trim();
49
+ };
50
+
51
+ export const stripBoilerplate = (text: string): string => {
52
+ return stripSignature(stripQuoted(text));
53
+ };
54
+
55
+ export const shannonEntropy = (text: string): number => {
56
+ if (text.length === 0) return 0;
57
+ const counts = new Map<string, number>();
58
+ for (const ch of text) {
59
+ counts.set(ch, (counts.get(ch) ?? 0) + 1);
60
+ }
61
+ let entropy = 0;
62
+ const len = text.length;
63
+ for (const c of counts.values()) {
64
+ const p = c / len;
65
+ entropy -= p * Math.log2(p);
66
+ }
67
+ return entropy;
68
+ };
69
+
70
+ const MIN_CHUNK_CHARS = 80;
71
+ const MAX_CHUNK_CHARS = 2000;
72
+ const ENTROPY_THRESHOLD = 3.5;
73
+ const WINDOW_SIZE = 100;
74
+
75
+ /**
76
+ * Hard ceiling for any chunk text handed to the embedder. Titan v2
77
+ * (amazon.titan-embed-text-v2:0) rejects inputs over 8192 tokens; 6000 chars
78
+ * stays comfortably under that even for dense, low-whitespace text.
79
+ */
80
+ export const EMBED_CHAR_BUDGET = 6000;
81
+
82
+ const splitParagraphs = (text: string): string[] => {
83
+ return text
84
+ .split(/\n\s*\n/)
85
+ .map((p) => p.trim())
86
+ .filter((p) => p.length > 0);
87
+ };
88
+
89
+ const mergeUntilCap = (paragraphs: string[]): string[] => {
90
+ const merged: string[] = [];
91
+ let current = "";
92
+ for (const p of paragraphs) {
93
+ if (current.length === 0) {
94
+ current = p;
95
+ continue;
96
+ }
97
+ if (current.length + p.length + 2 <= MAX_CHUNK_CHARS) {
98
+ current = `${current}\n\n${p}`;
99
+ continue;
100
+ }
101
+ merged.push(current);
102
+ current = p;
103
+ }
104
+ if (current.length > 0) merged.push(current);
105
+ return merged;
106
+ };
107
+
108
+ const isHighEntropy = (text: string): boolean => {
109
+ if (text.length < WINDOW_SIZE) {
110
+ return shannonEntropy(text) >= ENTROPY_THRESHOLD;
111
+ }
112
+ let maxEntropy = 0;
113
+ for (let i = 0; i + WINDOW_SIZE <= text.length; i += WINDOW_SIZE) {
114
+ const window = text.slice(i, i + WINDOW_SIZE);
115
+ const e = shannonEntropy(window);
116
+ if (e > maxEntropy) maxEntropy = e;
117
+ }
118
+ return maxEntropy >= ENTROPY_THRESHOLD;
119
+ };
120
+
121
+ export const splitToCharBudget = (text: string, budget: number): string[] => {
122
+ if (text.length <= budget) return [text];
123
+ const parts: string[] = [];
124
+ let cursor = 0;
125
+ while (cursor < text.length) {
126
+ parts.push(text.slice(cursor, cursor + budget));
127
+ cursor += budget;
128
+ }
129
+ return parts;
130
+ };
131
+
132
+ const splitOversized = (text: string): string[] =>
133
+ splitToCharBudget(text, MAX_CHUNK_CHARS);
134
+
135
+ export const buildBodyChunks = (
136
+ text: string,
137
+ chunkIdFor: (suffix: string) => string,
138
+ ): Chunk[] => {
139
+ const stripped = stripBoilerplate(text);
140
+ if (stripped.length < MIN_CHUNK_CHARS) return [];
141
+
142
+ const paragraphs = splitParagraphs(stripped);
143
+ const informative = paragraphs.filter(isHighEntropy);
144
+ if (informative.length === 0) return [];
145
+
146
+ const merged = mergeUntilCap(informative);
147
+ const final: string[] = [];
148
+ for (const m of merged) {
149
+ for (const part of splitOversized(m)) {
150
+ if (part.length >= MIN_CHUNK_CHARS) final.push(part);
151
+ }
152
+ }
153
+
154
+ // Cap at MAX_CHUNKS_PER_TYPE so every produced body-N has a matching key in
155
+ // candidateChunkKeys (keys.ts) and stays reapable on delete.
156
+ return final.slice(0, MAX_CHUNKS_PER_TYPE).map((textChunk, idx) => ({
157
+ chunkId: chunkIdFor(`body-${idx}`),
158
+ chunkType: "body",
159
+ text: textChunk,
160
+ }));
161
+ };
@@ -0,0 +1,22 @@
1
+ const STRUCTURED_SUFFIXES = ["sender", "recipient", "subject", "attachment"];
2
+
3
+ // Ceiling on body / entity chunks per message, shared with the producers
4
+ // (entropy.ts, entities.ts) so every chunk a message can emit has a reapable
5
+ // delete key. Observed counts are 1-19; DeleteVectors ignores keys that don't
6
+ // exist, so the delete stays scan-free.
7
+ export const MAX_CHUNKS_PER_TYPE = 128;
8
+
9
+ export const chunkKeyFor = (messageId: string, suffix: string): string =>
10
+ `${messageId}::${suffix}`;
11
+
12
+ // Every deterministic key a message's chunks can occupy, in lockstep with the
13
+ // suffixes the chunker emits (structured.ts, entropy.ts, entities.ts). Lets a
14
+ // message be deleted by addressing its keys directly — never listing the index.
15
+ export const candidateChunkKeys = (messageId: string): string[] => {
16
+ const suffixes = [...STRUCTURED_SUFFIXES, "entities"];
17
+ for (let i = 0; i < MAX_CHUNKS_PER_TYPE; i++) {
18
+ suffixes.push(`body-${i}`);
19
+ suffixes.push(`entities-${i}`);
20
+ }
21
+ return suffixes.map((suffix) => chunkKeyFor(messageId, suffix));
22
+ };
@@ -0,0 +1,120 @@
1
+ import assert from "node:assert";
2
+ import { describe, it } from "node:test";
3
+ import type { EnvelopeChunkInput } from "../types.js";
4
+ import {
5
+ buildStructuredChunks,
6
+ extractAttachmentFileTypes,
7
+ } from "./structured.js";
8
+
9
+ const baseEnvelope: EnvelopeChunkInput = {
10
+ from: { name: "Alice", email: "alice@example.com" },
11
+ to: [{ name: "Bob", email: "bob@example.com" }],
12
+ cc: [],
13
+ bcc: [],
14
+ subject: "Q1 invoice review",
15
+ attachments: [],
16
+ };
17
+
18
+ const idFor = (suffix: string): string => `msg-1::${suffix}`;
19
+
20
+ describe("buildStructuredChunks", () => {
21
+ it("emits sender, recipient, and subject chunks for a basic envelope", () => {
22
+ const chunks = buildStructuredChunks(baseEnvelope, idFor);
23
+ const types = chunks.map((c) => c.chunkType);
24
+ assert.deepStrictEqual(types, ["sender", "recipient", "subject"]);
25
+
26
+ const sender = chunks.find((c) => c.chunkType === "sender");
27
+ assert.ok(sender);
28
+ assert.match(sender.text, /alice@example\.com/);
29
+ assert.match(sender.text, /Alice/);
30
+
31
+ const recipient = chunks.find((c) => c.chunkType === "recipient");
32
+ assert.ok(recipient);
33
+ assert.match(recipient.text, /bob@example\.com/);
34
+
35
+ const subject = chunks.find((c) => c.chunkType === "subject");
36
+ assert.ok(subject);
37
+ assert.match(subject.text, /Q1 invoice review/);
38
+ });
39
+
40
+ it("uses email-only when sender has no display name", () => {
41
+ const chunks = buildStructuredChunks(
42
+ { ...baseEnvelope, from: { name: null, email: "alice@example.com" } },
43
+ idFor,
44
+ );
45
+ const sender = chunks.find((c) => c.chunkType === "sender");
46
+ assert.ok(sender);
47
+ assert.strictEqual(sender.text, "From: alice@example.com");
48
+ });
49
+
50
+ it("merges to/cc/bcc into a single recipient chunk", () => {
51
+ const chunks = buildStructuredChunks(
52
+ {
53
+ ...baseEnvelope,
54
+ to: [{ name: "Bob", email: "bob@example.com" }],
55
+ cc: [{ name: "Carol", email: "carol@example.com" }],
56
+ bcc: [{ name: null, email: "dave@example.com" }],
57
+ },
58
+ idFor,
59
+ );
60
+ const recipients = chunks.filter((c) => c.chunkType === "recipient");
61
+ assert.strictEqual(recipients.length, 1);
62
+ assert.match(recipients[0].text, /bob@example\.com/);
63
+ assert.match(recipients[0].text, /carol@example\.com/);
64
+ assert.match(recipients[0].text, /dave@example\.com/);
65
+ });
66
+
67
+ it("emits an attachment chunk when attachments are present", () => {
68
+ const chunks = buildStructuredChunks(
69
+ {
70
+ ...baseEnvelope,
71
+ attachments: [
72
+ {
73
+ filename: "invoice-q1-2026.pdf",
74
+ contentType: "application/pdf",
75
+ size: 245_000,
76
+ },
77
+ {
78
+ filename: "summary.xlsx",
79
+ contentType:
80
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
81
+ size: 89_000,
82
+ },
83
+ ],
84
+ },
85
+ idFor,
86
+ );
87
+ const att = chunks.find((c) => c.chunkType === "attachment");
88
+ assert.ok(att);
89
+ assert.match(att.text, /invoice-q1-2026\.pdf/);
90
+ assert.match(att.text, /application\/pdf/);
91
+ assert.match(att.text, /summary\.xlsx/);
92
+ });
93
+
94
+ it("skips empty subject and missing recipients", () => {
95
+ const chunks = buildStructuredChunks(
96
+ { ...baseEnvelope, to: [], subject: "" },
97
+ idFor,
98
+ );
99
+ const types = chunks.map((c) => c.chunkType);
100
+ assert.deepStrictEqual(types, ["sender"]);
101
+ });
102
+
103
+ it("uses deterministic chunk ids derived from the messageId", () => {
104
+ const chunks = buildStructuredChunks(baseEnvelope, idFor);
105
+ assert.ok(chunks.every((c) => c.chunkId.startsWith("msg-1::")));
106
+ });
107
+ });
108
+
109
+ describe("extractAttachmentFileTypes", () => {
110
+ it("dedupes and lowercases extensions and content subtypes", () => {
111
+ const types = extractAttachmentFileTypes([
112
+ { filename: "INVOICE.PDF", contentType: "application/pdf", size: 100 },
113
+ { filename: "summary.xlsx", contentType: "application/xlsx", size: 200 },
114
+ { filename: "another.pdf", contentType: "application/pdf", size: 100 },
115
+ ]);
116
+ assert.ok(types.includes("pdf"));
117
+ assert.ok(types.includes("xlsx"));
118
+ assert.strictEqual(new Set(types).size, types.length);
119
+ });
120
+ });
@@ -0,0 +1,79 @@
1
+ import type {
2
+ AttachmentChunkInput,
3
+ Chunk,
4
+ EnvelopeChunkAddress,
5
+ EnvelopeChunkInput,
6
+ } from "../types.js";
7
+
8
+ const formatAddress = (addr: EnvelopeChunkAddress): string => {
9
+ if (addr.name && addr.name.trim().length > 0) {
10
+ return `${addr.name} <${addr.email}>`;
11
+ }
12
+ return addr.email;
13
+ };
14
+
15
+ const formatAttachment = (att: AttachmentChunkInput): string => {
16
+ const filename = att.filename ?? "unnamed";
17
+ const sizeKb = Math.max(1, Math.round(att.size / 1024));
18
+ return `${filename} (${att.contentType}, ${sizeKb}KB)`;
19
+ };
20
+
21
+ export const buildStructuredChunks = (
22
+ envelope: EnvelopeChunkInput,
23
+ chunkIdFor: (suffix: string) => string,
24
+ ): Chunk[] => {
25
+ const chunks: Chunk[] = [];
26
+
27
+ chunks.push({
28
+ chunkId: chunkIdFor("sender"),
29
+ chunkType: "sender",
30
+ text: `From: ${formatAddress(envelope.from)}`,
31
+ });
32
+
33
+ const recipients = [...envelope.to, ...envelope.cc, ...envelope.bcc];
34
+ if (recipients.length > 0) {
35
+ chunks.push({
36
+ chunkId: chunkIdFor("recipient"),
37
+ chunkType: "recipient",
38
+ text: `To: ${recipients.map(formatAddress).join(", ")}`,
39
+ });
40
+ }
41
+
42
+ const subject = envelope.subject.trim();
43
+ if (subject.length > 0) {
44
+ chunks.push({
45
+ chunkId: chunkIdFor("subject"),
46
+ chunkType: "subject",
47
+ text: `Subject: ${subject}`,
48
+ });
49
+ }
50
+
51
+ if (envelope.attachments.length > 0) {
52
+ chunks.push({
53
+ chunkId: chunkIdFor("attachment"),
54
+ chunkType: "attachment",
55
+ text: `Attachments: ${envelope.attachments.map(formatAttachment).join(", ")}`,
56
+ });
57
+ }
58
+
59
+ return chunks;
60
+ };
61
+
62
+ export const extractAttachmentFileTypes = (
63
+ attachments: AttachmentChunkInput[],
64
+ ): string[] => {
65
+ const types = new Set<string>();
66
+ for (const att of attachments) {
67
+ if (att.filename) {
68
+ const dot = att.filename.lastIndexOf(".");
69
+ if (dot > 0 && dot < att.filename.length - 1) {
70
+ types.add(att.filename.slice(dot + 1).toLowerCase());
71
+ }
72
+ }
73
+ const slash = att.contentType.lastIndexOf("/");
74
+ if (slash > 0) {
75
+ types.add(att.contentType.slice(slash + 1).toLowerCase());
76
+ }
77
+ }
78
+ return Array.from(types);
79
+ };
@@ -0,0 +1,27 @@
1
+ import assert from "node:assert";
2
+ import { describe, it } from "node:test";
3
+ import { computeContentHash } from "./content-hash.js";
4
+
5
+ describe("computeContentHash", () => {
6
+ it("is stable for the same model id and text", () => {
7
+ const a = computeContentHash("model@1024", "hello world");
8
+ const b = computeContentHash("model@1024", "hello world");
9
+ assert.strictEqual(a, b);
10
+ });
11
+
12
+ it("changes when the text changes", () => {
13
+ const a = computeContentHash("model@1024", "hello world");
14
+ const b = computeContentHash("model@1024", "hello there");
15
+ assert.notStrictEqual(a, b);
16
+ });
17
+
18
+ it("changes when the embedding model/version id changes", () => {
19
+ const a = computeContentHash("model@1024", "hello world");
20
+ const b = computeContentHash("model@512", "hello world");
21
+ assert.notStrictEqual(a, b);
22
+ });
23
+
24
+ it("returns a hex sha256 digest", () => {
25
+ assert.match(computeContentHash("m", "x"), /^[0-9a-f]{64}$/);
26
+ });
27
+ });
@@ -0,0 +1,10 @@
1
+ import { createHash } from "node:crypto";
2
+
3
+ /**
4
+ * Stable hash of an embeddable chunk. Folds in the embedding model/version id so
5
+ * the same text re-hashes differently after a model or dimension change, forcing
6
+ * a re-embed; under the same model, unchanged text hashes identically and the
7
+ * re-index skips the write.
8
+ */
9
+ export const computeContentHash = (embeddingId: string, text: string): string =>
10
+ createHash("sha256").update(`${embeddingId}\n${text}`).digest("hex");
@@ -0,0 +1,28 @@
1
+ import assert from "node:assert/strict";
2
+ import { describe, it } from "node:test";
3
+ import { LocalEmbeddingService } from "./embeddings.js";
4
+
5
+ describe("LocalEmbeddingService", () => {
6
+ it("keeps the historical embeddingId when dtype is unset", () => {
7
+ const service = new LocalEmbeddingService({
8
+ modelId: "Xenova/paraphrase-multilingual-MiniLM-L12-v2",
9
+ dimensions: 384,
10
+ });
11
+ assert.equal(
12
+ service.embeddingId,
13
+ "local:Xenova/paraphrase-multilingual-MiniLM-L12-v2@384",
14
+ );
15
+ });
16
+
17
+ it("includes dtype in the embeddingId so quantized vectors get their own content hashes", () => {
18
+ const service = new LocalEmbeddingService({
19
+ modelId: "Xenova/paraphrase-multilingual-MiniLM-L12-v2",
20
+ dimensions: 384,
21
+ dtype: "q8",
22
+ });
23
+ assert.equal(
24
+ service.embeddingId,
25
+ "local:Xenova/paraphrase-multilingual-MiniLM-L12-v2:q8@384",
26
+ );
27
+ });
28
+ });
@@ -0,0 +1,149 @@
1
+ import type {
2
+ DataType,
3
+ FeatureExtractionPipeline,
4
+ pipeline as PipelineFn,
5
+ } from "@huggingface/transformers";
6
+ import { runtimeImport } from "./backends/runtime-import.js";
7
+
8
+ export interface EmbeddingService {
9
+ embed(texts: string[]): Promise<number[][]>;
10
+ readonly dimensions: number;
11
+ /**
12
+ * Stable identifier for the embedding model and the config that affects its
13
+ * output (model id + dimensions). Folded into each chunk's content hash so a
14
+ * model or dimension change invalidates the hash and forces a re-embed, while
15
+ * unchanged content under the same model stays a no-op.
16
+ */
17
+ readonly embeddingId: string;
18
+ }
19
+
20
+ export interface DeterministicEmbeddingConfig {
21
+ dimensions?: number;
22
+ }
23
+
24
+ const hashString = (s: string, seed: number): number => {
25
+ let h = seed;
26
+ for (let i = 0; i < s.length; i++) {
27
+ h = (h * 31 + s.charCodeAt(i)) | 0;
28
+ }
29
+ return h;
30
+ };
31
+
32
+ const tokenize = (text: string): string[] => {
33
+ return text
34
+ .toLowerCase()
35
+ .replace(/[^a-z0-9@._\-+]+/g, " ")
36
+ .split(/\s+/)
37
+ .filter((t) => t.length > 0);
38
+ };
39
+
40
+ /**
41
+ * Deterministic bag-of-words hashing embedder. Used in tests so the SearchService
42
+ * can be exercised without calling Bedrock. Same input text always produces the
43
+ * same vector, and overlapping tokens produce non-zero cosine similarity.
44
+ */
45
+ export class DeterministicEmbeddingService implements EmbeddingService {
46
+ readonly dimensions: number;
47
+ readonly embeddingId: string;
48
+
49
+ constructor(config: DeterministicEmbeddingConfig = {}) {
50
+ this.dimensions = config.dimensions ?? 64;
51
+ this.embeddingId = `deterministic@${this.dimensions}`;
52
+ }
53
+
54
+ embed = async (texts: string[]): Promise<number[][]> => {
55
+ return texts.map((t) => this.embedOne(t));
56
+ };
57
+
58
+ private embedOne(text: string): number[] {
59
+ const vector = new Array<number>(this.dimensions).fill(0);
60
+ const tokens = tokenize(text);
61
+ for (const token of tokens) {
62
+ const idx = Math.abs(hashString(token, 0x9e3779b1)) % this.dimensions;
63
+ const sign = hashString(token, 0x85ebca6b) & 1 ? 1 : -1;
64
+ vector[idx] += sign;
65
+ }
66
+ let norm = 0;
67
+ for (const v of vector) norm += v * v;
68
+ if (norm === 0) return vector;
69
+ const scale = 1 / Math.sqrt(norm);
70
+ return vector.map((v) => v * scale);
71
+ }
72
+ }
73
+
74
+ export const createDeterministicEmbeddingService = (
75
+ config?: DeterministicEmbeddingConfig,
76
+ ): EmbeddingService => new DeterministicEmbeddingService(config);
77
+
78
+ export interface LocalEmbeddingConfig {
79
+ modelId?: string;
80
+ dimensions?: number;
81
+ /**
82
+ * ONNX weight precision Transformers.js loads. Left unset it defaults to
83
+ * `fp32` (the full-precision `model.onnx`); set to `q8` to load the
84
+ * int8-quantized `model_quantized.onnx`, which the search-index-worker
85
+ * container bakes to keep the image small. The bake step and this runtime
86
+ * must pass the same value so they resolve the identical cached file.
87
+ */
88
+ dtype?: DataType;
89
+ }
90
+
91
+ const DEFAULT_LOCAL_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
92
+ const DEFAULT_LOCAL_DIMENSIONS = 384;
93
+
94
+ type TransformersModule = { pipeline: typeof PipelineFn };
95
+
96
+ /**
97
+ * In-process CPU embedder backed by Transformers.js, used for local dev so the
98
+ * SearchService produces real semantic vectors without calling Bedrock. The
99
+ * model is downloaded once (cached on disk by `@huggingface/transformers`) and
100
+ * loaded lazily on first use; the heavy dependency is never imported in the
101
+ * production bundle (see runtime-import.ts).
102
+ */
103
+ export class LocalEmbeddingService implements EmbeddingService {
104
+ readonly dimensions: number;
105
+ readonly embeddingId: string;
106
+ private modelId: string;
107
+ private dtype?: DataType;
108
+ private pipelinePromise: Promise<FeatureExtractionPipeline> | null = null;
109
+
110
+ constructor(config: LocalEmbeddingConfig = {}) {
111
+ this.modelId = config.modelId ?? DEFAULT_LOCAL_MODEL_ID;
112
+ this.dimensions = config.dimensions ?? DEFAULT_LOCAL_DIMENSIONS;
113
+ this.dtype = config.dtype;
114
+ // dtype is part of the embedding identity: quantized weights produce
115
+ // different vectors, and embeddingId feeds the content hash that gates
116
+ // re-embedding. Unset keeps the historical id so existing fp32 indexes
117
+ // are not invalidated.
118
+ this.embeddingId = this.dtype
119
+ ? `local:${this.modelId}:${this.dtype}@${this.dimensions}`
120
+ : `local:${this.modelId}@${this.dimensions}`;
121
+ }
122
+
123
+ private getPipeline = async (): Promise<FeatureExtractionPipeline> => {
124
+ if (this.pipelinePromise) return this.pipelinePromise;
125
+ this.pipelinePromise = (async () => {
126
+ const { pipeline } = await runtimeImport<TransformersModule>(
127
+ "@huggingface/transformers",
128
+ );
129
+ return pipeline("feature-extraction", this.modelId, {
130
+ dtype: this.dtype,
131
+ });
132
+ })();
133
+ return this.pipelinePromise;
134
+ };
135
+
136
+ embed = async (texts: string[]): Promise<number[][]> => {
137
+ if (texts.length === 0) return [];
138
+ const extractor = await this.getPipeline();
139
+ const tensor = await extractor(texts, {
140
+ pooling: "mean",
141
+ normalize: true,
142
+ });
143
+ return tensor.tolist() as number[][];
144
+ };
145
+ }
146
+
147
+ export const createLocalEmbeddingService = (
148
+ config?: LocalEmbeddingConfig,
149
+ ): EmbeddingService => new LocalEmbeddingService(config);
@@ -0,0 +1,62 @@
1
+ import assert from "node:assert/strict";
2
+ import { afterEach, describe, it } from "node:test";
3
+ import { buildEmbeddingServiceFromEnv } from "./from-env.js";
4
+
5
+ const ENV_KEYS = [
6
+ "SEARCH_EMBEDDING_PROVIDER",
7
+ "SEARCH_EMBEDDING_MODEL_ID",
8
+ "SEARCH_EMBEDDING_DIMENSIONS",
9
+ "SEARCH_EMBEDDING_DTYPE",
10
+ ] as const;
11
+
12
+ const saved = new Map<string, string | undefined>(
13
+ ENV_KEYS.map((key) => [key, process.env[key]]),
14
+ );
15
+
16
+ afterEach(() => {
17
+ for (const [key, value] of saved) {
18
+ if (value === undefined) delete process.env[key];
19
+ else process.env[key] = value;
20
+ }
21
+ });
22
+
23
+ describe("buildEmbeddingServiceFromEnv dtype handling", () => {
24
+ it("builds a local embedder without dtype in its id when SEARCH_EMBEDDING_DTYPE is unset", () => {
25
+ process.env.SEARCH_EMBEDDING_PROVIDER = "local";
26
+ process.env.SEARCH_EMBEDDING_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
27
+ delete process.env.SEARCH_EMBEDDING_DTYPE;
28
+
29
+ const service = buildEmbeddingServiceFromEnv();
30
+
31
+ assert.equal(service.embeddingId, "local:Xenova/all-MiniLM-L6-v2@384");
32
+ });
33
+
34
+ it("threads SEARCH_EMBEDDING_DTYPE into the local embedder identity", () => {
35
+ process.env.SEARCH_EMBEDDING_PROVIDER = "local";
36
+ process.env.SEARCH_EMBEDDING_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
37
+ process.env.SEARCH_EMBEDDING_DTYPE = "q8";
38
+
39
+ const service = buildEmbeddingServiceFromEnv();
40
+
41
+ assert.equal(service.embeddingId, "local:Xenova/all-MiniLM-L6-v2:q8@384");
42
+ });
43
+
44
+ it("rejects an unknown SEARCH_EMBEDDING_DTYPE loudly", () => {
45
+ process.env.SEARCH_EMBEDDING_PROVIDER = "local";
46
+ process.env.SEARCH_EMBEDDING_DTYPE = "int7";
47
+
48
+ assert.throws(
49
+ () => buildEmbeddingServiceFromEnv(),
50
+ /SEARCH_EMBEDDING_DTYPE must be one of/,
51
+ );
52
+ });
53
+
54
+ it("ignores SEARCH_EMBEDDING_DTYPE for non-local providers", () => {
55
+ delete process.env.SEARCH_EMBEDDING_PROVIDER;
56
+ process.env.SEARCH_EMBEDDING_DTYPE = "not-a-dtype";
57
+
58
+ const service = buildEmbeddingServiceFromEnv();
59
+
60
+ assert.equal(service.embeddingId, "deterministic@64");
61
+ });
62
+ });