libsql-search 0.10.1 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -15
- package/dist/index.cjs +17 -127
- package/dist/index.d.ts +10 -10
- package/dist/index.esm.js +17 -127
- package/dist/turso.cjs +90 -7
- package/dist/turso.d.ts +29 -8
- package/dist/turso.esm.js +90 -7
- package/docs/API.md +22 -17
- package/docs/INDEXING.md +13 -7
- package/docs/INTEGRATIONS.md +18 -24
- package/docs/MIGRATIONS.md +10 -10
- package/docs/PROVIDERS.md +3 -23
- package/docs/README.md +1 -1
- package/docs/TESTING.md +13 -12
- package/docs/TROUBLESHOOTING.md +0 -6
- package/docs/TURSO.md +55 -16
- package/package.json +1 -2
- package/docs/TROUBLESHOOTING-SHARP.md +0 -65
package/README.md
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
Use it when you want:
|
|
11
11
|
|
|
12
|
-
- one indexing/search API across
|
|
12
|
+
- one indexing/search API across external embedding providers
|
|
13
13
|
- direct control over vector dimensions, table names, and deployment shape
|
|
14
14
|
- a lightweight library instead of a hosted search product
|
|
15
15
|
|
|
@@ -45,7 +45,7 @@ Note for `0.17.x`: the client no longer exports `./package.json`, so `require("@
|
|
|
45
45
|
|
|
46
46
|
## Quick Start
|
|
47
47
|
|
|
48
|
-
This example uses
|
|
48
|
+
This example uses a separately deployed OpenAI-compatible embedding service. `libsql-search` never loads or hosts an embedding model in-process.
|
|
49
49
|
|
|
50
50
|
```ts
|
|
51
51
|
import { createClient } from "@libsql/client";
|
|
@@ -56,25 +56,29 @@ const client = createClient({
|
|
|
56
56
|
authToken: "your-auth-token",
|
|
57
57
|
});
|
|
58
58
|
|
|
59
|
-
|
|
59
|
+
const embeddingOptions = {
|
|
60
|
+
provider: "openai-compatible" as const,
|
|
61
|
+
baseUrl: process.env.EMBEDDING_BASE_URL!,
|
|
62
|
+
apiKey: process.env.EMBEDDING_API_KEY,
|
|
63
|
+
model: "bge-large-en-v1.5",
|
|
64
|
+
dimensions: 1024,
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
await createTable(client, "articles_bge_1024", 1024);
|
|
60
68
|
|
|
61
69
|
await indexContent({
|
|
62
70
|
client,
|
|
63
71
|
contentPath: "./content",
|
|
64
|
-
tableName: "
|
|
65
|
-
embeddingOptions
|
|
66
|
-
provider: "local",
|
|
67
|
-
},
|
|
72
|
+
tableName: "articles_bge_1024",
|
|
73
|
+
embeddingOptions,
|
|
68
74
|
});
|
|
69
75
|
|
|
70
76
|
const results = await search({
|
|
71
77
|
client,
|
|
72
78
|
query: "how do I deploy my docs site",
|
|
73
|
-
tableName: "
|
|
79
|
+
tableName: "articles_bge_1024",
|
|
74
80
|
limit: 5,
|
|
75
|
-
embeddingOptions
|
|
76
|
-
provider: "local",
|
|
77
|
-
},
|
|
81
|
+
embeddingOptions,
|
|
78
82
|
});
|
|
79
83
|
|
|
80
84
|
console.log(results.map((result) => ({
|
|
@@ -91,7 +95,7 @@ Important behavior:
|
|
|
91
95
|
- `indexContent()` embeds every document before it touches the database, then replaces the table in one transaction, so a failed rebuild leaves the previous index intact.
|
|
92
96
|
- `indexContent()` throws `IndexingError` when a file fails; pass `failurePolicy: "skip"` to rebuild from the remaining files.
|
|
93
97
|
- `indexContent()` throws `IndexingError` when no source files are found; pass `allowEmptyIndex: true` to intentionally empty the index.
|
|
94
|
-
-
|
|
98
|
+
- Every provider sends indexed and queried text to an external service; review that service's privacy, retention, and pricing terms.
|
|
95
99
|
|
|
96
100
|
## Search Accuracy And Performance
|
|
97
101
|
|
|
@@ -103,10 +107,10 @@ Two options control the trade-off:
|
|
|
103
107
|
|
|
104
108
|
```ts
|
|
105
109
|
// Widen the index probe to raise recall (default: max(limit * 4, 32))
|
|
106
|
-
await search({ client, query, limit: 10, candidates: 200 });
|
|
110
|
+
await search({ client, query, embeddingOptions, limit: 10, candidates: 200 });
|
|
107
111
|
|
|
108
112
|
// Bypass the index entirely: exact, but linear in table size
|
|
109
|
-
await search({ client, query, exact: true });
|
|
113
|
+
await search({ client, query, embeddingOptions, exact: true });
|
|
110
114
|
```
|
|
111
115
|
|
|
112
116
|
`exact: true` is the only way to guarantee exactness. Use it for small corpora, for correctness checks against the index path, and for tables that have no vector index.
|
|
@@ -117,7 +121,6 @@ Requirements: `vector_top_k()` and `libsql_vector_idx()` need a libSQL build wit
|
|
|
117
121
|
|
|
118
122
|
Built-in providers:
|
|
119
123
|
|
|
120
|
-
- `local` with `Xenova/all-MiniLM-L6-v2` at 384 dimensions
|
|
121
124
|
- `cloudflare` with `@cf/baai/bge-m3` at 1024 dimensions
|
|
122
125
|
- `mistral` with `mistral-embed` at 1024 dimensions
|
|
123
126
|
- `gemini` with `gemini-embedding-2` at 128-3072 dimensions, default 3072
|
package/dist/index.cjs
CHANGED
|
@@ -6,10 +6,8 @@ var matter = require('gray-matter');
|
|
|
6
6
|
|
|
7
7
|
const OPENAI_DEFAULT_DIMENSIONS = 768;
|
|
8
8
|
const OPENAI_COMPATIBLE_DEFAULT_BATCH_SIZE = 32;
|
|
9
|
-
const LOCAL_DIMENSIONS = 384;
|
|
10
9
|
const DEFAULT_MAX_LENGTH = 8e3;
|
|
11
10
|
const DEFAULT_TIMEOUT_MS = 3e4;
|
|
12
|
-
const LOCAL_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
13
11
|
const GEMINI_MODEL = "gemini-embedding-2";
|
|
14
12
|
const GEMINI_DIMENSIONS = 3072;
|
|
15
13
|
const GEMINI_MIN_DIMENSIONS = 128;
|
|
@@ -22,7 +20,6 @@ const MISTRAL_EMBEDDINGS_URL = "https://api.mistral.ai/v1/embeddings";
|
|
|
22
20
|
const MISTRAL_DIMENSIONS = 1024;
|
|
23
21
|
const CLOUDFLARE_MODEL = "@cf/baai/bge-m3";
|
|
24
22
|
const CLOUDFLARE_DIMENSIONS = 1024;
|
|
25
|
-
const localModelCacheByModel = /* @__PURE__ */ new Map();
|
|
26
23
|
function getEnvironmentVariable(name) {
|
|
27
24
|
const runtime = globalThis;
|
|
28
25
|
const nodeValue = runtime.process?.env?.[name];
|
|
@@ -35,67 +32,6 @@ function getEnvironmentVariable(name) {
|
|
|
35
32
|
return void 0;
|
|
36
33
|
}
|
|
37
34
|
}
|
|
38
|
-
function deletePendingLocalModelCache(modelName, entry) {
|
|
39
|
-
if (!entry.settled && entry.waiters === 0 && localModelCacheByModel.get(modelName) === entry) {
|
|
40
|
-
localModelCacheByModel.delete(modelName);
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
async function getLocalEmbeddingModel(modelName, signal) {
|
|
44
|
-
const cached = localModelCacheByModel.get(modelName);
|
|
45
|
-
if (cached) {
|
|
46
|
-
cached.waiters++;
|
|
47
|
-
try {
|
|
48
|
-
return await waitForLocalEmbeddingModel(modelName, cached, signal);
|
|
49
|
-
} finally {
|
|
50
|
-
cached.waiters--;
|
|
51
|
-
deletePendingLocalModelCache(modelName, cached);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
const modelPromise = (async () => {
|
|
55
|
-
console.log(`Loading local embedding model (${modelName})...`);
|
|
56
|
-
const { pipeline } = await import('@huggingface/transformers');
|
|
57
|
-
const model = await pipeline("feature-extraction", modelName);
|
|
58
|
-
console.log("Local model loaded successfully");
|
|
59
|
-
return model;
|
|
60
|
-
})();
|
|
61
|
-
const entry = {
|
|
62
|
-
promise: modelPromise,
|
|
63
|
-
settled: false,
|
|
64
|
-
waiters: 1
|
|
65
|
-
};
|
|
66
|
-
localModelCacheByModel.set(modelName, entry);
|
|
67
|
-
modelPromise.then(() => {
|
|
68
|
-
entry.settled = true;
|
|
69
|
-
}).catch(() => {
|
|
70
|
-
localModelCacheByModel.delete(modelName);
|
|
71
|
-
});
|
|
72
|
-
try {
|
|
73
|
-
return await waitForLocalEmbeddingModel(modelName, entry, signal);
|
|
74
|
-
} finally {
|
|
75
|
-
entry.waiters--;
|
|
76
|
-
deletePendingLocalModelCache(modelName, entry);
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
async function waitForLocalEmbeddingModel(modelName, entry, signal) {
|
|
80
|
-
if (signal.aborted) {
|
|
81
|
-
deletePendingLocalModelCache(modelName, entry);
|
|
82
|
-
throw providerError("local", "model inference was aborted");
|
|
83
|
-
}
|
|
84
|
-
let rejectAbort = () => {
|
|
85
|
-
};
|
|
86
|
-
const abortPromise = new Promise((_resolve, reject) => {
|
|
87
|
-
rejectAbort = reject;
|
|
88
|
-
});
|
|
89
|
-
const onAbort = () => {
|
|
90
|
-
rejectAbort(providerError("local", "model inference was aborted"));
|
|
91
|
-
};
|
|
92
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
93
|
-
try {
|
|
94
|
-
return await Promise.race([entry.promise, abortPromise]);
|
|
95
|
-
} finally {
|
|
96
|
-
signal.removeEventListener("abort", onAbort);
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
35
|
function getPositiveInteger(value, optionName) {
|
|
100
36
|
if (!Number.isInteger(value) || value <= 0) {
|
|
101
37
|
throw new Error(`Invalid ${optionName}: expected a positive integer`);
|
|
@@ -106,15 +42,17 @@ function getTimeoutMs(value) {
|
|
|
106
42
|
return getPositiveInteger(value ?? DEFAULT_TIMEOUT_MS, "timeoutMs");
|
|
107
43
|
}
|
|
108
44
|
function resolveProviderName(provider) {
|
|
109
|
-
switch (provider
|
|
110
|
-
case "local":
|
|
45
|
+
switch (provider) {
|
|
111
46
|
case "gemini":
|
|
112
47
|
case "openai":
|
|
113
48
|
case "mistral":
|
|
114
49
|
case "cloudflare":
|
|
115
50
|
case "openai-compatible":
|
|
116
|
-
return provider
|
|
51
|
+
return provider;
|
|
117
52
|
default:
|
|
53
|
+
if (provider === void 0) {
|
|
54
|
+
throw new Error("Embedding provider is required");
|
|
55
|
+
}
|
|
118
56
|
throw new Error(`Unknown embedding provider: ${String(provider)}`);
|
|
119
57
|
}
|
|
120
58
|
}
|
|
@@ -326,13 +264,6 @@ function normalizeOpenAICompatibleEmbeddingsUrl(baseUrl) {
|
|
|
326
264
|
}
|
|
327
265
|
function createProviderMetadata(provider, dimensions, model) {
|
|
328
266
|
switch (provider) {
|
|
329
|
-
case "local":
|
|
330
|
-
return Object.freeze({
|
|
331
|
-
name: "local",
|
|
332
|
-
model: LOCAL_MODEL,
|
|
333
|
-
dimensions,
|
|
334
|
-
batch: Object.freeze({ mode: "sequential" })
|
|
335
|
-
});
|
|
336
267
|
case "gemini":
|
|
337
268
|
return Object.freeze({
|
|
338
269
|
name: "gemini",
|
|
@@ -371,15 +302,6 @@ function createProviderMetadata(provider, dimensions, model) {
|
|
|
371
302
|
}
|
|
372
303
|
}
|
|
373
304
|
function getEffectiveDimensions(provider, dimensions) {
|
|
374
|
-
if (provider === "local") {
|
|
375
|
-
if (dimensions !== void 0 && dimensions !== LOCAL_DIMENSIONS) {
|
|
376
|
-
throw providerError(
|
|
377
|
-
"local",
|
|
378
|
-
`${LOCAL_MODEL} returns ${LOCAL_DIMENSIONS} dimensions; received dimensions ${String(dimensions)}`
|
|
379
|
-
);
|
|
380
|
-
}
|
|
381
|
-
return LOCAL_DIMENSIONS;
|
|
382
|
-
}
|
|
383
305
|
if (provider === "gemini") {
|
|
384
306
|
const effectiveDimensions = dimensions ?? GEMINI_DIMENSIONS;
|
|
385
307
|
if (!Number.isInteger(effectiveDimensions) || effectiveDimensions < GEMINI_MIN_DIMENSIONS || effectiveDimensions > GEMINI_MAX_DIMENSIONS) {
|
|
@@ -440,42 +362,6 @@ function createEmbeddingBatchResult(metadata, intent, embeddings) {
|
|
|
440
362
|
intent
|
|
441
363
|
});
|
|
442
364
|
}
|
|
443
|
-
class LocalEmbeddingProvider {
|
|
444
|
-
metadata;
|
|
445
|
-
#timeoutMs;
|
|
446
|
-
constructor(metadata, timeoutMs) {
|
|
447
|
-
this.metadata = metadata;
|
|
448
|
-
this.#timeoutMs = timeoutMs;
|
|
449
|
-
}
|
|
450
|
-
async embed(texts, options = {}) {
|
|
451
|
-
const intent = resolveIntent(options.intent);
|
|
452
|
-
if (texts.length === 0) {
|
|
453
|
-
return createEmbeddingBatchResult(this.metadata, intent, []);
|
|
454
|
-
}
|
|
455
|
-
assertBatchSize(this.metadata, texts.length);
|
|
456
|
-
const vectors = await withTimeout(
|
|
457
|
-
"local",
|
|
458
|
-
"model inference",
|
|
459
|
-
this.#timeoutMs,
|
|
460
|
-
async (signal) => {
|
|
461
|
-
const model = await getLocalEmbeddingModel(this.metadata.model, signal);
|
|
462
|
-
return embedSequentially("local", "model inference", texts, signal, async (text) => {
|
|
463
|
-
const output = await model(text, {
|
|
464
|
-
pooling: "mean",
|
|
465
|
-
normalize: true
|
|
466
|
-
});
|
|
467
|
-
return Array.from(output.data);
|
|
468
|
-
});
|
|
469
|
-
},
|
|
470
|
-
options.signal
|
|
471
|
-
);
|
|
472
|
-
return createEmbeddingBatchResult(
|
|
473
|
-
this.metadata,
|
|
474
|
-
intent,
|
|
475
|
-
validateEmbeddingBatch(vectors, texts.length, this.metadata.dimensions, "local")
|
|
476
|
-
);
|
|
477
|
-
}
|
|
478
|
-
}
|
|
479
365
|
class GeminiEmbeddingProvider {
|
|
480
366
|
metadata;
|
|
481
367
|
#apiKey;
|
|
@@ -628,13 +514,11 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
628
514
|
);
|
|
629
515
|
}
|
|
630
516
|
}
|
|
631
|
-
function createEmbeddingProvider(options
|
|
517
|
+
function createEmbeddingProvider(options) {
|
|
632
518
|
const provider = resolveProviderName(options.provider);
|
|
633
519
|
const metadata = getEmbeddingProviderMetadata(options);
|
|
634
520
|
const timeoutMs = getTimeoutMs(options.timeoutMs);
|
|
635
521
|
switch (provider) {
|
|
636
|
-
case "local":
|
|
637
|
-
return new LocalEmbeddingProvider(metadata, timeoutMs);
|
|
638
522
|
case "gemini": {
|
|
639
523
|
const key = getOptionalTrimmedCredential(
|
|
640
524
|
options.apiKey ?? getEnvironmentVariable("GEMINI_API_KEY")
|
|
@@ -733,7 +617,7 @@ function createEmbeddingProvider(options = {}) {
|
|
|
733
617
|
}
|
|
734
618
|
}
|
|
735
619
|
}
|
|
736
|
-
function getEmbeddingProviderMetadata(options
|
|
620
|
+
function getEmbeddingProviderMetadata(options) {
|
|
737
621
|
const provider = resolveProviderName(options.provider);
|
|
738
622
|
if (provider === "openai-compatible") {
|
|
739
623
|
normalizeOpenAICompatibleEmbeddingsUrl(getRequiredTrimmedString(options.baseUrl, "baseUrl"));
|
|
@@ -743,7 +627,7 @@ function getEmbeddingProviderMetadata(options = {}) {
|
|
|
743
627
|
}
|
|
744
628
|
return createProviderMetadata(provider, getEffectiveDimensions(provider, options.dimensions), options.model);
|
|
745
629
|
}
|
|
746
|
-
async function generateEmbeddings(texts, options
|
|
630
|
+
async function generateEmbeddings(texts, options) {
|
|
747
631
|
const maxLength = getPositiveInteger(options.maxLength ?? DEFAULT_MAX_LENGTH, "maxLength");
|
|
748
632
|
const intent = resolveIntent(options.intent);
|
|
749
633
|
if (texts.length === 0) {
|
|
@@ -756,7 +640,7 @@ async function generateEmbeddings(texts, options = {}) {
|
|
|
756
640
|
});
|
|
757
641
|
return result.embeddings;
|
|
758
642
|
}
|
|
759
|
-
async function generateEmbedding(text, options
|
|
643
|
+
async function generateEmbedding(text, options) {
|
|
760
644
|
const [embedding] = await generateEmbeddings([text], options);
|
|
761
645
|
if (!embedding) {
|
|
762
646
|
throw new Error("Embedding provider returned no embedding");
|
|
@@ -889,7 +773,7 @@ async function indexContent(options) {
|
|
|
889
773
|
const {
|
|
890
774
|
client,
|
|
891
775
|
contentPath,
|
|
892
|
-
embeddingOptions
|
|
776
|
+
embeddingOptions,
|
|
893
777
|
fileExtensions = [".md", ".markdown"],
|
|
894
778
|
exclude = ["node_modules", ".git", "dist", "build"],
|
|
895
779
|
tableName = "articles",
|
|
@@ -899,6 +783,9 @@ async function indexContent(options) {
|
|
|
899
783
|
} = options;
|
|
900
784
|
const database = resolveDatabase(client);
|
|
901
785
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
786
|
+
if (embeddingOptions === void 0) {
|
|
787
|
+
throw new TypeError("embeddingOptions is required");
|
|
788
|
+
}
|
|
902
789
|
let files;
|
|
903
790
|
try {
|
|
904
791
|
files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
|
|
@@ -1178,7 +1065,7 @@ async function search(options) {
|
|
|
1178
1065
|
query,
|
|
1179
1066
|
limit = 10,
|
|
1180
1067
|
tableName = "articles",
|
|
1181
|
-
embeddingOptions
|
|
1068
|
+
embeddingOptions,
|
|
1182
1069
|
candidates,
|
|
1183
1070
|
exact = false
|
|
1184
1071
|
} = options;
|
|
@@ -1190,6 +1077,9 @@ async function search(options) {
|
|
|
1190
1077
|
"embedding index name"
|
|
1191
1078
|
);
|
|
1192
1079
|
const candidateCount = normalizeSearchCandidates(candidates, resultLimit);
|
|
1080
|
+
if (embeddingOptions === void 0) {
|
|
1081
|
+
throw new TypeError("embeddingOptions is required");
|
|
1082
|
+
}
|
|
1193
1083
|
const queryEmbedding = await generateEmbedding(query, {
|
|
1194
1084
|
...embeddingOptions,
|
|
1195
1085
|
intent: embeddingOptions.intent ?? "query"
|
package/dist/index.d.ts
CHANGED
|
@@ -2,10 +2,10 @@ import { Client } from '@libsql/client';
|
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Multi-provider embedding generation
|
|
5
|
-
* Supports
|
|
6
|
-
*
|
|
5
|
+
* Supports Gemini, OpenAI, Mistral, Cloudflare Workers AI, and custom
|
|
6
|
+
* OpenAI-compatible endpoints.
|
|
7
7
|
*/
|
|
8
|
-
type EmbeddingProvider = '
|
|
8
|
+
type EmbeddingProvider = 'gemini' | 'openai' | 'mistral' | 'cloudflare' | 'openai-compatible';
|
|
9
9
|
type EmbeddingIntent = 'document' | 'query';
|
|
10
10
|
type EmbeddingBatchMode = 'native' | 'sequential';
|
|
11
11
|
interface EmbeddingBatchBehavior {
|
|
@@ -34,7 +34,7 @@ interface EmbeddingBatchResult {
|
|
|
34
34
|
intent: EmbeddingIntent;
|
|
35
35
|
}
|
|
36
36
|
interface EmbeddingOptions {
|
|
37
|
-
provider
|
|
37
|
+
provider: EmbeddingProvider;
|
|
38
38
|
apiKey?: string;
|
|
39
39
|
accountId?: string;
|
|
40
40
|
apiToken?: string;
|
|
@@ -53,16 +53,16 @@ interface EmbeddingBatchItemResult {
|
|
|
53
53
|
}
|
|
54
54
|
type EmbeddingBatchItem = number[] | EmbeddingBatchItemResult;
|
|
55
55
|
declare function validateEmbeddingBatch(items: EmbeddingBatchItem[], expectedCount: number, expectedDimensions: number, provider: EmbeddingProvider): number[][];
|
|
56
|
-
declare function createEmbeddingProvider(options
|
|
57
|
-
declare function getEmbeddingProviderMetadata(options
|
|
56
|
+
declare function createEmbeddingProvider(options: EmbeddingOptions): EmbeddingProviderClient;
|
|
57
|
+
declare function getEmbeddingProviderMetadata(options: EmbeddingOptions): EmbeddingProviderMetadata;
|
|
58
58
|
/**
|
|
59
59
|
* Generate embeddings using the specified provider.
|
|
60
60
|
*/
|
|
61
|
-
declare function generateEmbeddings(texts: string[], options
|
|
61
|
+
declare function generateEmbeddings(texts: string[], options: EmbeddingOptions): Promise<number[][]>;
|
|
62
62
|
/**
|
|
63
63
|
* Generate an embedding using the specified provider.
|
|
64
64
|
*/
|
|
65
|
-
declare function generateEmbedding(text: string, options
|
|
65
|
+
declare function generateEmbedding(text: string, options: EmbeddingOptions): Promise<number[]>;
|
|
66
66
|
/**
|
|
67
67
|
* Pad or truncate embedding to target dimensions
|
|
68
68
|
*/
|
|
@@ -189,7 +189,7 @@ interface IndexerOptions {
|
|
|
189
189
|
*/
|
|
190
190
|
client: DatabaseClient;
|
|
191
191
|
contentPath: string;
|
|
192
|
-
embeddingOptions
|
|
192
|
+
embeddingOptions: EmbeddingOptions;
|
|
193
193
|
fileExtensions?: string[];
|
|
194
194
|
exclude?: string[];
|
|
195
195
|
tableName?: string;
|
|
@@ -282,7 +282,7 @@ interface SearchOptions {
|
|
|
282
282
|
query: string;
|
|
283
283
|
limit?: number;
|
|
284
284
|
tableName?: string;
|
|
285
|
-
embeddingOptions
|
|
285
|
+
embeddingOptions: EmbeddingOptions;
|
|
286
286
|
/**
|
|
287
287
|
* How many candidates to pull from the vector index before the exact
|
|
288
288
|
* re-rank. Must be an integer from `limit` through {@link MAX_SEARCH_CANDIDATES}.
|
package/dist/index.esm.js
CHANGED
|
@@ -4,10 +4,8 @@ import matter from 'gray-matter';
|
|
|
4
4
|
|
|
5
5
|
const OPENAI_DEFAULT_DIMENSIONS = 768;
|
|
6
6
|
const OPENAI_COMPATIBLE_DEFAULT_BATCH_SIZE = 32;
|
|
7
|
-
const LOCAL_DIMENSIONS = 384;
|
|
8
7
|
const DEFAULT_MAX_LENGTH = 8e3;
|
|
9
8
|
const DEFAULT_TIMEOUT_MS = 3e4;
|
|
10
|
-
const LOCAL_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
11
9
|
const GEMINI_MODEL = "gemini-embedding-2";
|
|
12
10
|
const GEMINI_DIMENSIONS = 3072;
|
|
13
11
|
const GEMINI_MIN_DIMENSIONS = 128;
|
|
@@ -20,7 +18,6 @@ const MISTRAL_EMBEDDINGS_URL = "https://api.mistral.ai/v1/embeddings";
|
|
|
20
18
|
const MISTRAL_DIMENSIONS = 1024;
|
|
21
19
|
const CLOUDFLARE_MODEL = "@cf/baai/bge-m3";
|
|
22
20
|
const CLOUDFLARE_DIMENSIONS = 1024;
|
|
23
|
-
const localModelCacheByModel = /* @__PURE__ */ new Map();
|
|
24
21
|
function getEnvironmentVariable(name) {
|
|
25
22
|
const runtime = globalThis;
|
|
26
23
|
const nodeValue = runtime.process?.env?.[name];
|
|
@@ -33,67 +30,6 @@ function getEnvironmentVariable(name) {
|
|
|
33
30
|
return void 0;
|
|
34
31
|
}
|
|
35
32
|
}
|
|
36
|
-
function deletePendingLocalModelCache(modelName, entry) {
|
|
37
|
-
if (!entry.settled && entry.waiters === 0 && localModelCacheByModel.get(modelName) === entry) {
|
|
38
|
-
localModelCacheByModel.delete(modelName);
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
async function getLocalEmbeddingModel(modelName, signal) {
|
|
42
|
-
const cached = localModelCacheByModel.get(modelName);
|
|
43
|
-
if (cached) {
|
|
44
|
-
cached.waiters++;
|
|
45
|
-
try {
|
|
46
|
-
return await waitForLocalEmbeddingModel(modelName, cached, signal);
|
|
47
|
-
} finally {
|
|
48
|
-
cached.waiters--;
|
|
49
|
-
deletePendingLocalModelCache(modelName, cached);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
const modelPromise = (async () => {
|
|
53
|
-
console.log(`Loading local embedding model (${modelName})...`);
|
|
54
|
-
const { pipeline } = await import('@huggingface/transformers');
|
|
55
|
-
const model = await pipeline("feature-extraction", modelName);
|
|
56
|
-
console.log("Local model loaded successfully");
|
|
57
|
-
return model;
|
|
58
|
-
})();
|
|
59
|
-
const entry = {
|
|
60
|
-
promise: modelPromise,
|
|
61
|
-
settled: false,
|
|
62
|
-
waiters: 1
|
|
63
|
-
};
|
|
64
|
-
localModelCacheByModel.set(modelName, entry);
|
|
65
|
-
modelPromise.then(() => {
|
|
66
|
-
entry.settled = true;
|
|
67
|
-
}).catch(() => {
|
|
68
|
-
localModelCacheByModel.delete(modelName);
|
|
69
|
-
});
|
|
70
|
-
try {
|
|
71
|
-
return await waitForLocalEmbeddingModel(modelName, entry, signal);
|
|
72
|
-
} finally {
|
|
73
|
-
entry.waiters--;
|
|
74
|
-
deletePendingLocalModelCache(modelName, entry);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
async function waitForLocalEmbeddingModel(modelName, entry, signal) {
|
|
78
|
-
if (signal.aborted) {
|
|
79
|
-
deletePendingLocalModelCache(modelName, entry);
|
|
80
|
-
throw providerError("local", "model inference was aborted");
|
|
81
|
-
}
|
|
82
|
-
let rejectAbort = () => {
|
|
83
|
-
};
|
|
84
|
-
const abortPromise = new Promise((_resolve, reject) => {
|
|
85
|
-
rejectAbort = reject;
|
|
86
|
-
});
|
|
87
|
-
const onAbort = () => {
|
|
88
|
-
rejectAbort(providerError("local", "model inference was aborted"));
|
|
89
|
-
};
|
|
90
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
91
|
-
try {
|
|
92
|
-
return await Promise.race([entry.promise, abortPromise]);
|
|
93
|
-
} finally {
|
|
94
|
-
signal.removeEventListener("abort", onAbort);
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
33
|
function getPositiveInteger(value, optionName) {
|
|
98
34
|
if (!Number.isInteger(value) || value <= 0) {
|
|
99
35
|
throw new Error(`Invalid ${optionName}: expected a positive integer`);
|
|
@@ -104,15 +40,17 @@ function getTimeoutMs(value) {
|
|
|
104
40
|
return getPositiveInteger(value ?? DEFAULT_TIMEOUT_MS, "timeoutMs");
|
|
105
41
|
}
|
|
106
42
|
function resolveProviderName(provider) {
|
|
107
|
-
switch (provider
|
|
108
|
-
case "local":
|
|
43
|
+
switch (provider) {
|
|
109
44
|
case "gemini":
|
|
110
45
|
case "openai":
|
|
111
46
|
case "mistral":
|
|
112
47
|
case "cloudflare":
|
|
113
48
|
case "openai-compatible":
|
|
114
|
-
return provider
|
|
49
|
+
return provider;
|
|
115
50
|
default:
|
|
51
|
+
if (provider === void 0) {
|
|
52
|
+
throw new Error("Embedding provider is required");
|
|
53
|
+
}
|
|
116
54
|
throw new Error(`Unknown embedding provider: ${String(provider)}`);
|
|
117
55
|
}
|
|
118
56
|
}
|
|
@@ -324,13 +262,6 @@ function normalizeOpenAICompatibleEmbeddingsUrl(baseUrl) {
|
|
|
324
262
|
}
|
|
325
263
|
function createProviderMetadata(provider, dimensions, model) {
|
|
326
264
|
switch (provider) {
|
|
327
|
-
case "local":
|
|
328
|
-
return Object.freeze({
|
|
329
|
-
name: "local",
|
|
330
|
-
model: LOCAL_MODEL,
|
|
331
|
-
dimensions,
|
|
332
|
-
batch: Object.freeze({ mode: "sequential" })
|
|
333
|
-
});
|
|
334
265
|
case "gemini":
|
|
335
266
|
return Object.freeze({
|
|
336
267
|
name: "gemini",
|
|
@@ -369,15 +300,6 @@ function createProviderMetadata(provider, dimensions, model) {
|
|
|
369
300
|
}
|
|
370
301
|
}
|
|
371
302
|
function getEffectiveDimensions(provider, dimensions) {
|
|
372
|
-
if (provider === "local") {
|
|
373
|
-
if (dimensions !== void 0 && dimensions !== LOCAL_DIMENSIONS) {
|
|
374
|
-
throw providerError(
|
|
375
|
-
"local",
|
|
376
|
-
`${LOCAL_MODEL} returns ${LOCAL_DIMENSIONS} dimensions; received dimensions ${String(dimensions)}`
|
|
377
|
-
);
|
|
378
|
-
}
|
|
379
|
-
return LOCAL_DIMENSIONS;
|
|
380
|
-
}
|
|
381
303
|
if (provider === "gemini") {
|
|
382
304
|
const effectiveDimensions = dimensions ?? GEMINI_DIMENSIONS;
|
|
383
305
|
if (!Number.isInteger(effectiveDimensions) || effectiveDimensions < GEMINI_MIN_DIMENSIONS || effectiveDimensions > GEMINI_MAX_DIMENSIONS) {
|
|
@@ -438,42 +360,6 @@ function createEmbeddingBatchResult(metadata, intent, embeddings) {
|
|
|
438
360
|
intent
|
|
439
361
|
});
|
|
440
362
|
}
|
|
441
|
-
class LocalEmbeddingProvider {
|
|
442
|
-
metadata;
|
|
443
|
-
#timeoutMs;
|
|
444
|
-
constructor(metadata, timeoutMs) {
|
|
445
|
-
this.metadata = metadata;
|
|
446
|
-
this.#timeoutMs = timeoutMs;
|
|
447
|
-
}
|
|
448
|
-
async embed(texts, options = {}) {
|
|
449
|
-
const intent = resolveIntent(options.intent);
|
|
450
|
-
if (texts.length === 0) {
|
|
451
|
-
return createEmbeddingBatchResult(this.metadata, intent, []);
|
|
452
|
-
}
|
|
453
|
-
assertBatchSize(this.metadata, texts.length);
|
|
454
|
-
const vectors = await withTimeout(
|
|
455
|
-
"local",
|
|
456
|
-
"model inference",
|
|
457
|
-
this.#timeoutMs,
|
|
458
|
-
async (signal) => {
|
|
459
|
-
const model = await getLocalEmbeddingModel(this.metadata.model, signal);
|
|
460
|
-
return embedSequentially("local", "model inference", texts, signal, async (text) => {
|
|
461
|
-
const output = await model(text, {
|
|
462
|
-
pooling: "mean",
|
|
463
|
-
normalize: true
|
|
464
|
-
});
|
|
465
|
-
return Array.from(output.data);
|
|
466
|
-
});
|
|
467
|
-
},
|
|
468
|
-
options.signal
|
|
469
|
-
);
|
|
470
|
-
return createEmbeddingBatchResult(
|
|
471
|
-
this.metadata,
|
|
472
|
-
intent,
|
|
473
|
-
validateEmbeddingBatch(vectors, texts.length, this.metadata.dimensions, "local")
|
|
474
|
-
);
|
|
475
|
-
}
|
|
476
|
-
}
|
|
477
363
|
class GeminiEmbeddingProvider {
|
|
478
364
|
metadata;
|
|
479
365
|
#apiKey;
|
|
@@ -626,13 +512,11 @@ class OpenAICompatibleEmbeddingProvider {
|
|
|
626
512
|
);
|
|
627
513
|
}
|
|
628
514
|
}
|
|
629
|
-
function createEmbeddingProvider(options
|
|
515
|
+
function createEmbeddingProvider(options) {
|
|
630
516
|
const provider = resolveProviderName(options.provider);
|
|
631
517
|
const metadata = getEmbeddingProviderMetadata(options);
|
|
632
518
|
const timeoutMs = getTimeoutMs(options.timeoutMs);
|
|
633
519
|
switch (provider) {
|
|
634
|
-
case "local":
|
|
635
|
-
return new LocalEmbeddingProvider(metadata, timeoutMs);
|
|
636
520
|
case "gemini": {
|
|
637
521
|
const key = getOptionalTrimmedCredential(
|
|
638
522
|
options.apiKey ?? getEnvironmentVariable("GEMINI_API_KEY")
|
|
@@ -731,7 +615,7 @@ function createEmbeddingProvider(options = {}) {
|
|
|
731
615
|
}
|
|
732
616
|
}
|
|
733
617
|
}
|
|
734
|
-
function getEmbeddingProviderMetadata(options
|
|
618
|
+
function getEmbeddingProviderMetadata(options) {
|
|
735
619
|
const provider = resolveProviderName(options.provider);
|
|
736
620
|
if (provider === "openai-compatible") {
|
|
737
621
|
normalizeOpenAICompatibleEmbeddingsUrl(getRequiredTrimmedString(options.baseUrl, "baseUrl"));
|
|
@@ -741,7 +625,7 @@ function getEmbeddingProviderMetadata(options = {}) {
|
|
|
741
625
|
}
|
|
742
626
|
return createProviderMetadata(provider, getEffectiveDimensions(provider, options.dimensions), options.model);
|
|
743
627
|
}
|
|
744
|
-
async function generateEmbeddings(texts, options
|
|
628
|
+
async function generateEmbeddings(texts, options) {
|
|
745
629
|
const maxLength = getPositiveInteger(options.maxLength ?? DEFAULT_MAX_LENGTH, "maxLength");
|
|
746
630
|
const intent = resolveIntent(options.intent);
|
|
747
631
|
if (texts.length === 0) {
|
|
@@ -754,7 +638,7 @@ async function generateEmbeddings(texts, options = {}) {
|
|
|
754
638
|
});
|
|
755
639
|
return result.embeddings;
|
|
756
640
|
}
|
|
757
|
-
async function generateEmbedding(text, options
|
|
641
|
+
async function generateEmbedding(text, options) {
|
|
758
642
|
const [embedding] = await generateEmbeddings([text], options);
|
|
759
643
|
if (!embedding) {
|
|
760
644
|
throw new Error("Embedding provider returned no embedding");
|
|
@@ -887,7 +771,7 @@ async function indexContent(options) {
|
|
|
887
771
|
const {
|
|
888
772
|
client,
|
|
889
773
|
contentPath,
|
|
890
|
-
embeddingOptions
|
|
774
|
+
embeddingOptions,
|
|
891
775
|
fileExtensions = [".md", ".markdown"],
|
|
892
776
|
exclude = ["node_modules", ".git", "dist", "build"],
|
|
893
777
|
tableName = "articles",
|
|
@@ -897,6 +781,9 @@ async function indexContent(options) {
|
|
|
897
781
|
} = options;
|
|
898
782
|
const database = resolveDatabase(client);
|
|
899
783
|
const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
|
|
784
|
+
if (embeddingOptions === void 0) {
|
|
785
|
+
throw new TypeError("embeddingOptions is required");
|
|
786
|
+
}
|
|
900
787
|
let files;
|
|
901
788
|
try {
|
|
902
789
|
files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
|
|
@@ -1176,7 +1063,7 @@ async function search(options) {
|
|
|
1176
1063
|
query,
|
|
1177
1064
|
limit = 10,
|
|
1178
1065
|
tableName = "articles",
|
|
1179
|
-
embeddingOptions
|
|
1066
|
+
embeddingOptions,
|
|
1180
1067
|
candidates,
|
|
1181
1068
|
exact = false
|
|
1182
1069
|
} = options;
|
|
@@ -1188,6 +1075,9 @@ async function search(options) {
|
|
|
1188
1075
|
"embedding index name"
|
|
1189
1076
|
);
|
|
1190
1077
|
const candidateCount = normalizeSearchCandidates(candidates, resultLimit);
|
|
1078
|
+
if (embeddingOptions === void 0) {
|
|
1079
|
+
throw new TypeError("embeddingOptions is required");
|
|
1080
|
+
}
|
|
1191
1081
|
const queryEmbedding = await generateEmbedding(query, {
|
|
1192
1082
|
...embeddingOptions,
|
|
1193
1083
|
intent: embeddingOptions.intent ?? "query"
|