libsql-search 0.5.0 → 0.7.0

package/README.md

@@ -20,10 +20,11 @@ Use it when you want:
 
 - Markdown indexing from local directories with frontmatter via `gray-matter`
 - libSQL/Turso storage and vector search
-- Embedding providers: local `Xenova/all-MiniLM-L6-v2`, Cloudflare Workers AI
-  `@cf/baai/bge-m3`, Mistral `mistral-embed`, Google Gemini
+- Embedding providers: local Hugging Face `Xenova/all-MiniLM-L6-v2`,
+  Cloudflare Workers AI `@cf/baai/bge-m3`, Mistral `mistral-embed`, Google Gemini
   `gemini-embedding-2`, and OpenAI `text-embedding-3-small` /
-  `text-embedding-3-large`
+  `text-embedding-3-large`; self-hosted OpenAI-compatible endpoints are also
+  available for TEI and similar trusted deployments
 - npm distribution plus JSR publishing
 
 ## Install
@@ -65,14 +66,13 @@ const client = createClient({
   authToken: "your-auth-token",
 });
 
-await createTable(client, "articles", 768);
+await createTable(client);
 
 await indexContent({
   client,
   contentPath: "./content",
   embeddingOptions: {
     provider: "local",
-    dimensions: 768,
   },
 });
 
@@ -82,7 +82,6 @@ const results = await search({
   limit: 5,
   embeddingOptions: {
     provider: "local",
-    dimensions: 768,
   },
 });
 
@@ -98,9 +97,9 @@ Important behavior:
 - Call `createTable()` before indexing or searching.
 - Keep dimensions aligned across table creation, indexing, and search queries.
 - `indexContent()` clears existing rows before rebuilding the index.
-- `local` is the default offline provider; Cloudflare is the recommended hosted
-  option. Cloudflare and Mistral use 1024 dimensions. Gemini defaults to 3072
-  dimensions and supports 128-3072.
+- `local` is the default offline provider and uses 384 dimensions. Cloudflare is
+  the recommended hosted option. Cloudflare and Mistral use 1024 dimensions.
+  Gemini defaults to 3072 dimensions and supports 128-3072.
 
 ## Core API
 

package/dist/index.cjs

@@ -4,7 +4,9 @@ var promises = require('fs/promises');
 var path = require('path');
 var matter = require('gray-matter');
 
-const DEFAULT_DIMENSIONS = 768;
+const OPENAI_DEFAULT_DIMENSIONS = 768;
+const OPENAI_COMPATIBLE_DEFAULT_BATCH_SIZE = 32;
+const LOCAL_DIMENSIONS = 384;
 const DEFAULT_MAX_LENGTH = 8e3;
 const DEFAULT_TIMEOUT_MS = 3e4;
 const LOCAL_MODEL = "Xenova/all-MiniLM-L6-v2";
@@ -33,17 +35,66 @@ function getEnvironmentVariable(name) {
     return void 0;
   }
 }
-async function getLocalEmbeddingModel(modelName) {
+function deletePendingLocalModelCache(modelName, entry) {
+  if (!entry.settled && entry.waiters === 0 && localModelCacheByModel.get(modelName) === entry) {
+    localModelCacheByModel.delete(modelName);
+  }
+}
+async function getLocalEmbeddingModel(modelName, signal) {
   const cached = localModelCacheByModel.get(modelName);
-  if (cached?.model) {
-    return cached.model;
+  if (cached) {
+    cached.waiters++;
+    try {
+      return await waitForLocalEmbeddingModel(modelName, cached, signal);
+    } finally {
+      cached.waiters--;
+      deletePendingLocalModelCache(modelName, cached);
+    }
+  }
+  const modelPromise = (async () => {
+    console.log(`Loading local embedding model (${modelName})...`);
+    const { pipeline } = await import('@huggingface/transformers');
+    const model = await pipeline("feature-extraction", modelName);
+    console.log("Local model loaded successfully");
+    return model;
+  })();
+  const entry = {
+    promise: modelPromise,
+    settled: false,
+    waiters: 1
+  };
+  localModelCacheByModel.set(modelName, entry);
+  modelPromise.then(() => {
+    entry.settled = true;
+  }).catch(() => {
+    localModelCacheByModel.delete(modelName);
+  });
+  try {
+    return await waitForLocalEmbeddingModel(modelName, entry, signal);
+  } finally {
+    entry.waiters--;
+    deletePendingLocalModelCache(modelName, entry);
+  }
+}
+async function waitForLocalEmbeddingModel(modelName, entry, signal) {
+  if (signal.aborted) {
+    deletePendingLocalModelCache(modelName, entry);
+    throw providerError("local", "model inference was aborted");
+  }
+  let rejectAbort = () => {
+  };
+  const abortPromise = new Promise((_resolve, reject) => {
+    rejectAbort = reject;
+  });
+  const onAbort = () => {
+    rejectAbort(providerError("local", "model inference was aborted"));
+  };
+  signal.addEventListener("abort", onAbort, { once: true });
+  try {
+    return await Promise.race([entry.promise, abortPromise]);
+  } finally {
+    signal.removeEventListener("abort", onAbort);
   }
-  console.log(`Loading local embedding model (${modelName})...`);
-  const { pipeline } = await import('@xenova/transformers');
-  const model = await pipeline("feature-extraction", modelName);
-  localModelCacheByModel.set(modelName, { model });
-  console.log("Local model loaded successfully");
-  return model;
 }
 function getPositiveInteger(value, optionName) {
   if (!Number.isInteger(value) || value <= 0) {
@@ -61,6 +112,7 @@ function resolveProviderName(provider) {
     case "openai":
     case "mistral":
     case "cloudflare":
+    case "openai-compatible":
       return provider ?? "local";
     default:
       throw new Error(`Unknown embedding provider: ${String(provider)}`);
@@ -85,6 +137,13 @@ function getOptionalTrimmedCredential(value) {
   const trimmed = value?.trim();
   return trimmed === void 0 || trimmed.length === 0 ? void 0 : trimmed;
 }
+function getRequiredTrimmedString(value, optionName) {
+  const trimmed = getOptionalTrimmedCredential(value);
+  if (!trimmed) {
+    throw new Error(`Invalid ${optionName}: expected a non-empty string`);
+  }
+  return trimmed;
+}
 function hasOwnProperty(value, property) {
   return Object.prototype.hasOwnProperty.call(value, property);
 }
@@ -245,7 +304,27 @@ async function embedSequentially(provider, operation, texts, signal, embedOne) {
   }
   return results;
 }
-function createProviderMetadata(provider, dimensions) {
+function normalizeOpenAICompatibleEmbeddingsUrl(baseUrl) {
+  let url;
+  try {
+    url = new URL(baseUrl);
+  } catch {
+    throw new Error("Invalid baseUrl: expected an absolute http or https URL");
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new Error("Invalid baseUrl: expected an absolute http or https URL");
+  }
+  if (url.username || url.password) {
+    throw new Error("Invalid baseUrl: URL credentials are not allowed");
+  }
+  if (url.search || url.hash) {
+    throw new Error("Invalid baseUrl: query strings and fragments are not allowed");
+  }
+  const path = url.pathname.replace(/\/+$/, "");
+  url.pathname = path.endsWith("/embeddings") ? path : `${path}/embeddings`;
+  return url.toString();
+}
+function createProviderMetadata(provider, dimensions, model) {
   switch (provider) {
     case "local":
       return Object.freeze({
@@ -282,9 +361,25 @@ function createProviderMetadata(provider, dimensions) {
         dimensions: CLOUDFLARE_DIMENSIONS,
         batch: Object.freeze({ mode: "native" })
       });
+    case "openai-compatible":
+      return Object.freeze({
+        name: "openai-compatible",
+        model: getRequiredTrimmedString(model, "model"),
+        dimensions,
+        batch: Object.freeze({ mode: "native" })
+      });
   }
 }
 function getEffectiveDimensions(provider, dimensions) {
+  if (provider === "local") {
+    if (dimensions !== void 0 && dimensions !== LOCAL_DIMENSIONS) {
+      throw providerError(
+        "local",
+        `${LOCAL_MODEL} returns ${LOCAL_DIMENSIONS} dimensions; received dimensions ${String(dimensions)}`
+      );
+    }
+    return LOCAL_DIMENSIONS;
+  }
   if (provider === "gemini") {
     const effectiveDimensions = dimensions ?? GEMINI_DIMENSIONS;
     if (!Number.isInteger(effectiveDimensions) || effectiveDimensions < GEMINI_MIN_DIMENSIONS || effectiveDimensions > GEMINI_MAX_DIMENSIONS) {
@@ -313,7 +408,13 @@ function getEffectiveDimensions(provider, dimensions) {
     }
     return CLOUDFLARE_DIMENSIONS;
   }
-  return getPositiveInteger(dimensions ?? DEFAULT_DIMENSIONS, "dimensions");
+  if (provider === "openai-compatible") {
+    if (dimensions === void 0) {
+      throw new Error("Invalid dimensions: expected a positive integer");
+    }
+    return getPositiveInteger(dimensions, "dimensions");
+  }
+  return getPositiveInteger(dimensions ?? OPENAI_DEFAULT_DIMENSIONS, "dimensions");
 }
 function assertBatchSize(metadata, count) {
   if (metadata.batch.maxSize !== void 0 && count > metadata.batch.maxSize) {
@@ -323,6 +424,13 @@ function assertBatchSize(metadata, count) {
     );
   }
 }
+function chunkTexts(texts, batchSize) {
+  const chunks = [];
+  for (let index = 0; index < texts.length; index += batchSize) {
+    chunks.push(texts.slice(index, index + batchSize));
+  }
+  return chunks;
+}
 function createEmbeddingBatchResult(metadata, intent, embeddings) {
   return Object.freeze({
     embeddings,
@@ -350,14 +458,13 @@ class LocalEmbeddingProvider {
       "model inference",
       this.#timeoutMs,
       async (signal) => {
-        const model = await getLocalEmbeddingModel(this.metadata.model);
+        const model = await getLocalEmbeddingModel(this.metadata.model, signal);
         return embedSequentially("local", "model inference", texts, signal, async (text) => {
           const output = await model(text, {
             pooling: "mean",
             normalize: true
           });
-          const embedding = Array.from(output.data);
-          return padEmbedding(embedding, this.metadata.dimensions);
+          return Array.from(output.data);
         });
       },
       options.signal
@@ -424,18 +531,23 @@ async function requestOpenAICompatibleEmbeddings(request) {
   if (request.encodingFormat !== void 0) {
     body.encoding_format = request.encodingFormat;
   }
+  const headers = {
+    "Content-Type": "application/json"
+  };
+  if (request.apiKey !== void 0) {
+    headers.Authorization = `Bearer ${request.apiKey}`;
+  }
   const response = await fetch(request.endpoint, {
     method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      "Authorization": `Bearer ${request.apiKey}`
-    },
+    headers,
     signal: request.signal,
-    body: JSON.stringify(body)
+    body: JSON.stringify(body),
+    ...request.redirect ? { redirect: request.redirect } : {}
   });
   if (!response.ok) {
-    const requestId = getSafeResponseHeader(response, "x-request-id");
-    const context = requestId ? ` status ${response.status}, request ${requestId}` : ` status ${response.status}`;
+    const statusHeader = request.statusHeader ?? { name: "x-request-id", label: "request" };
+    const statusHeaderValue = getSafeResponseHeader(response, statusHeader.name);
+    const context = statusHeaderValue ? ` status ${response.status}, ${statusHeader.label} ${statusHeaderValue}` : ` status ${response.status}`;
     throw new Error(context);
   }
   const data = await response.json();
@@ -459,92 +571,13 @@ async function requestOpenAICompatibleEmbeddings(request) {
     return parsed;
   });
 }
-class OpenAIEmbeddingProvider {
-  metadata;
-  #apiKey;
-  #timeoutMs;
-  constructor(metadata, apiKey, timeoutMs) {
-    this.metadata = metadata;
-    this.#apiKey = apiKey;
-    this.#timeoutMs = timeoutMs;
-  }
-  async embed(texts, options = {}) {
-    const intent = resolveIntent(options.intent);
-    if (texts.length === 0) {
-      return createEmbeddingBatchResult(this.metadata, intent, []);
-    }
-    assertBatchSize(this.metadata, texts.length);
-    const items = await withTimeout(
-      "openai",
-      "API request",
-      this.#timeoutMs,
-      async (signal) => requestOpenAICompatibleEmbeddings({
-        responseLabel: "OpenAI",
-        endpoint: OPENAI_EMBEDDINGS_URL,
-        apiKey: this.#apiKey,
-        model: this.metadata.model,
-        texts,
-        signal,
-        dimensions: this.metadata.dimensions
-      }),
-      options.signal,
-      [this.#apiKey]
-    );
-    return createEmbeddingBatchResult(
-      this.metadata,
-      intent,
-      validateEmbeddingBatch(items, texts.length, this.metadata.dimensions, "openai")
-    );
-  }
-}
-class MistralEmbeddingProvider {
-  metadata;
-  #apiKey;
-  #timeoutMs;
-  constructor(metadata, apiKey, timeoutMs) {
-    this.metadata = metadata;
-    this.#apiKey = apiKey;
-    this.#timeoutMs = timeoutMs;
-  }
-  async embed(texts, options = {}) {
-    const intent = resolveIntent(options.intent);
-    if (texts.length === 0) {
-      return createEmbeddingBatchResult(this.metadata, intent, []);
-    }
-    assertBatchSize(this.metadata, texts.length);
-    const items = await withTimeout(
-      "mistral",
-      "API request",
-      this.#timeoutMs,
-      async (signal) => requestOpenAICompatibleEmbeddings({
-        responseLabel: "Mistral",
-        endpoint: MISTRAL_EMBEDDINGS_URL,
-        apiKey: this.#apiKey,
-        model: this.metadata.model,
-        texts,
-        signal,
-        encodingFormat: "float",
-        requireIndex: true
-      }),
-      options.signal,
-      [this.#apiKey]
-    );
-    return createEmbeddingBatchResult(
-      this.metadata,
-      intent,
-      validateEmbeddingBatch(items, texts.length, this.metadata.dimensions, "mistral")
-    );
-  }
-}
-class CloudflareEmbeddingProvider {
+class OpenAICompatibleEmbeddingProvider {
   metadata;
-  #accountId;
-  #apiToken;
   #timeoutMs;
-  constructor(metadata, accountId, apiToken, timeoutMs) {
+  #profile;
+  constructor(metadata, profile, timeoutMs) {
     this.metadata = metadata;
-    this.#accountId = accountId;
-    this.#apiToken = apiToken;
+    this.#profile = profile;
     this.#timeoutMs = timeoutMs;
   }
   async embed(texts, options = {}) {
@@ -553,53 +586,45 @@ class CloudflareEmbeddingProvider {
       return createEmbeddingBatchResult(this.metadata, intent, []);
     }
     assertBatchSize(this.metadata, texts.length);
-    const endpoint = createCloudflareEmbeddingsUrl(this.#accountId);
     const items = await withTimeout(
-      "cloudflare",
+      this.#profile.provider,
       "API request",
       this.#timeoutMs,
       async (signal) => {
-        const response = await fetch(endpoint, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            "Authorization": `Bearer ${this.#apiToken}`
-          },
-          signal,
-          body: JSON.stringify({
-            model: this.metadata.model,
-            input: texts
-          })
-        });
-        if (!response.ok) {
-          const cfRay = getSafeResponseHeader(response, "cf-ray");
-          const context = cfRay ? ` status ${response.status}, cf-ray ${cfRay}` : ` status ${response.status}`;
-          throw new Error(context);
-        }
-        const data = await response.json();
-        if (!Array.isArray(data.data)) {
-          throw new Error("Cloudflare response did not include a data array");
+        const batches = this.#profile.batchSize === void 0 ? [texts] : chunkTexts(texts, this.#profile.batchSize);
+        const results = [];
+        for (const batch of batches) {
+          throwIfAborted(this.#profile.provider, "API request", signal);
+          const batchItems = await requestOpenAICompatibleEmbeddings({
+            responseLabel: this.#profile.responseLabel,
+            endpoint: this.#profile.endpoint,
+            ...this.#profile.apiKey !== void 0 ? { apiKey: this.#profile.apiKey } : {},
+            model: this.#profile.model,
+            texts: batch,
+            signal,
+            ...this.#profile.includeDimensions ? { dimensions: this.#profile.dimensions } : {},
+            ...this.#profile.encodingFormat ? { encodingFormat: this.#profile.encodingFormat } : {},
+            ...this.#profile.requireIndex !== void 0 ? { requireIndex: this.#profile.requireIndex } : {},
+            ...this.#profile.redirect ? { redirect: this.#profile.redirect } : {},
+            ...this.#profile.statusHeader ? { statusHeader: this.#profile.statusHeader } : {}
+          });
+          results.push(...validateEmbeddingBatch(
+            batchItems,
+            batch.length,
+            this.metadata.dimensions,
+            this.#profile.provider
+          ));
+          throwIfAborted(this.#profile.provider, "API request", signal);
         }
-        return data.data.map((item) => {
-          const typed = item;
-          const embedding = typed.embedding;
-          if (!Array.isArray(embedding)) {
-            throw new Error("Cloudflare response item did not include an embedding array");
-          }
-          const parsed = { embedding };
-          if (hasOwnProperty(typed, "index")) {
-            parsed.index = typed.index;
-          }
-          return parsed;
-        });
+        return results;
       },
       options.signal,
-      [this.#apiToken, this.#accountId, encodeURIComponent(this.#accountId)]
+      this.#profile.exactSecrets ?? []
     );
     return createEmbeddingBatchResult(
       this.metadata,
       intent,
-      validateEmbeddingBatch(items, texts.length, this.metadata.dimensions, "cloudflare")
+      validateEmbeddingBatch(items, texts.length, this.metadata.dimensions, this.#profile.provider)
     );
   }
 }
@@ -624,7 +649,16 @@ function createEmbeddingProvider(options = {}) {
       if (!key) {
         throw new Error("OPENAI_API_KEY is required for OpenAI embeddings");
       }
-      return new OpenAIEmbeddingProvider(metadata, key, timeoutMs);
+      return new OpenAICompatibleEmbeddingProvider(metadata, {
+        provider: "openai",
+        responseLabel: "OpenAI",
+        endpoint: OPENAI_EMBEDDINGS_URL,
+        apiKey: key,
+        model: metadata.model,
+        dimensions: metadata.dimensions,
+        includeDimensions: true,
+        exactSecrets: [key]
+      }, timeoutMs);
     }
     case "mistral": {
       const key = getOptionalTrimmedCredential(
@@ -633,7 +667,17 @@ function createEmbeddingProvider(options = {}) {
       if (!key) {
         throw new Error("MISTRAL_API_KEY is required for Mistral embeddings");
       }
-      return new MistralEmbeddingProvider(metadata, key, timeoutMs);
+      return new OpenAICompatibleEmbeddingProvider(metadata, {
+        provider: "mistral",
+        responseLabel: "Mistral",
+        endpoint: MISTRAL_EMBEDDINGS_URL,
+        apiKey: key,
+        model: metadata.model,
+        dimensions: metadata.dimensions,
+        encodingFormat: "float",
+        requireIndex: true,
+        exactSecrets: [key]
+      }, timeoutMs);
     }
     case "cloudflare": {
       const accountId = getOptionalTrimmedCredential(
@@ -648,13 +692,56 @@ function createEmbeddingProvider(options = {}) {
       if (!apiToken) {
         throw new Error("CLOUDFLARE_API_TOKEN is required for Cloudflare embeddings");
       }
-      return new CloudflareEmbeddingProvider(metadata, accountId, apiToken, timeoutMs);
+      const endpoint = createCloudflareEmbeddingsUrl(accountId);
+      return new OpenAICompatibleEmbeddingProvider(metadata, {
+        provider: "cloudflare",
+        responseLabel: "Cloudflare",
+        endpoint,
+        apiKey: apiToken,
+        model: metadata.model,
+        dimensions: metadata.dimensions,
+        statusHeader: { name: "cf-ray", label: "cf-ray" },
+        exactSecrets: [apiToken, accountId, encodeURIComponent(accountId), endpoint]
+      }, timeoutMs);
+    }
+    case "openai-compatible": {
+      const baseUrl = getRequiredTrimmedString(options.baseUrl, "baseUrl");
+      const endpoint = normalizeOpenAICompatibleEmbeddingsUrl(baseUrl);
+      const apiKey = getOptionalTrimmedCredential(options.apiKey);
+      const batchSize = getPositiveInteger(
+        options.batchSize ?? OPENAI_COMPATIBLE_DEFAULT_BATCH_SIZE,
+        "batchSize"
+      );
+      return new OpenAICompatibleEmbeddingProvider(metadata, {
+        provider: "openai-compatible",
+        responseLabel: "OpenAI-compatible",
+        endpoint,
+        ...apiKey ? { apiKey } : {},
+        model: metadata.model,
+        dimensions: metadata.dimensions,
+        includeDimensions: true,
+        encodingFormat: "float",
+        requireIndex: true,
+        redirect: "error",
+        batchSize,
+        exactSecrets: [
+          ...apiKey ? [apiKey] : [],
+          baseUrl,
+          endpoint
+        ]
+      }, timeoutMs);
     }
   }
 }
 function getEmbeddingProviderMetadata(options = {}) {
   const provider = resolveProviderName(options.provider);
-  return createProviderMetadata(provider, getEffectiveDimensions(provider, options.dimensions));
+  if (provider === "openai-compatible") {
+    normalizeOpenAICompatibleEmbeddingsUrl(getRequiredTrimmedString(options.baseUrl, "baseUrl"));
+    if (options.batchSize !== void 0) {
+      getPositiveInteger(options.batchSize, "batchSize");
+    }
+  }
+  return createProviderMetadata(provider, getEffectiveDimensions(provider, options.dimensions), options.model);
 }
 async function generateEmbeddings(texts, options = {}) {
   const maxLength = getPositiveInteger(options.maxLength ?? DEFAULT_MAX_LENGTH, "maxLength");
@@ -824,7 +911,7 @@ async function insertDocument(client, document, quotedTableName) {
     ]
   });
 }
-async function createTable(client, tableName = "articles", dimensions = 768) {
+async function createTable(client, tableName = "articles", dimensions = 384) {
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const vectorDimensions = normalizeVectorDimensions(dimensions);
   const quotedEmbeddingIndexName = quoteSqlIdentifier(`${tableName}_embedding_idx`, "embedding index name");

package/dist/index.d.ts

@@ -1,6 +1,11 @@
 import { Client } from '@libsql/client';
 
-type EmbeddingProvider = 'local' | 'gemini' | 'openai' | 'mistral' | 'cloudflare';
+/**
+ * Multi-provider embedding generation
+ * Supports local Hugging Face Transformers, Gemini, OpenAI, Mistral,
+ * Cloudflare Workers AI, and custom OpenAI-compatible endpoints
+ */
+type EmbeddingProvider = 'local' | 'gemini' | 'openai' | 'mistral' | 'cloudflare' | 'openai-compatible';
 type EmbeddingIntent = 'document' | 'query';
 type EmbeddingBatchMode = 'native' | 'sequential';
 interface EmbeddingBatchBehavior {
@@ -33,6 +38,9 @@ interface EmbeddingOptions {
     apiKey?: string;
     accountId?: string;
     apiToken?: string;
+    baseUrl?: string;
+    model?: string;
+    batchSize?: number;
     dimensions?: number;
     maxLength?: number;
     intent?: EmbeddingIntent;