libsql-search 0.6.0 → 0.7.1

package/README.md

@@ -5,26 +5,13 @@
 [![CI](https://github.com/llbbl/libsql-search/actions/workflows/ci.yml/badge.svg)](https://github.com/llbbl/libsql-search/actions/workflows/ci.yml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
-`libsql-search` adds semantic search to Markdown-backed sites using libSQL/Turso.
-It indexes frontmatter and content from files on disk, stores vectors in your
-database, and lets you query by meaning instead of exact keywords.
+`libsql-search` adds semantic search to Markdown-backed sites with a small TypeScript API. It indexes frontmatter and content from files on disk, stores vectors in libSQL/Turso, and lets you query by meaning instead of exact keywords.
 
 Use it when you want:
 
-- a small TypeScript library instead of a hosted search product
-- one search index shared across static-site builds and app routes
-- local or hosted embeddings behind the same indexing/search API
-- direct control over table names, dimensions, content shape, and deployment
-
-## What It Supports
-
-- Markdown indexing from local directories with frontmatter via `gray-matter`
-- libSQL/Turso storage and vector search
-- Embedding providers: local Hugging Face `Xenova/all-MiniLM-L6-v2`,
-  Cloudflare Workers AI `@cf/baai/bge-m3`, Mistral `mistral-embed`, Google Gemini
-  `gemini-embedding-2`, and OpenAI `text-embedding-3-small` /
-  `text-embedding-3-large`
-- npm distribution plus JSR publishing
+- one indexing/search API across local and hosted embedding providers
+- direct control over vector dimensions, table names, and deployment shape
+- a lightweight library instead of a hosted search product
 
 ## Install
 
@@ -43,18 +30,10 @@ deno add jsr:@logan/libsql-search npm:@libsql/client
 ```
 
 For npm usage, the package requires Node `>=22.12.0`.
-Node examples in this README import from `libsql-search` and `@libsql/client`.
-In Deno, after `deno add`, import from `@logan/libsql-search` and
-`@libsql/client`.
 
 ## Quick Start
 
-The shortest working flow is:
-
-1. create a libSQL client
-2. create the search table
-3. index a Markdown directory
-4. query it with the same embedding provider and dimensions
+This example uses the default local provider. Local embeddings run in-process after the initial model download and cache warmup; they are not automatically air-gapped.
 
 ```ts
 import { createClient } from "@libsql/client";
@@ -65,11 +44,12 @@ const client = createClient({
   authToken: "your-auth-token",
 });
 
-await createTable(client);
+await createTable(client, "articles_local_384", 384);
 
 await indexContent({
   client,
   contentPath: "./content",
+  tableName: "articles_local_384",
   embeddingOptions: {
     provider: "local",
   },
@@ -78,6 +58,7 @@ await indexContent({
 const results = await search({
   client,
   query: "how do I deploy my docs site",
+  tableName: "articles_local_384",
   limit: 5,
   embeddingOptions: {
     provider: "local",
@@ -94,33 +75,31 @@ console.log(results.map((result) => ({
 Important behavior:
 
 - Call `createTable()` before indexing or searching.
-- Keep dimensions aligned across table creation, indexing, and search queries.
-- `indexContent()` clears existing rows before rebuilding the index.
-- `local` is the default offline provider and uses 384 dimensions. Cloudflare is
-  the recommended hosted option. Cloudflare and Mistral use 1024 dimensions.
-  Gemini defaults to 3072 dimensions and supports 128-3072.
-
-## Core API
-
-- `createTable(client, tableName?, dimensions?)`
-- `indexContent(options)`
-- `search(options)`
-- `getAllArticles(client, tableName?)`
-- `getArticleBySlug(client, slug, tableName?)`
-- `getArticlesByFolder(client, folder, tableName?)`
-- `getFolders(client, tableName?)`
-- `generateEmbedding(text, options?)`
-- `prepareTextForEmbedding(fields)`
+- Keep table width, provider, and dimensions aligned across create/index/query.
+- `indexContent()` clears existing rows before rebuilding and is not transactional.
+- Hosted providers send indexed and queried text to external services and may incur provider charges.
+
+## Providers
+
+Built-in providers:
+
+- `local` with `Xenova/all-MiniLM-L6-v2` at 384 dimensions
+- `cloudflare` with `@cf/baai/bge-m3` at 1024 dimensions
+- `mistral` with `mistral-embed` at 1024 dimensions
+- `gemini` with `gemini-embedding-2` at 128-3072 dimensions, default 3072
+- `openai` with `text-embedding-3-small` or `text-embedding-3-large`, default 768
+- `openai-compatible` for trusted OpenAI-compatible endpoints such as TEI
 
 ## Docs
 
-- [Docs index](./docs/README.md)
-- [Provider guide](./docs/PROVIDERS.md)
+- [Documentation index](./docs/README.md)
+- [Provider selection and configuration](./docs/PROVIDERS.md)
 - [API reference](./docs/API.md)
 - [Integration examples](./docs/INTEGRATIONS.md)
+- [Migration and reindexing guide](./docs/MIGRATIONS.md)
+- [Testing guidance](./docs/TESTING.md)
 - [Indexing and operations](./docs/INDEXING.md)
 - [Troubleshooting](./docs/TROUBLESHOOTING.md)
-- [Release workflow](./docs/RELEASING.md)
 
 ## License
 

package/dist/index.cjs

@@ -5,6 +5,7 @@ var path = require('path');
 var matter = require('gray-matter');
 
 const OPENAI_DEFAULT_DIMENSIONS = 768;
+const OPENAI_COMPATIBLE_DEFAULT_BATCH_SIZE = 32;
 const LOCAL_DIMENSIONS = 384;
 const DEFAULT_MAX_LENGTH = 8e3;
 const DEFAULT_TIMEOUT_MS = 3e4;
@@ -111,6 +112,7 @@ function resolveProviderName(provider) {
     case "openai":
     case "mistral":
     case "cloudflare":
+    case "openai-compatible":
       return provider ?? "local";
     default:
       throw new Error(`Unknown embedding provider: ${String(provider)}`);
@@ -135,6 +137,13 @@ function getOptionalTrimmedCredential(value) {
   const trimmed = value?.trim();
   return trimmed === void 0 || trimmed.length === 0 ? void 0 : trimmed;
 }
+function getRequiredTrimmedString(value, optionName) {
+  const trimmed = getOptionalTrimmedCredential(value);
+  if (!trimmed) {
+    throw new Error(`Invalid ${optionName}: expected a non-empty string`);
+  }
+  return trimmed;
+}
 function hasOwnProperty(value, property) {
   return Object.prototype.hasOwnProperty.call(value, property);
 }
@@ -295,7 +304,27 @@ async function embedSequentially(provider, operation, texts, signal, embedOne) {
   }
   return results;
 }
-function createProviderMetadata(provider, dimensions) {
+function normalizeOpenAICompatibleEmbeddingsUrl(baseUrl) {
+  let url;
+  try {
+    url = new URL(baseUrl);
+  } catch {
+    throw new Error("Invalid baseUrl: expected an absolute http or https URL");
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new Error("Invalid baseUrl: expected an absolute http or https URL");
+  }
+  if (url.username || url.password) {
+    throw new Error("Invalid baseUrl: URL credentials are not allowed");
+  }
+  if (url.search || url.hash) {
+    throw new Error("Invalid baseUrl: query strings and fragments are not allowed");
+  }
+  const path = url.pathname.replace(/\/+$/, "");
+  url.pathname = path.endsWith("/embeddings") ? path : `${path}/embeddings`;
+  return url.toString();
+}
+function createProviderMetadata(provider, dimensions, model) {
   switch (provider) {
     case "local":
       return Object.freeze({
@@ -332,6 +361,13 @@ function createProviderMetadata(provider, dimensions) {
         dimensions: CLOUDFLARE_DIMENSIONS,
         batch: Object.freeze({ mode: "native" })
       });
+    case "openai-compatible":
+      return Object.freeze({
+        name: "openai-compatible",
+        model: getRequiredTrimmedString(model, "model"),
+        dimensions,
+        batch: Object.freeze({ mode: "native" })
+      });
   }
 }
 function getEffectiveDimensions(provider, dimensions) {
@@ -372,6 +408,12 @@ function getEffectiveDimensions(provider, dimensions) {
     }
     return CLOUDFLARE_DIMENSIONS;
   }
+  if (provider === "openai-compatible") {
+    if (dimensions === void 0) {
+      throw new Error("Invalid dimensions: expected a positive integer");
+    }
+    return getPositiveInteger(dimensions, "dimensions");
+  }
   return getPositiveInteger(dimensions ?? OPENAI_DEFAULT_DIMENSIONS, "dimensions");
 }
 function assertBatchSize(metadata, count) {
@@ -382,6 +424,13 @@ function assertBatchSize(metadata, count) {
     );
   }
 }
+function chunkTexts(texts, batchSize) {
+  const chunks = [];
+  for (let index = 0; index < texts.length; index += batchSize) {
+    chunks.push(texts.slice(index, index + batchSize));
+  }
+  return chunks;
+}
 function createEmbeddingBatchResult(metadata, intent, embeddings) {
   return Object.freeze({
     embeddings,
@@ -482,18 +531,23 @@ async function requestOpenAICompatibleEmbeddings(request) {
   if (request.encodingFormat !== void 0) {
     body.encoding_format = request.encodingFormat;
   }
+  const headers = {
+    "Content-Type": "application/json"
+  };
+  if (request.apiKey !== void 0) {
+    headers.Authorization = `Bearer ${request.apiKey}`;
+  }
   const response = await fetch(request.endpoint, {
     method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      "Authorization": `Bearer ${request.apiKey}`
-    },
+    headers,
     signal: request.signal,
-    body: JSON.stringify(body)
+    body: JSON.stringify(body),
+    ...request.redirect ? { redirect: request.redirect } : {}
   });
   if (!response.ok) {
-    const requestId = getSafeResponseHeader(response, "x-request-id");
-    const context = requestId ? ` status ${response.status}, request ${requestId}` : ` status ${response.status}`;
+    const statusHeader = request.statusHeader ?? { name: "x-request-id", label: "request" };
+    const statusHeaderValue = getSafeResponseHeader(response, statusHeader.name);
+    const context = statusHeaderValue ? ` status ${response.status}, ${statusHeader.label} ${statusHeaderValue}` : ` status ${response.status}`;
     throw new Error(context);
   }
   const data = await response.json();
@@ -517,13 +571,13 @@ async function requestOpenAICompatibleEmbeddings(request) {
     return parsed;
   });
 }
-class OpenAIEmbeddingProvider {
+class OpenAICompatibleEmbeddingProvider {
   metadata;
-  #apiKey;
   #timeoutMs;
-  constructor(metadata, apiKey, timeoutMs) {
+  #profile;
+  constructor(metadata, profile, timeoutMs) {
     this.metadata = metadata;
-    this.#apiKey = apiKey;
+    this.#profile = profile;
     this.#timeoutMs = timeoutMs;
   }
   async embed(texts, options = {}) {
@@ -533,131 +587,44 @@ class OpenAIEmbeddingProvider {
     }
     assertBatchSize(this.metadata, texts.length);
     const items = await withTimeout(
-      "openai",
-      "API request",
-      this.#timeoutMs,
-      async (signal) => requestOpenAICompatibleEmbeddings({
-        responseLabel: "OpenAI",
-        endpoint: OPENAI_EMBEDDINGS_URL,
-        apiKey: this.#apiKey,
-        model: this.metadata.model,
-        texts,
-        signal,
-        dimensions: this.metadata.dimensions
-      }),
-      options.signal,
-      [this.#apiKey]
-    );
-    return createEmbeddingBatchResult(
-      this.metadata,
-      intent,
-      validateEmbeddingBatch(items, texts.length, this.metadata.dimensions, "openai")
-    );
-  }
-}
-class MistralEmbeddingProvider {
-  metadata;
-  #apiKey;
-  #timeoutMs;
-  constructor(metadata, apiKey, timeoutMs) {
-    this.metadata = metadata;
-    this.#apiKey = apiKey;
-    this.#timeoutMs = timeoutMs;
-  }
-  async embed(texts, options = {}) {
-    const intent = resolveIntent(options.intent);
-    if (texts.length === 0) {
-      return createEmbeddingBatchResult(this.metadata, intent, []);
-    }
-    assertBatchSize(this.metadata, texts.length);
-    const items = await withTimeout(
-      "mistral",
-      "API request",
-      this.#timeoutMs,
-      async (signal) => requestOpenAICompatibleEmbeddings({
-        responseLabel: "Mistral",
-        endpoint: MISTRAL_EMBEDDINGS_URL,
-        apiKey: this.#apiKey,
-        model: this.metadata.model,
-        texts,
-        signal,
-        encodingFormat: "float",
-        requireIndex: true
-      }),
-      options.signal,
-      [this.#apiKey]
-    );
-    return createEmbeddingBatchResult(
-      this.metadata,
-      intent,
-      validateEmbeddingBatch(items, texts.length, this.metadata.dimensions, "mistral")
-    );
-  }
-}
-class CloudflareEmbeddingProvider {
-  metadata;
-  #accountId;
-  #apiToken;
-  #timeoutMs;
-  constructor(metadata, accountId, apiToken, timeoutMs) {
-    this.metadata = metadata;
-    this.#accountId = accountId;
-    this.#apiToken = apiToken;
-    this.#timeoutMs = timeoutMs;
-  }
-  async embed(texts, options = {}) {
-    const intent = resolveIntent(options.intent);
-    if (texts.length === 0) {
-      return createEmbeddingBatchResult(this.metadata, intent, []);
-    }
-    assertBatchSize(this.metadata, texts.length);
-    const endpoint = createCloudflareEmbeddingsUrl(this.#accountId);
-    const items = await withTimeout(
-      "cloudflare",
+      this.#profile.provider,
       "API request",
       this.#timeoutMs,
       async (signal) => {
-        const response = await fetch(endpoint, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            "Authorization": `Bearer ${this.#apiToken}`
-          },
-          signal,
-          body: JSON.stringify({
-            model: this.metadata.model,
-            input: texts
-          })
-        });
-        if (!response.ok) {
-          const cfRay = getSafeResponseHeader(response, "cf-ray");
-          const context = cfRay ? ` status ${response.status}, cf-ray ${cfRay}` : ` status ${response.status}`;
-          throw new Error(context);
-        }
-        const data = await response.json();
-        if (!Array.isArray(data.data)) {
-          throw new Error("Cloudflare response did not include a data array");
+        const batches = this.#profile.batchSize === void 0 ? [texts] : chunkTexts(texts, this.#profile.batchSize);
+        const results = [];
+        for (const batch of batches) {
+          throwIfAborted(this.#profile.provider, "API request", signal);
+          const batchItems = await requestOpenAICompatibleEmbeddings({
+            responseLabel: this.#profile.responseLabel,
+            endpoint: this.#profile.endpoint,
+            ...this.#profile.apiKey !== void 0 ? { apiKey: this.#profile.apiKey } : {},
+            model: this.#profile.model,
+            texts: batch,
+            signal,
+            ...this.#profile.includeDimensions ? { dimensions: this.#profile.dimensions } : {},
+            ...this.#profile.encodingFormat ? { encodingFormat: this.#profile.encodingFormat } : {},
+            ...this.#profile.requireIndex !== void 0 ? { requireIndex: this.#profile.requireIndex } : {},
+            ...this.#profile.redirect ? { redirect: this.#profile.redirect } : {},
+            ...this.#profile.statusHeader ? { statusHeader: this.#profile.statusHeader } : {}
+          });
+          results.push(...validateEmbeddingBatch(
+            batchItems,
+            batch.length,
+            this.metadata.dimensions,
+            this.#profile.provider
+          ));
+          throwIfAborted(this.#profile.provider, "API request", signal);
         }
-        return data.data.map((item) => {
-          const typed = item;
-          const embedding = typed.embedding;
-          if (!Array.isArray(embedding)) {
-            throw new Error("Cloudflare response item did not include an embedding array");
-          }
-          const parsed = { embedding };
-          if (hasOwnProperty(typed, "index")) {
-            parsed.index = typed.index;
-          }
-          return parsed;
-        });
+        return results;
       },
       options.signal,
-      [this.#apiToken, this.#accountId, encodeURIComponent(this.#accountId)]
+      this.#profile.exactSecrets ?? []
     );
     return createEmbeddingBatchResult(
       this.metadata,
       intent,
-      validateEmbeddingBatch(items, texts.length, this.metadata.dimensions, "cloudflare")
+      validateEmbeddingBatch(items, texts.length, this.metadata.dimensions, this.#profile.provider)
     );
   }
 }
@@ -682,7 +649,16 @@ function createEmbeddingProvider(options = {}) {
       if (!key) {
         throw new Error("OPENAI_API_KEY is required for OpenAI embeddings");
       }
-      return new OpenAIEmbeddingProvider(metadata, key, timeoutMs);
+      return new OpenAICompatibleEmbeddingProvider(metadata, {
+        provider: "openai",
+        responseLabel: "OpenAI",
+        endpoint: OPENAI_EMBEDDINGS_URL,
+        apiKey: key,
+        model: metadata.model,
+        dimensions: metadata.dimensions,
+        includeDimensions: true,
+        exactSecrets: [key]
+      }, timeoutMs);
     }
     case "mistral": {
       const key = getOptionalTrimmedCredential(
@@ -691,7 +667,17 @@ function createEmbeddingProvider(options = {}) {
       if (!key) {
         throw new Error("MISTRAL_API_KEY is required for Mistral embeddings");
       }
-      return new MistralEmbeddingProvider(metadata, key, timeoutMs);
+      return new OpenAICompatibleEmbeddingProvider(metadata, {
+        provider: "mistral",
+        responseLabel: "Mistral",
+        endpoint: MISTRAL_EMBEDDINGS_URL,
+        apiKey: key,
+        model: metadata.model,
+        dimensions: metadata.dimensions,
+        encodingFormat: "float",
+        requireIndex: true,
+        exactSecrets: [key]
+      }, timeoutMs);
     }
     case "cloudflare": {
       const accountId = getOptionalTrimmedCredential(
@@ -706,13 +692,56 @@ function createEmbeddingProvider(options = {}) {
       if (!apiToken) {
         throw new Error("CLOUDFLARE_API_TOKEN is required for Cloudflare embeddings");
       }
-      return new CloudflareEmbeddingProvider(metadata, accountId, apiToken, timeoutMs);
+      const endpoint = createCloudflareEmbeddingsUrl(accountId);
+      return new OpenAICompatibleEmbeddingProvider(metadata, {
+        provider: "cloudflare",
+        responseLabel: "Cloudflare",
+        endpoint,
+        apiKey: apiToken,
+        model: metadata.model,
+        dimensions: metadata.dimensions,
+        statusHeader: { name: "cf-ray", label: "cf-ray" },
+        exactSecrets: [apiToken, accountId, encodeURIComponent(accountId), endpoint]
+      }, timeoutMs);
+    }
+    case "openai-compatible": {
+      const baseUrl = getRequiredTrimmedString(options.baseUrl, "baseUrl");
+      const endpoint = normalizeOpenAICompatibleEmbeddingsUrl(baseUrl);
+      const apiKey = getOptionalTrimmedCredential(options.apiKey);
+      const batchSize = getPositiveInteger(
+        options.batchSize ?? OPENAI_COMPATIBLE_DEFAULT_BATCH_SIZE,
+        "batchSize"
+      );
+      return new OpenAICompatibleEmbeddingProvider(metadata, {
+        provider: "openai-compatible",
+        responseLabel: "OpenAI-compatible",
+        endpoint,
+        ...apiKey ? { apiKey } : {},
+        model: metadata.model,
+        dimensions: metadata.dimensions,
+        includeDimensions: true,
+        encodingFormat: "float",
+        requireIndex: true,
+        redirect: "error",
+        batchSize,
+        exactSecrets: [
+          ...apiKey ? [apiKey] : [],
+          baseUrl,
+          endpoint
+        ]
+      }, timeoutMs);
     }
   }
 }
 function getEmbeddingProviderMetadata(options = {}) {
   const provider = resolveProviderName(options.provider);
-  return createProviderMetadata(provider, getEffectiveDimensions(provider, options.dimensions));
+  if (provider === "openai-compatible") {
+    normalizeOpenAICompatibleEmbeddingsUrl(getRequiredTrimmedString(options.baseUrl, "baseUrl"));
+    if (options.batchSize !== void 0) {
+      getPositiveInteger(options.batchSize, "batchSize");
+    }
+  }
+  return createProviderMetadata(provider, getEffectiveDimensions(provider, options.dimensions), options.model);
 }
 async function generateEmbeddings(texts, options = {}) {
   const maxLength = getPositiveInteger(options.maxLength ?? DEFAULT_MAX_LENGTH, "maxLength");

package/dist/index.d.ts

@@ -2,9 +2,10 @@ import { Client } from '@libsql/client';
 
 /**
  * Multi-provider embedding generation
- * Supports local Hugging Face Transformers, Gemini, OpenAI, Mistral, and Cloudflare Workers AI
+ * Supports local Hugging Face Transformers, Gemini, OpenAI, Mistral,
+ * Cloudflare Workers AI, and custom OpenAI-compatible endpoints
  */
-type EmbeddingProvider = 'local' | 'gemini' | 'openai' | 'mistral' | 'cloudflare';
+type EmbeddingProvider = 'local' | 'gemini' | 'openai' | 'mistral' | 'cloudflare' | 'openai-compatible';
 type EmbeddingIntent = 'document' | 'query';
 type EmbeddingBatchMode = 'native' | 'sequential';
 interface EmbeddingBatchBehavior {
@@ -37,6 +38,9 @@ interface EmbeddingOptions {
     apiKey?: string;
     accountId?: string;
     apiToken?: string;
+    baseUrl?: string;
+    model?: string;
+    batchSize?: number;
     dimensions?: number;
     maxLength?: number;
     intent?: EmbeddingIntent;