libsql-search 0.5.0 → 0.6.0

package/README.md

@@ -20,8 +20,8 @@ Use it when you want:
 
 - Markdown indexing from local directories with frontmatter via `gray-matter`
 - libSQL/Turso storage and vector search
-- Embedding providers: local `Xenova/all-MiniLM-L6-v2`, Cloudflare Workers AI
-  `@cf/baai/bge-m3`, Mistral `mistral-embed`, Google Gemini
+- Embedding providers: local Hugging Face `Xenova/all-MiniLM-L6-v2`,
+  Cloudflare Workers AI `@cf/baai/bge-m3`, Mistral `mistral-embed`, Google Gemini
   `gemini-embedding-2`, and OpenAI `text-embedding-3-small` /
   `text-embedding-3-large`
 - npm distribution plus JSR publishing
@@ -65,14 +65,13 @@ const client = createClient({
   authToken: "your-auth-token",
 });
 
-await createTable(client, "articles", 768);
+await createTable(client);
 
 await indexContent({
   client,
   contentPath: "./content",
   embeddingOptions: {
     provider: "local",
-    dimensions: 768,
   },
 });
 
@@ -82,7 +81,6 @@ const results = await search({
   limit: 5,
   embeddingOptions: {
     provider: "local",
-    dimensions: 768,
   },
 });
 
@@ -98,9 +96,9 @@ Important behavior:
 - Call `createTable()` before indexing or searching.
 - Keep dimensions aligned across table creation, indexing, and search queries.
 - `indexContent()` clears existing rows before rebuilding the index.
-- `local` is the default offline provider; Cloudflare is the recommended hosted
-  option. Cloudflare and Mistral use 1024 dimensions. Gemini defaults to 3072
-  dimensions and supports 128-3072.
+- `local` is the default offline provider and uses 384 dimensions. Cloudflare is
+  the recommended hosted option. Cloudflare and Mistral use 1024 dimensions.
+  Gemini defaults to 3072 dimensions and supports 128-3072.
 
 ## Core API
 

package/dist/index.cjs

@@ -4,7 +4,8 @@ var promises = require('fs/promises');
 var path = require('path');
 var matter = require('gray-matter');
 
-const DEFAULT_DIMENSIONS = 768;
+const OPENAI_DEFAULT_DIMENSIONS = 768;
+const LOCAL_DIMENSIONS = 384;
 const DEFAULT_MAX_LENGTH = 8e3;
 const DEFAULT_TIMEOUT_MS = 3e4;
 const LOCAL_MODEL = "Xenova/all-MiniLM-L6-v2";
@@ -33,17 +34,66 @@ function getEnvironmentVariable(name) {
     return void 0;
   }
 }
-async function getLocalEmbeddingModel(modelName) {
+function deletePendingLocalModelCache(modelName, entry) {
+  if (!entry.settled && entry.waiters === 0 && localModelCacheByModel.get(modelName) === entry) {
+    localModelCacheByModel.delete(modelName);
+  }
+}
+async function getLocalEmbeddingModel(modelName, signal) {
   const cached = localModelCacheByModel.get(modelName);
-  if (cached?.model) {
-    return cached.model;
+  if (cached) {
+    cached.waiters++;
+    try {
+      return await waitForLocalEmbeddingModel(modelName, cached, signal);
+    } finally {
+      cached.waiters--;
+      deletePendingLocalModelCache(modelName, cached);
+    }
+  }
+  const modelPromise = (async () => {
+    console.log(`Loading local embedding model (${modelName})...`);
+    const { pipeline } = await import('@huggingface/transformers');
+    const model = await pipeline("feature-extraction", modelName);
+    console.log("Local model loaded successfully");
+    return model;
+  })();
+  const entry = {
+    promise: modelPromise,
+    settled: false,
+    waiters: 1
+  };
+  localModelCacheByModel.set(modelName, entry);
+  modelPromise.then(() => {
+    entry.settled = true;
+  }).catch(() => {
+    localModelCacheByModel.delete(modelName);
+  });
+  try {
+    return await waitForLocalEmbeddingModel(modelName, entry, signal);
+  } finally {
+    entry.waiters--;
+    deletePendingLocalModelCache(modelName, entry);
+  }
+}
+async function waitForLocalEmbeddingModel(modelName, entry, signal) {
+  if (signal.aborted) {
+    deletePendingLocalModelCache(modelName, entry);
+    throw providerError("local", "model inference was aborted");
+  }
+  let rejectAbort = () => {
+  };
+  const abortPromise = new Promise((_resolve, reject) => {
+    rejectAbort = reject;
+  });
+  const onAbort = () => {
+    rejectAbort(providerError("local", "model inference was aborted"));
+  };
+  signal.addEventListener("abort", onAbort, { once: true });
+  try {
+    return await Promise.race([entry.promise, abortPromise]);
+  } finally {
+    signal.removeEventListener("abort", onAbort);
   }
-  console.log(`Loading local embedding model (${modelName})...`);
-  const { pipeline } = await import('@xenova/transformers');
-  const model = await pipeline("feature-extraction", modelName);
-  localModelCacheByModel.set(modelName, { model });
-  console.log("Local model loaded successfully");
-  return model;
 }
 function getPositiveInteger(value, optionName) {
   if (!Number.isInteger(value) || value <= 0) {
@@ -285,6 +335,15 @@ function createProviderMetadata(provider, dimensions) {
   }
 }
 function getEffectiveDimensions(provider, dimensions) {
+  if (provider === "local") {
+    if (dimensions !== void 0 && dimensions !== LOCAL_DIMENSIONS) {
+      throw providerError(
+        "local",
+        `${LOCAL_MODEL} returns ${LOCAL_DIMENSIONS} dimensions; received dimensions ${String(dimensions)}`
+      );
+    }
+    return LOCAL_DIMENSIONS;
+  }
   if (provider === "gemini") {
     const effectiveDimensions = dimensions ?? GEMINI_DIMENSIONS;
     if (!Number.isInteger(effectiveDimensions) || effectiveDimensions < GEMINI_MIN_DIMENSIONS || effectiveDimensions > GEMINI_MAX_DIMENSIONS) {
@@ -313,7 +372,7 @@ function getEffectiveDimensions(provider, dimensions) {
     }
     return CLOUDFLARE_DIMENSIONS;
   }
-  return getPositiveInteger(dimensions ?? DEFAULT_DIMENSIONS, "dimensions");
+  return getPositiveInteger(dimensions ?? OPENAI_DEFAULT_DIMENSIONS, "dimensions");
 }
 function assertBatchSize(metadata, count) {
   if (metadata.batch.maxSize !== void 0 && count > metadata.batch.maxSize) {
@@ -350,14 +409,13 @@ class LocalEmbeddingProvider {
       "model inference",
       this.#timeoutMs,
       async (signal) => {
-        const model = await getLocalEmbeddingModel(this.metadata.model);
+        const model = await getLocalEmbeddingModel(this.metadata.model, signal);
         return embedSequentially("local", "model inference", texts, signal, async (text) => {
           const output = await model(text, {
             pooling: "mean",
             normalize: true
           });
-          const embedding = Array.from(output.data);
-          return padEmbedding(embedding, this.metadata.dimensions);
+          return Array.from(output.data);
         });
       },
       options.signal
@@ -824,7 +882,7 @@ async function insertDocument(client, document, quotedTableName) {
     ]
   });
 }
-async function createTable(client, tableName = "articles", dimensions = 768) {
+async function createTable(client, tableName = "articles", dimensions = 384) {
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const vectorDimensions = normalizeVectorDimensions(dimensions);
   const quotedEmbeddingIndexName = quoteSqlIdentifier(`${tableName}_embedding_idx`, "embedding index name");

package/dist/index.d.ts

@@ -1,5 +1,9 @@
 import { Client } from '@libsql/client';
 
+/**
+ * Multi-provider embedding generation
+ * Supports local Hugging Face Transformers, Gemini, OpenAI, Mistral, and Cloudflare Workers AI
+ */
 type EmbeddingProvider = 'local' | 'gemini' | 'openai' | 'mistral' | 'cloudflare';
 type EmbeddingIntent = 'document' | 'query';
 type EmbeddingBatchMode = 'native' | 'sequential';

package/dist/index.esm.js

@@ -2,7 +2,8 @@ import { readdir, readFile } from 'fs/promises';
 import { join, extname, relative, dirname } from 'path';
 import matter from 'gray-matter';
 
-const DEFAULT_DIMENSIONS = 768;
+const OPENAI_DEFAULT_DIMENSIONS = 768;
+const LOCAL_DIMENSIONS = 384;
 const DEFAULT_MAX_LENGTH = 8e3;
 const DEFAULT_TIMEOUT_MS = 3e4;
 const LOCAL_MODEL = "Xenova/all-MiniLM-L6-v2";
@@ -31,17 +32,66 @@ function getEnvironmentVariable(name) {
     return void 0;
   }
 }
-async function getLocalEmbeddingModel(modelName) {
+function deletePendingLocalModelCache(modelName, entry) {
+  if (!entry.settled && entry.waiters === 0 && localModelCacheByModel.get(modelName) === entry) {
+    localModelCacheByModel.delete(modelName);
+  }
+}
+async function getLocalEmbeddingModel(modelName, signal) {
   const cached = localModelCacheByModel.get(modelName);
-  if (cached?.model) {
-    return cached.model;
+  if (cached) {
+    cached.waiters++;
+    try {
+      return await waitForLocalEmbeddingModel(modelName, cached, signal);
+    } finally {
+      cached.waiters--;
+      deletePendingLocalModelCache(modelName, cached);
+    }
+  }
+  const modelPromise = (async () => {
+    console.log(`Loading local embedding model (${modelName})...`);
+    const { pipeline } = await import('@huggingface/transformers');
+    const model = await pipeline("feature-extraction", modelName);
+    console.log("Local model loaded successfully");
+    return model;
+  })();
+  const entry = {
+    promise: modelPromise,
+    settled: false,
+    waiters: 1
+  };
+  localModelCacheByModel.set(modelName, entry);
+  modelPromise.then(() => {
+    entry.settled = true;
+  }).catch(() => {
+    localModelCacheByModel.delete(modelName);
+  });
+  try {
+    return await waitForLocalEmbeddingModel(modelName, entry, signal);
+  } finally {
+    entry.waiters--;
+    deletePendingLocalModelCache(modelName, entry);
+  }
+}
+async function waitForLocalEmbeddingModel(modelName, entry, signal) {
+  if (signal.aborted) {
+    deletePendingLocalModelCache(modelName, entry);
+    throw providerError("local", "model inference was aborted");
+  }
+  let rejectAbort = () => {
+  };
+  const abortPromise = new Promise((_resolve, reject) => {
+    rejectAbort = reject;
+  });
+  const onAbort = () => {
+    rejectAbort(providerError("local", "model inference was aborted"));
+  };
+  signal.addEventListener("abort", onAbort, { once: true });
+  try {
+    return await Promise.race([entry.promise, abortPromise]);
+  } finally {
+    signal.removeEventListener("abort", onAbort);
   }
-  console.log(`Loading local embedding model (${modelName})...`);
-  const { pipeline } = await import('@xenova/transformers');
-  const model = await pipeline("feature-extraction", modelName);
-  localModelCacheByModel.set(modelName, { model });
-  console.log("Local model loaded successfully");
-  return model;
 }
 function getPositiveInteger(value, optionName) {
   if (!Number.isInteger(value) || value <= 0) {
@@ -283,6 +333,15 @@ function createProviderMetadata(provider, dimensions) {
   }
 }
 function getEffectiveDimensions(provider, dimensions) {
+  if (provider === "local") {
+    if (dimensions !== void 0 && dimensions !== LOCAL_DIMENSIONS) {
+      throw providerError(
+        "local",
+        `${LOCAL_MODEL} returns ${LOCAL_DIMENSIONS} dimensions; received dimensions ${String(dimensions)}`
+      );
+    }
+    return LOCAL_DIMENSIONS;
+  }
   if (provider === "gemini") {
     const effectiveDimensions = dimensions ?? GEMINI_DIMENSIONS;
     if (!Number.isInteger(effectiveDimensions) || effectiveDimensions < GEMINI_MIN_DIMENSIONS || effectiveDimensions > GEMINI_MAX_DIMENSIONS) {
@@ -311,7 +370,7 @@ function getEffectiveDimensions(provider, dimensions) {
     }
     return CLOUDFLARE_DIMENSIONS;
   }
-  return getPositiveInteger(dimensions ?? DEFAULT_DIMENSIONS, "dimensions");
+  return getPositiveInteger(dimensions ?? OPENAI_DEFAULT_DIMENSIONS, "dimensions");
 }
 function assertBatchSize(metadata, count) {
   if (metadata.batch.maxSize !== void 0 && count > metadata.batch.maxSize) {
@@ -348,14 +407,13 @@ class LocalEmbeddingProvider {
       "model inference",
       this.#timeoutMs,
       async (signal) => {
-        const model = await getLocalEmbeddingModel(this.metadata.model);
+        const model = await getLocalEmbeddingModel(this.metadata.model, signal);
         return embedSequentially("local", "model inference", texts, signal, async (text) => {
           const output = await model(text, {
             pooling: "mean",
             normalize: true
           });
-          const embedding = Array.from(output.data);
-          return padEmbedding(embedding, this.metadata.dimensions);
+          return Array.from(output.data);
         });
       },
       options.signal
@@ -822,7 +880,7 @@ async function insertDocument(client, document, quotedTableName) {
     ]
   });
 }
-async function createTable(client, tableName = "articles", dimensions = 768) {
+async function createTable(client, tableName = "articles", dimensions = 384) {
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const vectorDimensions = normalizeVectorDimensions(dimensions);
   const quotedEmbeddingIndexName = quoteSqlIdentifier(`${tableName}_embedding_idx`, "embedding index name");

package/docs/API.md

@@ -42,13 +42,13 @@ It also exports these types:
 Creates the table and supporting indexes used by search.
 
 ```ts
-await createTable(client, "articles", 768);
+await createTable(client);
 ```
 
 Defaults:
 
 - `tableName`: `"articles"`
-- `dimensions`: `768`
+- `dimensions`: `384`
 
 `tableName` must be an ASCII SQLite identifier matching
 `[A-Za-z_][A-Za-z0-9_]*`. Valid identifiers are quoted internally, so reserved
@@ -216,6 +216,10 @@ credentials or configuration.
 Gemini uses `gemini-embedding-2`. Its default is 3072 dimensions, and explicit
 Gemini dimensions must be an integer from 128 through 3072.
 
+Local embeddings use `Xenova/all-MiniLM-L6-v2` through
+`@huggingface/transformers` and are fixed at the model's native 384 dimensions.
+Passing any other local dimension is rejected before the runtime is loaded.
+
 ### `getEmbeddingProviderMetadata(options?)`
 
 Returns the same metadata exposed by `createEmbeddingProvider(options).metadata`
@@ -279,6 +283,9 @@ Cloudflare uses `accountId` and `apiToken`, which fall back to
 
 Pads or truncates an embedding array to the requested length.
 
+This helper remains exported for callers that used it directly. The local
+provider does not use it; local vectors are validated at native 384 dimensions.
+
 ### `prepareTextForEmbedding(fields)`
 
 Combines title, description, tags, and content into the text sent to the

package/docs/INDEXING.md

@@ -25,7 +25,6 @@ await indexContent({
   tableName: "articles",
   embeddingOptions: {
     provider: "local",
-    dimensions: 768,
   },
 });
 ```
@@ -34,6 +33,11 @@ That keeps the implementation simple, but it also means a failed rebuild can
 leave the index partially repopulated.
 
 Changing an embedding provider or dimension count requires a full re-embed.
+For existing local indexes created with the older padded-local behavior, create
+or recreate a 384-dimensional table before rebuilding. Those older local
+768-dimensional tables stored 384 model values followed by zero padding;
+`indexContent()` clears rows but does not change the table's `F32_BLOB` width.
+
 For Gemini specifically, indexes created with the retired `text-embedding-004`
 model must be rebuilt for `gemini-embedding-2` even when staying at 768
 dimensions, because the model and query/document formatting both changed. If
@@ -72,7 +76,7 @@ database calls or embedding generation.
 
 ## Runtime Notes
 
-- local embeddings may download a model on the first run
+- local embeddings may download and cache a model on the first run
 - Node users need `@libsql/client` installed alongside the package
 - the repository validates both the npm package build and `deno check`, but the
   indexing flow itself still depends on filesystem access

package/docs/INTEGRATIONS.md

@@ -26,7 +26,6 @@ export const POST: APIRoute = async ({ request }) => {
     limit,
     embeddingOptions: {
       provider: "local",
-      dimensions: 768,
     },
   });
 
@@ -79,7 +78,6 @@ export async function POST(request: NextRequest) {
     limit,
     embeddingOptions: {
       provider: "local",
-      dimensions: 768,
     },
   });
 
@@ -135,12 +133,16 @@ const embeddingProvider =
     | "gemini"
     | "openai"
     | undefined;
-const embeddingDimensions =
-  embeddingProvider === "cloudflare" || embeddingProvider === "mistral"
-    ? 1024
-    : embeddingProvider === "gemini"
-      ? 3072
-    : 768;
+
+const dimensionsByProvider = {
+  local: 384,
+  cloudflare: 1024,
+  mistral: 1024,
+  gemini: 3072,
+  openai: 1536,
+} as const;
+
+const embeddingDimensions = dimensionsByProvider[embeddingProvider ?? "local"];
 
 await createTable(client, "articles", embeddingDimensions);
 

package/docs/PROVIDERS.md

@@ -29,8 +29,9 @@ interface EmbeddingOptions {
 ```
 
 - `provider` defaults to `"local"`
-- `dimensions` defaults to `768` for the library's default local provider.
-  Provider-specific defaults can differ; Gemini defaults to `3072`.
+- `dimensions` defaults to `384` for the library's default local provider.
+  Provider-specific defaults can differ; Cloudflare and Mistral use `1024`,
+  and Gemini defaults to `3072`.
 - `maxLength` defaults to `8000`
 - `intent` can be `"document"` or `"query"`; indexing defaults to
   `"document"` and search defaults to `"query"` unless explicitly set
@@ -87,7 +88,8 @@ vectors plus provider, model, dimensions, and intent. The compatibility helpers
 
 Cloudflare, Mistral, Gemini, and OpenAI clients are scoped to their current
 options. They are not cached globally across different credentials or
-configurations. The local Xenova model can be cached by model name.
+configurations. The local Hugging Face Transformers pipeline is loaded lazily
+and cached by model name.
 
 Hosted provider failures are reported with bounded provider/status/request-id
 context and without raw upstream bodies, credentials, Authorization headers, or
@@ -98,24 +100,26 @@ full URLs with query strings.
 Provider value: `local`
 
 The local provider loads `Xenova/all-MiniLM-L6-v2` through
-`@xenova/transformers`.
+`@huggingface/transformers`.
 
 ```ts
 embeddingOptions: {
   provider: "local",
-  dimensions: 768,
 }
 ```
 
 Notes:
 
-- the model emits 384 dimensions and `libsql-search` pads or truncates to your
-  requested size
-- metadata reports the requested output dimensions
+- the model emits 384 dimensions, and local vectors are validated at exactly
+  384 finite numbers
+- `dimensions: 384` is accepted explicitly; any other local dimension is
+  rejected before the runtime is imported or loaded
+- metadata reports 384 dimensions
 - batch metadata is `{ mode: "sequential" }`
-- the first run downloads the model and can take longer on a fresh machine
+- the first run downloads and caches the model and can take longer on a fresh
+  machine
 - no API key is required
-- this remains the default provider for backward compatibility and offline use
+- this remains the default provider for offline use
 
 ## Cloudflare Workers AI
 
@@ -227,10 +231,9 @@ Behavior:
 
 ## Dimension Guidelines
 
-- `local` defaults to `768`
+- `local` is fixed at `384`
 - `cloudflare` is fixed at `1024`
 - `mistral` is fixed at `1024`
-- local embeddings are padded from 384 to your target size
 - Gemini defaults to `3072` and accepts explicit dimensions from `128` through
   `3072`; use `768`, `1536`, or `3072` unless you have a specific reason
 - OpenAI can be used at 1536 or 3072, or another supported OpenAI dimension
@@ -245,3 +248,14 @@ because both the model and query/document input formatting changed. If you move
 to the new 3072-dimensional default, create a new table or recreate the vector
 table first; `indexContent()` clears rows but does not change the `F32_BLOB`
 width. A separate table is safer because rebuilds are not transactional.
+
+Existing local indexes created with the older padded-local behavior usually have
+`F32_BLOB(768)` rows containing the 384 model values followed by zero padding.
+The current local contract stores the native 384-dimensional model output. To
+migrate, create or recreate a `F32_BLOB(384)` table and run a full re-index
+before querying it. Using the same model ID avoids an intentional model-space
+change, but bit-identical vectors are not promised across runtime, model
+revision, dtype, pooling, or normalization changes; validate search quality and
+re-index when those details change.
+
+Routine unit tests mock the local runtime and do not download the model.

package/docs/TROUBLESHOOTING-SHARP.md

@@ -1,11 +1,12 @@
 # Troubleshooting: Transitive `sharp` Install Errors
 
-`libsql-search` does not directly depend on `sharp`. If you see an install error
-mentioning `sharp`, it is coming from another dependency in your application or
-toolchain.
+`libsql-search` does not directly import `sharp`, but local embeddings use
+`@huggingface/transformers`, which currently brings in `sharp` as a transitive
+runtime dependency. If you see an install error mentioning `sharp`, it is
+usually a native-package install or approval issue.
 
-This page exists because the error can show up in environments that also use
-`libsql-search`, and it is easy to misattribute the failure to this package.
+This page exists because the error can show up before your application reaches
+any `libsql-search` code.
 
 ## Typical Error
 
@@ -48,7 +49,7 @@ in `pnpm-workspace.yaml` with `onlyBuiltDependencies`.
 
 ## Relation To `libsql-search`
 
-- local embeddings use `@xenova/transformers`
+- local embeddings use `@huggingface/transformers`
 - the first local embedding run may download a model at runtime
 - that runtime model download is separate from a pnpm native-module install
   failure

package/docs/TROUBLESHOOTING.md

@@ -15,3 +15,5 @@ Common operational checks:
 - after upgrading an existing Gemini index, fully re-embed with
   `gemini-embedding-2`; for 3072-dimensional Gemini indexes, recreate the table
   or use a new table name before rebuilding
+- after upgrading an existing local 768-dimensional padded index, create or
+  recreate a 384-dimensional table and fully re-index before querying it

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libsql-search",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Semantic search for static sites using libSQL/Turso with multi-provider embeddings",
   "type": "module",
   "packageManager": "pnpm@10.34.5",
@@ -63,7 +63,7 @@
     "@libsql/client": "^0.15.0"
   },
   "dependencies": {
-    "@xenova/transformers": "^2.17.2",
+    "@huggingface/transformers": "4.2.0",
     "gray-matter": "^4.0.3"
   },
   "devDependencies": {