libsql-search 0.7.0 → 0.8.0

package/README.md

@@ -5,27 +5,13 @@
 [![CI](https://github.com/llbbl/libsql-search/actions/workflows/ci.yml/badge.svg)](https://github.com/llbbl/libsql-search/actions/workflows/ci.yml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
-`libsql-search` adds semantic search to Markdown-backed sites using libSQL/Turso.
-It indexes frontmatter and content from files on disk, stores vectors in your
-database, and lets you query by meaning instead of exact keywords.
+`libsql-search` adds semantic search to Markdown-backed sites with a small TypeScript API. It indexes frontmatter and content from files on disk, stores vectors in libSQL/Turso, and lets you query by meaning instead of exact keywords.
 
 Use it when you want:
 
-- a small TypeScript library instead of a hosted search product
-- one search index shared across static-site builds and app routes
-- local or hosted embeddings behind the same indexing/search API
-- direct control over table names, dimensions, content shape, and deployment
-
-## What It Supports
-
-- Markdown indexing from local directories with frontmatter via `gray-matter`
-- libSQL/Turso storage and vector search
-- Embedding providers: local Hugging Face `Xenova/all-MiniLM-L6-v2`,
-  Cloudflare Workers AI `@cf/baai/bge-m3`, Mistral `mistral-embed`, Google Gemini
-  `gemini-embedding-2`, and OpenAI `text-embedding-3-small` /
-  `text-embedding-3-large`; self-hosted OpenAI-compatible endpoints are also
-  available for TEI and similar trusted deployments
-- npm distribution plus JSR publishing
+- one indexing/search API across local and hosted embedding providers
+- direct control over vector dimensions, table names, and deployment shape
+- a lightweight library instead of a hosted search product
 
 ## Install
 
@@ -44,18 +30,10 @@ deno add jsr:@logan/libsql-search npm:@libsql/client
 ```
 
 For npm usage, the package requires Node `>=22.12.0`.
-Node examples in this README import from `libsql-search` and `@libsql/client`.
-In Deno, after `deno add`, import from `@logan/libsql-search` and
-`@libsql/client`.
 
 ## Quick Start
 
-The shortest working flow is:
-
-1. create a libSQL client
-2. create the search table
-3. index a Markdown directory
-4. query it with the same embedding provider and dimensions
+This example uses the default local provider. Local embeddings run in-process after the initial model download and cache warmup; they are not automatically air-gapped.
 
 ```ts
 import { createClient } from "@libsql/client";
@@ -66,11 +44,12 @@ const client = createClient({
   authToken: "your-auth-token",
 });
 
-await createTable(client);
+await createTable(client, "articles_local_384", 384);
 
 await indexContent({
   client,
   contentPath: "./content",
+  tableName: "articles_local_384",
   embeddingOptions: {
     provider: "local",
   },
@@ -79,6 +58,7 @@ await indexContent({
 const results = await search({
   client,
   query: "how do I deploy my docs site",
+  tableName: "articles_local_384",
   limit: 5,
   embeddingOptions: {
     provider: "local",
@@ -95,33 +75,33 @@ console.log(results.map((result) => ({
 Important behavior:
 
 - Call `createTable()` before indexing or searching.
-- Keep dimensions aligned across table creation, indexing, and search queries.
-- `indexContent()` clears existing rows before rebuilding the index.
-- `local` is the default offline provider and uses 384 dimensions. Cloudflare is
-  the recommended hosted option. Cloudflare and Mistral use 1024 dimensions.
-  Gemini defaults to 3072 dimensions and supports 128-3072.
-
-## Core API
-
-- `createTable(client, tableName?, dimensions?)`
-- `indexContent(options)`
-- `search(options)`
-- `getAllArticles(client, tableName?)`
-- `getArticleBySlug(client, slug, tableName?)`
-- `getArticlesByFolder(client, folder, tableName?)`
-- `getFolders(client, tableName?)`
-- `generateEmbedding(text, options?)`
-- `prepareTextForEmbedding(fields)`
+- Keep table width, provider, and dimensions aligned across create/index/query.
+- `indexContent()` embeds every document before it touches the database, then replaces the table in one transaction, so a failed rebuild leaves the previous index intact.
+- `indexContent()` throws `IndexingError` when a file fails; pass `failurePolicy: "skip"` to rebuild from the remaining files.
+- `indexContent()` throws `IndexingError` when no source files are found; pass `allowEmptyIndex: true` to intentionally empty the index.
+- Hosted providers send indexed and queried text to external services and may incur provider charges.
+
+## Providers
+
+Built-in providers:
+
+- `local` with `Xenova/all-MiniLM-L6-v2` at 384 dimensions
+- `cloudflare` with `@cf/baai/bge-m3` at 1024 dimensions
+- `mistral` with `mistral-embed` at 1024 dimensions
+- `gemini` with `gemini-embedding-2` at 128-3072 dimensions, default 3072
+- `openai` with `text-embedding-3-small` or `text-embedding-3-large`, default 768
+- `openai-compatible` for trusted OpenAI-compatible endpoints such as TEI
 
 ## Docs
 
-- [Docs index](./docs/README.md)
-- [Provider guide](./docs/PROVIDERS.md)
+- [Documentation index](./docs/README.md)
+- [Provider selection and configuration](./docs/PROVIDERS.md)
 - [API reference](./docs/API.md)
 - [Integration examples](./docs/INTEGRATIONS.md)
+- [Migration and reindexing guide](./docs/MIGRATIONS.md)
+- [Testing guidance](./docs/TESTING.md)
 - [Indexing and operations](./docs/INDEXING.md)
 - [Troubleshooting](./docs/TROUBLESHOOTING.md)
-- [Release workflow](./docs/RELEASING.md)
 
 ## License
 

package/dist/index.cjs

@@ -813,6 +813,16 @@ function normalizeVectorDimensions(dimensions) {
   return dimensions;
 }
 
+class IndexingError extends Error {
+  phase;
+  failures;
+  constructor(message, phase, failures = [], options) {
+    super(message, options);
+    this.name = "IndexingError";
+    this.phase = phase;
+    this.failures = failures;
+  }
+}
 async function indexContent(options) {
   const {
     client,
@@ -821,32 +831,92 @@ async function indexContent(options) {
     fileExtensions = [".md", ".markdown"],
     exclude = ["node_modules", ".git", "dist", "build"],
     tableName = "articles",
-    onProgress
+    onProgress,
+    failurePolicy = "abort",
+    allowEmptyIndex = false
   } = options;
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
-  const files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
-  if (files.length === 0) {
-    console.warn(`No files found in ${contentPath}`);
-    return { success: 0, failed: 0, total: 0 };
-  }
-  await client.execute(`DELETE FROM ${quotedTableName}`);
-  let success = 0;
-  let failed = 0;
+  let files;
+  try {
+    files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
+  } catch (error) {
+    throw new IndexingError(
+      `Failed to scan ${contentPath} for source files. The existing index was left unchanged.`,
+      "build",
+      [],
+      { cause: toError(error) }
+    );
+  }
+  files.sort((a, b) => a.relativePath < b.relativePath ? -1 : a.relativePath > b.relativePath ? 1 : 0);
+  if (files.length === 0 && !allowEmptyIndex) {
+    throw new IndexingError(
+      `No source files found in ${contentPath}. The existing index was left unchanged. Pass allowEmptyIndex: true to intentionally empty the index.`,
+      "build"
+    );
+  }
+  const documents = [];
+  const failures = [];
+  const slugOwners = /* @__PURE__ */ new Map();
   for (let i = 0; i < files.length; i++) {
     const file = files[i];
     if (onProgress) {
       onProgress(i + 1, files.length, file.relativePath);
     }
-    try {
-      const document = await processFile(file, embeddingOptions);
-      await insertDocument(client, document, quotedTableName);
-      success++;
-    } catch (error) {
-      console.error(`Failed to index ${file.relativePath}:`, error);
-      failed++;
+    const outcome = await buildDocument(file, embeddingOptions);
+    let failure;
+    if (outcome.ok) {
+      const owner = slugOwners.get(outcome.document.slug);
+      if (owner === void 0) {
+        slugOwners.set(outcome.document.slug, file.relativePath);
+        documents.push(outcome.document);
+      } else {
+        failure = {
+          file: file.relativePath,
+          stage: "parse",
+          error: new Error(
+            `Duplicate slug "${outcome.document.slug}": ${file.relativePath} collides with ${owner}`
+          )
+        };
+      }
+    } else {
+      failure = {
+        file: file.relativePath,
+        stage: outcome.stage,
+        error: outcome.error
+      };
+    }
+    if (failure === void 0) {
+      continue;
     }
+    if (failurePolicy === "abort") {
+      throw new IndexingError(
+        `Failed to ${failure.stage} ${file.relativePath}: ${failure.error.message}. The existing index was left unchanged. Pass failurePolicy: 'skip' to rebuild from the remaining files.`,
+        "build",
+        [failure],
+        { cause: failure.error }
+      );
+    }
+    console.error(`Skipping ${file.relativePath} (${failure.stage} failed):`, failure.error);
+    failures.push(failure);
   }
-  return { success, failed, total: files.length };
+  if (files.length > 0 && documents.length === 0) {
+    throw new IndexingError(
+      `All ${files.length} source file(s) in ${contentPath} failed to index. The existing index was left unchanged.`,
+      "build",
+      failures,
+      { cause: failures[0].error }
+    );
+  }
+  const success = documents.length;
+  await replaceIndex(client, quotedTableName, documents, failures);
+  return {
+    success,
+    failed: failures.length,
+    total: files.length,
+    replaced: true,
+    partial: failures.length > 0,
+    failures
+  };
 }
 async function findFiles(dir, baseDir, extensions, exclude) {
   const files = [];
@@ -870,34 +940,113 @@ async function findFiles(dir, baseDir, extensions, exclude) {
   }
   return files;
 }
-async function processFile(file, embeddingOptions) {
-  const content = await promises.readFile(file.fullPath, "utf-8");
-  const { data: frontMatter, content: markdown } = matter(content);
+async function buildDocument(file, embeddingOptions) {
+  let raw;
+  try {
+    raw = await promises.readFile(file.fullPath, "utf-8");
+  } catch (error) {
+    return { ok: false, stage: "read", error: toError(error) };
+  }
+  let parsed;
+  try {
+    parsed = parseFile(file, raw);
+  } catch (error) {
+    return { ok: false, stage: "parse", error: toError(error) };
+  }
+  let embedding;
+  try {
+    embedding = await generateEmbedding(parsed.embeddingText, {
+      ...embeddingOptions,
+      intent: embeddingOptions.intent ?? "document"
+    });
+  } catch (error) {
+    return { ok: false, stage: "embed", error: toError(error) };
+  }
+  return {
+    ok: true,
+    document: {
+      slug: parsed.slug,
+      title: parsed.title,
+      content: parsed.content,
+      folder: file.folder,
+      tags: parsed.tags,
+      serializedTags: parsed.serializedTags,
+      embedding,
+      metadata: parsed.metadata
+    }
+  };
+}
+function parseFile(file, raw) {
+  const { data: frontMatter, content: markdown } = matter(raw);
   const slug = file.relativePath.replace(/\.(md|markdown)$/, "").replace(/\\/g, "/");
-  const title = frontMatter.title || file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
+  const title = resolveTitle(file, frontMatter.title);
   const tags = Array.isArray(frontMatter.tags) ? frontMatter.tags : [];
+  const serializedTags = JSON.stringify(tags);
   const embeddingText = prepareTextForEmbedding({
     title,
     description: frontMatter.description,
     content: markdown,
     tags
   });
-  const embedding = await generateEmbedding(embeddingText, {
-    ...embeddingOptions,
-    intent: embeddingOptions.intent ?? "document"
-  });
   return {
     slug,
     title,
     content: markdown,
-    folder: file.folder,
     tags,
-    embedding,
+    serializedTags,
+    embeddingText,
     metadata: frontMatter
   };
 }
-async function insertDocument(client, document, quotedTableName) {
-  await client.execute({
+function resolveTitle(file, rawTitle) {
+  if (!rawTitle) {
+    return fallbackTitle(file);
+  }
+  if (typeof rawTitle === "string") {
+    return rawTitle;
+  }
+  if (typeof rawTitle === "number" || typeof rawTitle === "bigint" || typeof rawTitle === "boolean") {
+    return String(rawTitle);
+  }
+  if (rawTitle instanceof Date) {
+    return rawTitle.toISOString();
+  }
+  throw new Error(
+    `Unsupported frontmatter title of type ${describeType(rawTitle)}: expected a string`
+  );
+}
+function fallbackTitle(file) {
+  return file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
+}
+function describeType(value) {
+  return Array.isArray(value) ? "array" : typeof value;
+}
+async function replaceIndex(client, quotedTableName, documents, failures) {
+  const statements = [`DELETE FROM ${quotedTableName}`];
+  try {
+    for (let i = 0; i < documents.length; i++) {
+      const document = documents[i];
+      if (document !== void 0) {
+        statements.push(createInsertStatement(document, quotedTableName));
+        documents[i] = void 0;
+      }
+    }
+  } finally {
+    documents.length = 0;
+  }
+  try {
+    await client.batch(statements, "write");
+  } catch (error) {
+    throw new IndexingError(
+      `Failed to replace the contents of ${quotedTableName}. The transaction was rolled back and the existing index was left unchanged.`,
+      "replace",
+      failures,
+      { cause: toError(error) }
+    );
+  }
+}
+function createInsertStatement(document, quotedTableName) {
+  return {
     sql: `INSERT INTO ${quotedTableName}
           (slug, title, content, folder, tags, embedding, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, vector(?), datetime('now'), datetime('now'))`,
@@ -906,10 +1055,13 @@ async function insertDocument(client, document, quotedTableName) {
       document.title,
       document.content,
       document.folder,
-      JSON.stringify(document.tags),
+      document.serializedTags,
       JSON.stringify(document.embedding)
     ]
-  });
+  };
+}
+function toError(error) {
+  return error instanceof Error ? error : new Error(String(error));
 }
 async function createTable(client, tableName = "articles", dimensions = 384) {
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
@@ -1059,6 +1211,7 @@ async function getFolders(client, tableName = "articles") {
   return results.rows.map((row) => row.folder);
 }
 
+exports.IndexingError = IndexingError;
 exports.createEmbeddingProvider = createEmbeddingProvider;
 exports.createTable = createTable;
 exports.generateEmbedding = generateEmbedding;

package/dist/index.d.ts

@@ -82,6 +82,28 @@ declare function prepareTextForEmbedding(fields: {
  * Content indexer for markdown and other formats
  */
 
+/**
+ * How build-phase failures are handled.
+ *
+ * - `abort` (default) rejects the whole rebuild on the first failure
+ * - `skip` drops the failing file and rebuilds from the survivors
+ */
+type IndexFailurePolicy = 'abort' | 'skip';
+/** The step that failed while turning a file into an indexable document. */
+type IndexFailureStage = 'read' | 'parse' | 'embed';
+/**
+ * The phase an {@link IndexingError} was raised in.
+ *
+ * - `build` means no database state was touched
+ * - `replace` means the replacement transaction failed and was rolled back
+ */
+type IndexingErrorPhase = 'build' | 'replace';
+interface IndexFailure {
+    /** Path of the file relative to `contentPath`. */
+    file: string;
+    stage: IndexFailureStage;
+    error: Error;
+}
 interface IndexerOptions {
     client: Client;
     contentPath: string;
@@ -90,6 +112,23 @@ interface IndexerOptions {
     exclude?: string[];
     tableName?: string;
     onProgress?: (current: number, total: number, file: string) => void;
+    /** Defaults to `abort`. */
+    failurePolicy?: IndexFailurePolicy;
+    /** Allow an empty source directory to empty the index. Defaults to `false`. */
+    allowEmptyIndex?: boolean;
+}
+interface IndexResult {
+    /** Documents written to the table. */
+    success: number;
+    /** Files that could not be indexed. */
+    failed: number;
+    /** Files discovered on disk. */
+    total: number;
+    /** Whether table contents were replaced by this call. */
+    replaced: boolean;
+    /** Replaced, but some files were skipped. */
+    partial: boolean;
+    failures: IndexFailure[];
 }
 interface IndexedDocument {
     slug: string;
@@ -101,13 +140,22 @@ interface IndexedDocument {
     metadata?: Record<string, any>;
 }
 /**
- * Index markdown content from a directory
+ * Raised when a rebuild cannot complete. The previously indexed rows are always
+ * left exactly as they were.
  */
-declare function indexContent(options: IndexerOptions): Promise<{
-    success: number;
-    failed: number;
-    total: number;
-}>;
+declare class IndexingError extends Error {
+    readonly phase: IndexingErrorPhase;
+    readonly failures: IndexFailure[];
+    constructor(message: string, phase: IndexingErrorPhase, failures?: IndexFailure[], options?: ErrorOptions);
+}
+/**
+ * Index markdown content from a directory.
+ *
+ * Every document is read, parsed, and embedded in memory before any database
+ * state changes. The table is then replaced inside a single write transaction,
+ * so a failure at any point leaves the previous index intact.
+ */
+declare function indexContent(options: IndexerOptions): Promise<IndexResult>;
 /**
  * Create the articles table if it doesn't exist
  */
@@ -178,5 +226,5 @@ declare function getArticlesByFolder(client: Client, folder: string, tableName?:
  */
 declare function getFolders(client: Client, tableName?: string): Promise<string[]>;
 
-export { createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };
-export type { EmbeddingBatchBehavior, EmbeddingBatchItem, EmbeddingBatchItemResult, EmbeddingBatchMode, EmbeddingBatchResult, EmbeddingIntent, EmbeddingOptions, EmbeddingProvider, EmbeddingProviderClient, EmbeddingProviderMetadata, EmbeddingRequestOptions, IndexedDocument, IndexerOptions, SearchOptions, SearchResult };
+export { IndexingError, createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };
+export type { EmbeddingBatchBehavior, EmbeddingBatchItem, EmbeddingBatchItemResult, EmbeddingBatchMode, EmbeddingBatchResult, EmbeddingIntent, EmbeddingOptions, EmbeddingProvider, EmbeddingProviderClient, EmbeddingProviderMetadata, EmbeddingRequestOptions, IndexFailure, IndexFailurePolicy, IndexFailureStage, IndexResult, IndexedDocument, IndexerOptions, IndexingErrorPhase, SearchOptions, SearchResult };