libsql-search 0.7.1 → 0.8.0

package/README.md

@@ -76,7 +76,9 @@ Important behavior:
 
 - Call `createTable()` before indexing or searching.
 - Keep table width, provider, and dimensions aligned across create/index/query.
-- `indexContent()` clears existing rows before rebuilding and is not transactional.
+- `indexContent()` embeds every document before it touches the database, then replaces the table in one transaction, so a failed rebuild leaves the previous index intact.
+- `indexContent()` throws `IndexingError` when a file fails; pass `failurePolicy: "skip"` to rebuild from the remaining files.
+- `indexContent()` throws `IndexingError` when no source files are found; pass `allowEmptyIndex: true` to intentionally empty the index.
 - Hosted providers send indexed and queried text to external services and may incur provider charges.
 
 ## Providers

package/dist/index.cjs

@@ -813,6 +813,16 @@ function normalizeVectorDimensions(dimensions) {
   return dimensions;
 }
 
+class IndexingError extends Error {
+  phase;
+  failures;
+  constructor(message, phase, failures = [], options) {
+    super(message, options);
+    this.name = "IndexingError";
+    this.phase = phase;
+    this.failures = failures;
+  }
+}
 async function indexContent(options) {
   const {
     client,
@@ -821,32 +831,92 @@ async function indexContent(options) {
     fileExtensions = [".md", ".markdown"],
     exclude = ["node_modules", ".git", "dist", "build"],
     tableName = "articles",
-    onProgress
+    onProgress,
+    failurePolicy = "abort",
+    allowEmptyIndex = false
   } = options;
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
-  const files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
-  if (files.length === 0) {
-    console.warn(`No files found in ${contentPath}`);
-    return { success: 0, failed: 0, total: 0 };
-  }
-  await client.execute(`DELETE FROM ${quotedTableName}`);
-  let success = 0;
-  let failed = 0;
+  let files;
+  try {
+    files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
+  } catch (error) {
+    throw new IndexingError(
+      `Failed to scan ${contentPath} for source files. The existing index was left unchanged.`,
+      "build",
+      [],
+      { cause: toError(error) }
+    );
+  }
+  files.sort((a, b) => a.relativePath < b.relativePath ? -1 : a.relativePath > b.relativePath ? 1 : 0);
+  if (files.length === 0 && !allowEmptyIndex) {
+    throw new IndexingError(
+      `No source files found in ${contentPath}. The existing index was left unchanged. Pass allowEmptyIndex: true to intentionally empty the index.`,
+      "build"
+    );
+  }
+  const documents = [];
+  const failures = [];
+  const slugOwners = /* @__PURE__ */ new Map();
   for (let i = 0; i < files.length; i++) {
     const file = files[i];
     if (onProgress) {
       onProgress(i + 1, files.length, file.relativePath);
     }
-    try {
-      const document = await processFile(file, embeddingOptions);
-      await insertDocument(client, document, quotedTableName);
-      success++;
-    } catch (error) {
-      console.error(`Failed to index ${file.relativePath}:`, error);
-      failed++;
+    const outcome = await buildDocument(file, embeddingOptions);
+    let failure;
+    if (outcome.ok) {
+      const owner = slugOwners.get(outcome.document.slug);
+      if (owner === void 0) {
+        slugOwners.set(outcome.document.slug, file.relativePath);
+        documents.push(outcome.document);
+      } else {
+        failure = {
+          file: file.relativePath,
+          stage: "parse",
+          error: new Error(
+            `Duplicate slug "${outcome.document.slug}": ${file.relativePath} collides with ${owner}`
+          )
+        };
+      }
+    } else {
+      failure = {
+        file: file.relativePath,
+        stage: outcome.stage,
+        error: outcome.error
+      };
+    }
+    if (failure === void 0) {
+      continue;
     }
+    if (failurePolicy === "abort") {
+      throw new IndexingError(
+        `Failed to ${failure.stage} ${file.relativePath}: ${failure.error.message}. The existing index was left unchanged. Pass failurePolicy: 'skip' to rebuild from the remaining files.`,
+        "build",
+        [failure],
+        { cause: failure.error }
+      );
+    }
+    console.error(`Skipping ${file.relativePath} (${failure.stage} failed):`, failure.error);
+    failures.push(failure);
   }
-  return { success, failed, total: files.length };
+  if (files.length > 0 && documents.length === 0) {
+    throw new IndexingError(
+      `All ${files.length} source file(s) in ${contentPath} failed to index. The existing index was left unchanged.`,
+      "build",
+      failures,
+      { cause: failures[0].error }
+    );
+  }
+  const success = documents.length;
+  await replaceIndex(client, quotedTableName, documents, failures);
+  return {
+    success,
+    failed: failures.length,
+    total: files.length,
+    replaced: true,
+    partial: failures.length > 0,
+    failures
+  };
 }
 async function findFiles(dir, baseDir, extensions, exclude) {
   const files = [];
@@ -870,34 +940,113 @@ async function findFiles(dir, baseDir, extensions, exclude) {
   }
   return files;
 }
-async function processFile(file, embeddingOptions) {
-  const content = await promises.readFile(file.fullPath, "utf-8");
-  const { data: frontMatter, content: markdown } = matter(content);
+async function buildDocument(file, embeddingOptions) {
+  let raw;
+  try {
+    raw = await promises.readFile(file.fullPath, "utf-8");
+  } catch (error) {
+    return { ok: false, stage: "read", error: toError(error) };
+  }
+  let parsed;
+  try {
+    parsed = parseFile(file, raw);
+  } catch (error) {
+    return { ok: false, stage: "parse", error: toError(error) };
+  }
+  let embedding;
+  try {
+    embedding = await generateEmbedding(parsed.embeddingText, {
+      ...embeddingOptions,
+      intent: embeddingOptions.intent ?? "document"
+    });
+  } catch (error) {
+    return { ok: false, stage: "embed", error: toError(error) };
+  }
+  return {
+    ok: true,
+    document: {
+      slug: parsed.slug,
+      title: parsed.title,
+      content: parsed.content,
+      folder: file.folder,
+      tags: parsed.tags,
+      serializedTags: parsed.serializedTags,
+      embedding,
+      metadata: parsed.metadata
+    }
+  };
+}
+function parseFile(file, raw) {
+  const { data: frontMatter, content: markdown } = matter(raw);
   const slug = file.relativePath.replace(/\.(md|markdown)$/, "").replace(/\\/g, "/");
-  const title = frontMatter.title || file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
+  const title = resolveTitle(file, frontMatter.title);
   const tags = Array.isArray(frontMatter.tags) ? frontMatter.tags : [];
+  const serializedTags = JSON.stringify(tags);
   const embeddingText = prepareTextForEmbedding({
     title,
     description: frontMatter.description,
     content: markdown,
     tags
   });
-  const embedding = await generateEmbedding(embeddingText, {
-    ...embeddingOptions,
-    intent: embeddingOptions.intent ?? "document"
-  });
   return {
     slug,
     title,
     content: markdown,
-    folder: file.folder,
     tags,
-    embedding,
+    serializedTags,
+    embeddingText,
     metadata: frontMatter
   };
 }
-async function insertDocument(client, document, quotedTableName) {
-  await client.execute({
+function resolveTitle(file, rawTitle) {
+  if (!rawTitle) {
+    return fallbackTitle(file);
+  }
+  if (typeof rawTitle === "string") {
+    return rawTitle;
+  }
+  if (typeof rawTitle === "number" || typeof rawTitle === "bigint" || typeof rawTitle === "boolean") {
+    return String(rawTitle);
+  }
+  if (rawTitle instanceof Date) {
+    return rawTitle.toISOString();
+  }
+  throw new Error(
+    `Unsupported frontmatter title of type ${describeType(rawTitle)}: expected a string`
+  );
+}
+function fallbackTitle(file) {
+  return file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
+}
+function describeType(value) {
+  return Array.isArray(value) ? "array" : typeof value;
+}
+async function replaceIndex(client, quotedTableName, documents, failures) {
+  const statements = [`DELETE FROM ${quotedTableName}`];
+  try {
+    for (let i = 0; i < documents.length; i++) {
+      const document = documents[i];
+      if (document !== void 0) {
+        statements.push(createInsertStatement(document, quotedTableName));
+        documents[i] = void 0;
+      }
+    }
+  } finally {
+    documents.length = 0;
+  }
+  try {
+    await client.batch(statements, "write");
+  } catch (error) {
+    throw new IndexingError(
+      `Failed to replace the contents of ${quotedTableName}. The transaction was rolled back and the existing index was left unchanged.`,
+      "replace",
+      failures,
+      { cause: toError(error) }
+    );
+  }
+}
+function createInsertStatement(document, quotedTableName) {
+  return {
     sql: `INSERT INTO ${quotedTableName}
           (slug, title, content, folder, tags, embedding, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, vector(?), datetime('now'), datetime('now'))`,
@@ -906,10 +1055,13 @@ async function insertDocument(client, document, quotedTableName) {
       document.title,
       document.content,
       document.folder,
-      JSON.stringify(document.tags),
+      document.serializedTags,
       JSON.stringify(document.embedding)
     ]
-  });
+  };
+}
+function toError(error) {
+  return error instanceof Error ? error : new Error(String(error));
 }
 async function createTable(client, tableName = "articles", dimensions = 384) {
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
@@ -1059,6 +1211,7 @@ async function getFolders(client, tableName = "articles") {
   return results.rows.map((row) => row.folder);
 }
 
+exports.IndexingError = IndexingError;
 exports.createEmbeddingProvider = createEmbeddingProvider;
 exports.createTable = createTable;
 exports.generateEmbedding = generateEmbedding;

package/dist/index.d.ts

@@ -82,6 +82,28 @@ declare function prepareTextForEmbedding(fields: {
  * Content indexer for markdown and other formats
  */
 
+/**
+ * How build-phase failures are handled.
+ *
+ * - `abort` (default) rejects the whole rebuild on the first failure
+ * - `skip` drops the failing file and rebuilds from the survivors
+ */
+type IndexFailurePolicy = 'abort' | 'skip';
+/** The step that failed while turning a file into an indexable document. */
+type IndexFailureStage = 'read' | 'parse' | 'embed';
+/**
+ * The phase an {@link IndexingError} was raised in.
+ *
+ * - `build` means no database state was touched
+ * - `replace` means the replacement transaction failed and was rolled back
+ */
+type IndexingErrorPhase = 'build' | 'replace';
+interface IndexFailure {
+    /** Path of the file relative to `contentPath`. */
+    file: string;
+    stage: IndexFailureStage;
+    error: Error;
+}
 interface IndexerOptions {
     client: Client;
     contentPath: string;
@@ -90,6 +112,23 @@ interface IndexerOptions {
     exclude?: string[];
     tableName?: string;
     onProgress?: (current: number, total: number, file: string) => void;
+    /** Defaults to `abort`. */
+    failurePolicy?: IndexFailurePolicy;
+    /** Allow an empty source directory to empty the index. Defaults to `false`. */
+    allowEmptyIndex?: boolean;
+}
+interface IndexResult {
+    /** Documents written to the table. */
+    success: number;
+    /** Files that could not be indexed. */
+    failed: number;
+    /** Files discovered on disk. */
+    total: number;
+    /** Whether table contents were replaced by this call. */
+    replaced: boolean;
+    /** Replaced, but some files were skipped. */
+    partial: boolean;
+    failures: IndexFailure[];
 }
 interface IndexedDocument {
     slug: string;
@@ -101,13 +140,22 @@ interface IndexedDocument {
     metadata?: Record<string, any>;
 }
 /**
- * Index markdown content from a directory
+ * Raised when a rebuild cannot complete. The previously indexed rows are always
+ * left exactly as they were.
  */
-declare function indexContent(options: IndexerOptions): Promise<{
-    success: number;
-    failed: number;
-    total: number;
-}>;
+declare class IndexingError extends Error {
+    readonly phase: IndexingErrorPhase;
+    readonly failures: IndexFailure[];
+    constructor(message: string, phase: IndexingErrorPhase, failures?: IndexFailure[], options?: ErrorOptions);
+}
+/**
+ * Index markdown content from a directory.
+ *
+ * Every document is read, parsed, and embedded in memory before any database
+ * state changes. The table is then replaced inside a single write transaction,
+ * so a failure at any point leaves the previous index intact.
+ */
+declare function indexContent(options: IndexerOptions): Promise<IndexResult>;
 /**
  * Create the articles table if it doesn't exist
  */
@@ -178,5 +226,5 @@ declare function getArticlesByFolder(client: Client, folder: string, tableName?:
  */
 declare function getFolders(client: Client, tableName?: string): Promise<string[]>;
 
-export { createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };
-export type { EmbeddingBatchBehavior, EmbeddingBatchItem, EmbeddingBatchItemResult, EmbeddingBatchMode, EmbeddingBatchResult, EmbeddingIntent, EmbeddingOptions, EmbeddingProvider, EmbeddingProviderClient, EmbeddingProviderMetadata, EmbeddingRequestOptions, IndexedDocument, IndexerOptions, SearchOptions, SearchResult };
+export { IndexingError, createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };
+export type { EmbeddingBatchBehavior, EmbeddingBatchItem, EmbeddingBatchItemResult, EmbeddingBatchMode, EmbeddingBatchResult, EmbeddingIntent, EmbeddingOptions, EmbeddingProvider, EmbeddingProviderClient, EmbeddingProviderMetadata, EmbeddingRequestOptions, IndexFailure, IndexFailurePolicy, IndexFailureStage, IndexResult, IndexedDocument, IndexerOptions, IndexingErrorPhase, SearchOptions, SearchResult };

package/dist/index.esm.js

@@ -811,6 +811,16 @@ function normalizeVectorDimensions(dimensions) {
   return dimensions;
 }
 
+class IndexingError extends Error {
+  phase;
+  failures;
+  constructor(message, phase, failures = [], options) {
+    super(message, options);
+    this.name = "IndexingError";
+    this.phase = phase;
+    this.failures = failures;
+  }
+}
 async function indexContent(options) {
   const {
     client,
@@ -819,32 +829,92 @@ async function indexContent(options) {
     fileExtensions = [".md", ".markdown"],
     exclude = ["node_modules", ".git", "dist", "build"],
     tableName = "articles",
-    onProgress
+    onProgress,
+    failurePolicy = "abort",
+    allowEmptyIndex = false
   } = options;
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
-  const files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
-  if (files.length === 0) {
-    console.warn(`No files found in ${contentPath}`);
-    return { success: 0, failed: 0, total: 0 };
-  }
-  await client.execute(`DELETE FROM ${quotedTableName}`);
-  let success = 0;
-  let failed = 0;
+  let files;
+  try {
+    files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
+  } catch (error) {
+    throw new IndexingError(
+      `Failed to scan ${contentPath} for source files. The existing index was left unchanged.`,
+      "build",
+      [],
+      { cause: toError(error) }
+    );
+  }
+  files.sort((a, b) => a.relativePath < b.relativePath ? -1 : a.relativePath > b.relativePath ? 1 : 0);
+  if (files.length === 0 && !allowEmptyIndex) {
+    throw new IndexingError(
+      `No source files found in ${contentPath}. The existing index was left unchanged. Pass allowEmptyIndex: true to intentionally empty the index.`,
+      "build"
+    );
+  }
+  const documents = [];
+  const failures = [];
+  const slugOwners = /* @__PURE__ */ new Map();
   for (let i = 0; i < files.length; i++) {
     const file = files[i];
     if (onProgress) {
       onProgress(i + 1, files.length, file.relativePath);
     }
-    try {
-      const document = await processFile(file, embeddingOptions);
-      await insertDocument(client, document, quotedTableName);
-      success++;
-    } catch (error) {
-      console.error(`Failed to index ${file.relativePath}:`, error);
-      failed++;
+    const outcome = await buildDocument(file, embeddingOptions);
+    let failure;
+    if (outcome.ok) {
+      const owner = slugOwners.get(outcome.document.slug);
+      if (owner === void 0) {
+        slugOwners.set(outcome.document.slug, file.relativePath);
+        documents.push(outcome.document);
+      } else {
+        failure = {
+          file: file.relativePath,
+          stage: "parse",
+          error: new Error(
+            `Duplicate slug "${outcome.document.slug}": ${file.relativePath} collides with ${owner}`
+          )
+        };
+      }
+    } else {
+      failure = {
+        file: file.relativePath,
+        stage: outcome.stage,
+        error: outcome.error
+      };
+    }
+    if (failure === void 0) {
+      continue;
     }
+    if (failurePolicy === "abort") {
+      throw new IndexingError(
+        `Failed to ${failure.stage} ${file.relativePath}: ${failure.error.message}. The existing index was left unchanged. Pass failurePolicy: 'skip' to rebuild from the remaining files.`,
+        "build",
+        [failure],
+        { cause: failure.error }
+      );
+    }
+    console.error(`Skipping ${file.relativePath} (${failure.stage} failed):`, failure.error);
+    failures.push(failure);
   }
-  return { success, failed, total: files.length };
+  if (files.length > 0 && documents.length === 0) {
+    throw new IndexingError(
+      `All ${files.length} source file(s) in ${contentPath} failed to index. The existing index was left unchanged.`,
+      "build",
+      failures,
+      { cause: failures[0].error }
+    );
+  }
+  const success = documents.length;
+  await replaceIndex(client, quotedTableName, documents, failures);
+  return {
+    success,
+    failed: failures.length,
+    total: files.length,
+    replaced: true,
+    partial: failures.length > 0,
+    failures
+  };
 }
 async function findFiles(dir, baseDir, extensions, exclude) {
   const files = [];
@@ -868,34 +938,113 @@ async function findFiles(dir, baseDir, extensions, exclude) {
   }
   return files;
 }
-async function processFile(file, embeddingOptions) {
-  const content = await readFile(file.fullPath, "utf-8");
-  const { data: frontMatter, content: markdown } = matter(content);
+async function buildDocument(file, embeddingOptions) {
+  let raw;
+  try {
+    raw = await readFile(file.fullPath, "utf-8");
+  } catch (error) {
+    return { ok: false, stage: "read", error: toError(error) };
+  }
+  let parsed;
+  try {
+    parsed = parseFile(file, raw);
+  } catch (error) {
+    return { ok: false, stage: "parse", error: toError(error) };
+  }
+  let embedding;
+  try {
+    embedding = await generateEmbedding(parsed.embeddingText, {
+      ...embeddingOptions,
+      intent: embeddingOptions.intent ?? "document"
+    });
+  } catch (error) {
+    return { ok: false, stage: "embed", error: toError(error) };
+  }
+  return {
+    ok: true,
+    document: {
+      slug: parsed.slug,
+      title: parsed.title,
+      content: parsed.content,
+      folder: file.folder,
+      tags: parsed.tags,
+      serializedTags: parsed.serializedTags,
+      embedding,
+      metadata: parsed.metadata
+    }
+  };
+}
+function parseFile(file, raw) {
+  const { data: frontMatter, content: markdown } = matter(raw);
   const slug = file.relativePath.replace(/\.(md|markdown)$/, "").replace(/\\/g, "/");
-  const title = frontMatter.title || file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
+  const title = resolveTitle(file, frontMatter.title);
   const tags = Array.isArray(frontMatter.tags) ? frontMatter.tags : [];
+  const serializedTags = JSON.stringify(tags);
   const embeddingText = prepareTextForEmbedding({
     title,
     description: frontMatter.description,
     content: markdown,
     tags
   });
-  const embedding = await generateEmbedding(embeddingText, {
-    ...embeddingOptions,
-    intent: embeddingOptions.intent ?? "document"
-  });
   return {
     slug,
     title,
     content: markdown,
-    folder: file.folder,
     tags,
-    embedding,
+    serializedTags,
+    embeddingText,
     metadata: frontMatter
   };
 }
-async function insertDocument(client, document, quotedTableName) {
-  await client.execute({
+function resolveTitle(file, rawTitle) {
+  if (!rawTitle) {
+    return fallbackTitle(file);
+  }
+  if (typeof rawTitle === "string") {
+    return rawTitle;
+  }
+  if (typeof rawTitle === "number" || typeof rawTitle === "bigint" || typeof rawTitle === "boolean") {
+    return String(rawTitle);
+  }
+  if (rawTitle instanceof Date) {
+    return rawTitle.toISOString();
+  }
+  throw new Error(
+    `Unsupported frontmatter title of type ${describeType(rawTitle)}: expected a string`
+  );
+}
+function fallbackTitle(file) {
+  return file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
+}
+function describeType(value) {
+  return Array.isArray(value) ? "array" : typeof value;
+}
+async function replaceIndex(client, quotedTableName, documents, failures) {
+  const statements = [`DELETE FROM ${quotedTableName}`];
+  try {
+    for (let i = 0; i < documents.length; i++) {
+      const document = documents[i];
+      if (document !== void 0) {
+        statements.push(createInsertStatement(document, quotedTableName));
+        documents[i] = void 0;
+      }
+    }
+  } finally {
+    documents.length = 0;
+  }
+  try {
+    await client.batch(statements, "write");
+  } catch (error) {
+    throw new IndexingError(
+      `Failed to replace the contents of ${quotedTableName}. The transaction was rolled back and the existing index was left unchanged.`,
+      "replace",
+      failures,
+      { cause: toError(error) }
+    );
+  }
+}
+function createInsertStatement(document, quotedTableName) {
+  return {
     sql: `INSERT INTO ${quotedTableName}
           (slug, title, content, folder, tags, embedding, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, vector(?), datetime('now'), datetime('now'))`,
@@ -904,10 +1053,13 @@ async function insertDocument(client, document, quotedTableName) {
       document.title,
       document.content,
       document.folder,
-      JSON.stringify(document.tags),
+      document.serializedTags,
       JSON.stringify(document.embedding)
     ]
-  });
+  };
+}
+function toError(error) {
+  return error instanceof Error ? error : new Error(String(error));
 }
 async function createTable(client, tableName = "articles", dimensions = 384) {
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
@@ -1057,4 +1209,4 @@ async function getFolders(client, tableName = "articles") {
   return results.rows.map((row) => row.folder);
 }
 
-export { createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };
+export { IndexingError, createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };

package/docs/API.md

@@ -18,6 +18,7 @@
 - `validateEmbeddingBatch`
 - `padEmbedding`
 - `prepareTextForEmbedding`
+- `IndexingError`
 
 It also exports these types:
 
@@ -34,6 +35,11 @@ It also exports these types:
 - `EmbeddingOptions`
 - `IndexerOptions`
 - `IndexedDocument`
+- `IndexResult`
+- `IndexFailure`
+- `IndexFailurePolicy`
+- `IndexFailureStage`
+- `IndexingErrorPhase`
 - `SearchOptions`
 - `SearchResult`
 
@@ -86,6 +92,8 @@ interface IndexerOptions {
   exclude?: string[];
   tableName?: string;
   onProgress?: (current: number, total: number, file: string) => void;
+  failurePolicy?: "abort" | "skip";
+  allowEmptyIndex?: boolean;
 }
 ```
 
@@ -94,25 +102,80 @@ Defaults:
 - `fileExtensions`: [".md", ".markdown"]
 - `exclude`: ["node_modules", ".git", "dist", "build"]
 - `tableName`: `"articles"`
+- `failurePolicy`: `"abort"`
+- `allowEmptyIndex`: `false`
 
 Return shape:
 
 ```ts
-{
-  success: number;
-  failed: number;
-  total: number;
+interface IndexResult {
+  success: number;   // documents written
+  failed: number;    // files that could not be indexed
+  total: number;     // files discovered on disk
+  replaced: boolean; // whether table contents were replaced by this call
+  partial: boolean;  // replaced, but some files were skipped
+  failures: IndexFailure[];
+}
+
+interface IndexFailure {
+  file: string;                        // path relative to contentPath
+  stage: "read" | "parse" | "embed";
+  error: Error;
 }
 ```
 
 Behavior notes:
 
-- `indexContent()` deletes existing rows in the target table before rebuilding
-- rebuilds are not transactional
+- every file is read, parsed, and embedded in memory before any database state changes
+- the target table is then replaced in a single write transaction, so a failed rebuild leaves the previous index exactly as it was
+- that costs peak memory proportional to the whole corpus, and against remote clients the replacement travels as a single un-chunked batch request; see [Costs of the two-phase rebuild](./INDEXING.md#costs-of-the-two-phase-rebuild) before rebuilding a very large corpus in place
+- files are discovered and indexed in sorted path order
+- frontmatter `title` must be a scalar; a structured title such as a YAML list fails the file at the `parse` stage
+- two files that reduce to the same slug (`foo.md` and `foo.markdown`) collide: the first in sorted path order keeps the slug and the later file is reported as a `parse` failure
+- `failurePolicy: "abort"` throws `IndexingError` on the first file that fails
+- `failurePolicy: "skip"` drops the failing file, records it in `failures`, and rebuilds from the survivors, returning `partial: true`
+- under `"skip"`, if every discovered file fails, the rebuild throws instead of replacing a valid index with an empty one
+- an empty source directory throws unless `allowEmptyIndex: true`, which intentionally empties the index
+- `onProgress` is called once per file during the build phase
 - frontmatter `title`, `description`, and `tags` are folded into the embedding text
 - embeddings default to `intent: "document"` unless `embeddingOptions.intent` is set explicitly
 - if a file has no frontmatter title, the filename becomes the title
 
+### `IndexingError`
+
+Thrown when a rebuild cannot complete. The previously indexed rows are always left unchanged.
+
+```ts
+class IndexingError extends Error {
+  readonly phase: "build" | "replace";
+  readonly failures: IndexFailure[];
+}
+```
+
+- `phase: "build"` means the failure happened before any database work: a file failed, every file failed, the source directory was empty, or it could not be scanned
+- `phase: "replace"` means the replacement transaction failed and was rolled back
+- `cause` carries the underlying error
+- on a `phase: "replace"` error, `failures` lists files skipped during the build phase. They are not the cause of the rollback, which is carried by `cause`
+
+```ts
+import { indexContent, IndexingError } from "libsql-search";
+
+try {
+  await indexContent({ client, contentPath: "./content" });
+} catch (error) {
+  if (error instanceof IndexingError) {
+    console.error(error.phase, error.failures);
+  }
+
+  throw error;
+}
+```
+
+Breaking changes in this behavior:
+
+- partial failures previously counted into `failed` and still replaced the table; they now throw. Pass `failurePolicy: "skip"` for the previous lenient behavior.
+- an empty source directory previously returned zeros and left stale rows in place; it now throws. Pass `allowEmptyIndex: true` to intentionally empty the index.
+
 ## `search(options)`
 
 Generates a query embedding and performs vector similarity search.

package/docs/INDEXING.md

@@ -15,7 +15,7 @@ The slug is derived from the file path relative to `contentPath`.
 
 ## Rebuild Behavior
 
-`indexContent()` clears the target table before rebuilding:
+`indexContent()` replaces the whole target table:
 
 ```ts
 await indexContent({
@@ -28,14 +28,89 @@ await indexContent({
 });
 ```
 
-That keeps the implementation simple, but it also means:
+The rebuild runs in two phases:
 
-- failed rebuilds can leave the table partially repopulated
-- provider or dimension changes should use a parallel table migration
+1. build: every file is read, parsed, and embedded in memory, touching no database state
+2. replace: the delete and all inserts run in a single write transaction
+
+That means:
+
+- a failed rebuild leaves the previously indexed rows exactly as they were
+- provider or dimension changes should still use a parallel table migration
 - `createTable()` does not resize an existing vector column
 
+Files are discovered and indexed in sorted path order, so a rebuild is deterministic.
+
 If provider, dimensions, model, endpoint, or embedding-space assumptions change, fully reindex into a new table. See the canonical [Migration and reindexing guide](./MIGRATIONS.md).
 
+### Costs Of The Two-Phase Rebuild
+
+Atomicity is not free, and both costs scale with corpus size:
+
+- **Peak memory holds the whole corpus.** The build phase keeps every document in memory: content, frontmatter, and one embedding array per document. The replace phase then builds insert statements including a JSON copy of each embedding, roughly 5-8 KB per document at 384 dimensions and considerably more at 3072. Documents are released as their statements are built, but peak usage is still proportional to the entire corpus rather than to one file.
+- **Remote clients send one request.** Against Turso or any remote client, the delete and every insert travel as a single batch. There is no chunking fallback, because splitting the batch would give up the atomicity this design exists to provide. A corpus large enough to exceed a remote request-size limit fails as an opaque `phase: "replace"` error.
+
+For very large corpora, index into a parallel table and switch reads over once it validates, rather than rebuilding a live table in place. See the [Migration and reindexing guide](./MIGRATIONS.md).
+
+## Content Requirements
+
+Two authoring mistakes fail a file at the `parse` stage rather than corrupting the rebuild:
+
+- **Frontmatter `title` must be a scalar.** Strings, numbers, booleans, and dates are accepted; dates are stored as ISO strings. A structured title such as a YAML list fails the file. A missing or empty title still falls back to the filename.
+- **Slugs must be unique.** The slug comes from the path with the extension removed, so `foo.md` and `foo.markdown` collide. Files are processed in sorted path order and the first file to claim a slug keeps it, so `foo.markdown` wins and `foo.md` is reported as the failure.
+
+Both are governed by `failurePolicy` like any other build failure, so they abort by default and are skippable.
+
+## Failure Handling
+
+`indexContent()` throws `IndexingError` instead of reporting a partially applied rebuild. The error carries `phase` (`"build"` or `"replace"`), a `failures` array, and the underlying error as `cause`.
+
+```ts
+import { indexContent, IndexingError } from "libsql-search";
+
+try {
+  await indexContent({ client, contentPath: "./content" });
+} catch (error) {
+  if (error instanceof IndexingError) {
+    for (const failure of error.failures) {
+      console.error(`${failure.file} failed during ${failure.stage}`);
+    }
+  }
+
+  throw error;
+}
+```
+
+By default one bad file aborts the whole rebuild. To index everything that can be indexed, opt into `failurePolicy: "skip"`:
+
+```ts
+const result = await indexContent({
+  client,
+  contentPath: "./content",
+  failurePolicy: "skip",
+});
+
+if (result.partial) {
+  console.warn(`Indexed ${result.success} of ${result.total} files`);
+}
+```
+
+Skipped rebuilds still replace the table, so treat `partial: true` as a build warning rather than a clean rebuild. If every discovered file fails, the rebuild throws rather than trading a valid index for an empty one.
+
+## Empty Source Directories
+
+An empty source directory throws by default, because silently leaving stale rows in place serves search traffic from content that no longer exists. Emptying an index has to be intentional:
+
+```ts
+await indexContent({
+  client,
+  contentPath: "./content",
+  allowEmptyIndex: true,
+});
+```
+
+Both behaviors changed in a breaking way: partial failures used to be counted and reported, and an empty directory used to return zeros without clearing the table.
+
 ## Quality Guidelines
 
 - include descriptive frontmatter titles

package/docs/MIGRATIONS.md

@@ -22,7 +22,7 @@ References:
 - if dimensions change, create a new table or recreate the old table, then fully re-embed
 - if dimensions stay the same but provider, model, endpoint, model revision, pooling, normalization, or input formatting changes, fully reindex anyway
 - never mix two embedding spaces in one table
-- prefer a parallel table migration because `indexContent()` clears rows first and is not transactional
+- prefer a parallel table migration because `indexContent()` replaces the whole target table, so an in-place rebuild leaves no way back to the old vectors
 
 In practice, this means:
 
@@ -39,6 +39,8 @@ In practice, this means:
 5. Switch application reads and writes to the new table.
 6. Retire the old table in a separate cleanup step.
 
+Step 3 is all or nothing. `indexContent()` throws `IndexingError` and leaves the target table untouched when a file or the replacement transaction fails, so a failed migration step can be retried without cleanup. See [Indexing and operational behavior](./INDEXING.md) for `failurePolicy` and `allowEmptyIndex`.
+
 Example:
 
 ```ts

package/docs/RELEASING.md

@@ -19,11 +19,19 @@ and JSR, then creates GitHub Release notes after both registries succeed.
    `.github/**` do not count as release-eligible and do not affect the bump. If
    the newest `main` commit is docs-only but an earlier untagged code or README
    commit is still pending, that newest run releases the accumulated eligible
-   changes. Breaking changes bump major, `feat:` bumps minor, and all other
-   eligible commits bump patch.
+   changes. `feat:` bumps minor and all other eligible commits bump patch.
+   Breaking changes (a `subject!:` prefix or a `BREAKING CHANGE:` footer) bump
+   major only once the package is `1.0.0` or higher. While the package is still
+   on the `0.x` line a breaking change bumps the minor instead, so an unattended
+   `fix!:` cannot auto-promote the package to `1.0.0`. Promoting off `0.x` is
+   deliberate and manual: see step 5.
 5. `package.json`, `jsr.json`, and `deno.json` are synchronized to the chosen
    version. If they already match the chosen version, no release commit is
-   created.
+   created. A manifest version that is already ahead of the latest tag wins over
+   the computed bump; this is the supported escape hatch for a deliberate
+   version jump, including promoting off the `0.x` line. Set all three manifests
+   to `1.0.0` on a reviewed PR and the next qualifying release publishes
+   `v1.0.0`.
 6. The workflow validates the candidate, creates an annotated tag, and atomically
    pushes the release commit plus tag.
 7. npm publishes `libsql-search` through trusted publishing OIDC with the GitHub

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libsql-search",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "description": "Semantic search for static sites using libSQL/Turso with multi-provider embeddings",
   "type": "module",
   "packageManager": "pnpm@10.34.5",