libsql-search 0.7.0 → 0.8.0

package/dist/index.esm.js

@@ -811,6 +811,16 @@ function normalizeVectorDimensions(dimensions) {
   return dimensions;
 }
 
+class IndexingError extends Error {
+  phase;
+  failures;
+  constructor(message, phase, failures = [], options) {
+    super(message, options);
+    this.name = "IndexingError";
+    this.phase = phase;
+    this.failures = failures;
+  }
+}
 async function indexContent(options) {
   const {
     client,
@@ -819,32 +829,92 @@ async function indexContent(options) {
     fileExtensions = [".md", ".markdown"],
     exclude = ["node_modules", ".git", "dist", "build"],
     tableName = "articles",
-    onProgress
+    onProgress,
+    failurePolicy = "abort",
+    allowEmptyIndex = false
   } = options;
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
-  const files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
-  if (files.length === 0) {
-    console.warn(`No files found in ${contentPath}`);
-    return { success: 0, failed: 0, total: 0 };
-  }
-  await client.execute(`DELETE FROM ${quotedTableName}`);
-  let success = 0;
-  let failed = 0;
+  let files;
+  try {
+    files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
+  } catch (error) {
+    throw new IndexingError(
+      `Failed to scan ${contentPath} for source files. The existing index was left unchanged.`,
+      "build",
+      [],
+      { cause: toError(error) }
+    );
+  }
+  files.sort((a, b) => a.relativePath < b.relativePath ? -1 : a.relativePath > b.relativePath ? 1 : 0);
+  if (files.length === 0 && !allowEmptyIndex) {
+    throw new IndexingError(
+      `No source files found in ${contentPath}. The existing index was left unchanged. Pass allowEmptyIndex: true to intentionally empty the index.`,
+      "build"
+    );
+  }
+  const documents = [];
+  const failures = [];
+  const slugOwners = /* @__PURE__ */ new Map();
   for (let i = 0; i < files.length; i++) {
     const file = files[i];
     if (onProgress) {
       onProgress(i + 1, files.length, file.relativePath);
     }
-    try {
-      const document = await processFile(file, embeddingOptions);
-      await insertDocument(client, document, quotedTableName);
-      success++;
-    } catch (error) {
-      console.error(`Failed to index ${file.relativePath}:`, error);
-      failed++;
+    const outcome = await buildDocument(file, embeddingOptions);
+    let failure;
+    if (outcome.ok) {
+      const owner = slugOwners.get(outcome.document.slug);
+      if (owner === void 0) {
+        slugOwners.set(outcome.document.slug, file.relativePath);
+        documents.push(outcome.document);
+      } else {
+        failure = {
+          file: file.relativePath,
+          stage: "parse",
+          error: new Error(
+            `Duplicate slug "${outcome.document.slug}": ${file.relativePath} collides with ${owner}`
+          )
+        };
+      }
+    } else {
+      failure = {
+        file: file.relativePath,
+        stage: outcome.stage,
+        error: outcome.error
+      };
+    }
+    if (failure === void 0) {
+      continue;
     }
+    if (failurePolicy === "abort") {
+      throw new IndexingError(
+        `Failed to ${failure.stage} ${file.relativePath}: ${failure.error.message}. The existing index was left unchanged. Pass failurePolicy: 'skip' to rebuild from the remaining files.`,
+        "build",
+        [failure],
+        { cause: failure.error }
+      );
+    }
+    console.error(`Skipping ${file.relativePath} (${failure.stage} failed):`, failure.error);
+    failures.push(failure);
   }
-  return { success, failed, total: files.length };
+  if (files.length > 0 && documents.length === 0) {
+    throw new IndexingError(
+      `All ${files.length} source file(s) in ${contentPath} failed to index. The existing index was left unchanged.`,
+      "build",
+      failures,
+      { cause: failures[0].error }
+    );
+  }
+  const success = documents.length;
+  await replaceIndex(client, quotedTableName, documents, failures);
+  return {
+    success,
+    failed: failures.length,
+    total: files.length,
+    replaced: true,
+    partial: failures.length > 0,
+    failures
+  };
 }
 async function findFiles(dir, baseDir, extensions, exclude) {
   const files = [];
@@ -868,34 +938,113 @@ async function findFiles(dir, baseDir, extensions, exclude) {
   }
   return files;
 }
-async function processFile(file, embeddingOptions) {
-  const content = await readFile(file.fullPath, "utf-8");
-  const { data: frontMatter, content: markdown } = matter(content);
+async function buildDocument(file, embeddingOptions) {
+  let raw;
+  try {
+    raw = await readFile(file.fullPath, "utf-8");
+  } catch (error) {
+    return { ok: false, stage: "read", error: toError(error) };
+  }
+  let parsed;
+  try {
+    parsed = parseFile(file, raw);
+  } catch (error) {
+    return { ok: false, stage: "parse", error: toError(error) };
+  }
+  let embedding;
+  try {
+    embedding = await generateEmbedding(parsed.embeddingText, {
+      ...embeddingOptions,
+      intent: embeddingOptions.intent ?? "document"
+    });
+  } catch (error) {
+    return { ok: false, stage: "embed", error: toError(error) };
+  }
+  return {
+    ok: true,
+    document: {
+      slug: parsed.slug,
+      title: parsed.title,
+      content: parsed.content,
+      folder: file.folder,
+      tags: parsed.tags,
+      serializedTags: parsed.serializedTags,
+      embedding,
+      metadata: parsed.metadata
+    }
+  };
+}
+function parseFile(file, raw) {
+  const { data: frontMatter, content: markdown } = matter(raw);
   const slug = file.relativePath.replace(/\.(md|markdown)$/, "").replace(/\\/g, "/");
-  const title = frontMatter.title || file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
+  const title = resolveTitle(file, frontMatter.title);
   const tags = Array.isArray(frontMatter.tags) ? frontMatter.tags : [];
+  const serializedTags = JSON.stringify(tags);
   const embeddingText = prepareTextForEmbedding({
     title,
     description: frontMatter.description,
     content: markdown,
     tags
   });
-  const embedding = await generateEmbedding(embeddingText, {
-    ...embeddingOptions,
-    intent: embeddingOptions.intent ?? "document"
-  });
   return {
     slug,
     title,
     content: markdown,
-    folder: file.folder,
     tags,
-    embedding,
+    serializedTags,
+    embeddingText,
     metadata: frontMatter
   };
 }
-async function insertDocument(client, document, quotedTableName) {
-  await client.execute({
+function resolveTitle(file, rawTitle) {
+  if (!rawTitle) {
+    return fallbackTitle(file);
+  }
+  if (typeof rawTitle === "string") {
+    return rawTitle;
+  }
+  if (typeof rawTitle === "number" || typeof rawTitle === "bigint" || typeof rawTitle === "boolean") {
+    return String(rawTitle);
+  }
+  if (rawTitle instanceof Date) {
+    return rawTitle.toISOString();
+  }
+  throw new Error(
+    `Unsupported frontmatter title of type ${describeType(rawTitle)}: expected a string`
+  );
+}
+function fallbackTitle(file) {
+  return file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
+}
+function describeType(value) {
+  return Array.isArray(value) ? "array" : typeof value;
+}
+async function replaceIndex(client, quotedTableName, documents, failures) {
+  const statements = [`DELETE FROM ${quotedTableName}`];
+  try {
+    for (let i = 0; i < documents.length; i++) {
+      const document = documents[i];
+      if (document !== void 0) {
+        statements.push(createInsertStatement(document, quotedTableName));
+        documents[i] = void 0;
+      }
+    }
+  } finally {
+    documents.length = 0;
+  }
+  try {
+    await client.batch(statements, "write");
+  } catch (error) {
+    throw new IndexingError(
+      `Failed to replace the contents of ${quotedTableName}. The transaction was rolled back and the existing index was left unchanged.`,
+      "replace",
+      failures,
+      { cause: toError(error) }
+    );
+  }
+}
+function createInsertStatement(document, quotedTableName) {
+  return {
     sql: `INSERT INTO ${quotedTableName}
           (slug, title, content, folder, tags, embedding, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, vector(?), datetime('now'), datetime('now'))`,
@@ -904,10 +1053,13 @@ async function insertDocument(client, document, quotedTableName) {
       document.title,
       document.content,
       document.folder,
-      JSON.stringify(document.tags),
+      document.serializedTags,
       JSON.stringify(document.embedding)
     ]
-  });
+  };
+}
+function toError(error) {
+  return error instanceof Error ? error : new Error(String(error));
 }
 async function createTable(client, tableName = "articles", dimensions = 384) {
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
@@ -1057,4 +1209,4 @@ async function getFolders(client, tableName = "articles") {
   return results.rows.map((row) => row.folder);
 }
 
-export { createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };
+export { IndexingError, createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };