libsql-search 0.1.2 → 0.1.4

package/dist/index.esm.js

@@ -1,244 +1,213 @@
-import { pipeline } from '@xenova/transformers';
 import { readdir, readFile } from 'fs/promises';
 import { join, extname, relative, dirname } from 'path';
 import matter from 'gray-matter';
 
-/**
- * Multi-provider embedding generation
- * Supports local (Xenova), Gemini, and OpenAI
- */
 const providerCache = {};
-/**
- * Generate embeddings using the specified provider
- */
+function getEnvironmentVariable(name) {
+  const runtime = globalThis;
+  const nodeValue = runtime.process?.env?.[name];
+  if (nodeValue) {
+    return nodeValue;
+  }
+  try {
+    return runtime.Deno?.env?.get?.(name);
+  } catch {
+    return void 0;
+  }
+}
+async function getLocalEmbeddingModel() {
+  if (!providerCache.local) {
+    console.log("Loading local embedding model (Xenova/all-MiniLM-L6-v2)...");
+    const { pipeline } = await import('@xenova/transformers');
+    providerCache.local = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2");
+    console.log("Local model loaded successfully");
+  }
+  return providerCache.local;
+}
 async function generateEmbedding(text, options = {}) {
-    const { provider = 'local', apiKey, dimensions = 768, maxLength = 8000 } = options;
-    const truncated = text.substring(0, maxLength);
-    switch (provider) {
-        case 'local':
-            return generateLocalEmbedding(truncated, dimensions);
-        case 'gemini':
-            return generateGeminiEmbedding(truncated, apiKey);
-        case 'openai':
-            return generateOpenAIEmbedding(truncated, apiKey, dimensions);
-        default:
-            throw new Error(`Unknown embedding provider: ${provider}`);
-    }
+  const {
+    provider = "local",
+    apiKey,
+    dimensions = 768,
+    maxLength = 8e3
+  } = options;
+  const truncated = text.substring(0, maxLength);
+  switch (provider) {
+    case "local":
+      return generateLocalEmbedding(truncated, dimensions);
+    case "gemini":
+      return generateGeminiEmbedding(truncated, apiKey);
+    case "openai":
+      return generateOpenAIEmbedding(truncated, apiKey, dimensions);
+    default:
+      throw new Error(`Unknown embedding provider: ${provider}`);
+  }
 }
-/**
- * Generate embeddings using local model (Xenova/all-MiniLM-L6-v2)
- * Returns 384 dimensions, padded to target dimensions
- */
 async function generateLocalEmbedding(text, targetDimensions) {
-    if (!providerCache.local) {
-        console.log('Loading local embedding model (Xenova/all-MiniLM-L6-v2)...');
-        providerCache.local = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
-        console.log('Local model loaded successfully');
-    }
-    const output = await providerCache.local(text, {
-        pooling: 'mean',
-        normalize: true
-    });
-    const embedding = Array.from(output.data);
-    return padEmbedding(embedding, targetDimensions);
+  const model = await getLocalEmbeddingModel();
+  const output = await model(text, {
+    pooling: "mean",
+    normalize: true
+  });
+  const embedding = Array.from(output.data);
+  return padEmbedding(embedding, targetDimensions);
 }
-/**
- * Generate embeddings using Google Gemini API
- * Returns 768 dimensions natively
- */
 async function generateGeminiEmbedding(text, apiKey) {
-    const key = apiKey || process.env.GEMINI_API_KEY;
-    if (!key) {
-        throw new Error('GEMINI_API_KEY is required for Gemini embeddings');
-    }
-    if (!providerCache.gemini) {
-        // Dynamic import to keep it optional
-        const { GoogleGenerativeAI } = await import('@google/generative-ai');
-        const genAI = new GoogleGenerativeAI(key);
-        providerCache.gemini = genAI.getGenerativeModel({ model: 'text-embedding-004' });
-    }
-    const result = await providerCache.gemini.embedContent(text);
-    return result.embedding.values;
+  const key = apiKey || getEnvironmentVariable("GEMINI_API_KEY");
+  if (!key) {
+    throw new Error("GEMINI_API_KEY is required for Gemini embeddings");
+  }
+  if (!providerCache.gemini) {
+    const { GoogleGenerativeAI } = await import('@google/generative-ai');
+    const genAI = new GoogleGenerativeAI(key);
+    providerCache.gemini = genAI.getGenerativeModel({ model: "text-embedding-004" });
+  }
+  const result = await providerCache.gemini.embedContent(text);
+  return result.embedding.values;
 }
-/**
- * Generate embeddings using OpenAI API
- * Supports text-embedding-3-small (1536 dims) and text-embedding-3-large (3072 dims)
- */
 async function generateOpenAIEmbedding(text, apiKey, dimensions = 1536) {
-    const key = apiKey || process.env.OPENAI_API_KEY;
-    if (!key) {
-        throw new Error('OPENAI_API_KEY is required for OpenAI embeddings');
-    }
-    const model = dimensions <= 1536 ? 'text-embedding-3-small' : 'text-embedding-3-large';
-    const response = await fetch('https://api.openai.com/v1/embeddings', {
-        method: 'POST',
-        headers: {
-            'Content-Type': 'application/json',
-            'Authorization': `Bearer ${key}`
-        },
-        body: JSON.stringify({
-            input: text,
-            model,
-            dimensions
-        })
-    });
-    if (!response.ok) {
-        const error = await response.text();
-        throw new Error(`OpenAI API error: ${error}`);
-    }
-    const data = await response.json();
-    return data.data[0].embedding;
+  const key = apiKey || getEnvironmentVariable("OPENAI_API_KEY");
+  if (!key) {
+    throw new Error("OPENAI_API_KEY is required for OpenAI embeddings");
+  }
+  const model = dimensions <= 1536 ? "text-embedding-3-small" : "text-embedding-3-large";
+  const response = await fetch("https://api.openai.com/v1/embeddings", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Authorization": `Bearer ${key}`
+    },
+    body: JSON.stringify({
+      input: text,
+      model,
+      dimensions
+    })
+  });
+  if (!response.ok) {
+    const error = await response.text();
+    throw new Error(`OpenAI API error: ${error}`);
+  }
+  const data = await response.json();
+  return data.data[0].embedding;
 }
-/**
- * Pad or truncate embedding to target dimensions
- */
 function padEmbedding(embedding, targetDimensions) {
-    if (embedding.length === targetDimensions) {
-        return embedding;
-    }
-    if (embedding.length > targetDimensions) {
-        return embedding.slice(0, targetDimensions);
-    }
-    const padded = new Array(targetDimensions).fill(0);
-    padded.splice(0, embedding.length, ...embedding);
-    return padded;
+  if (embedding.length === targetDimensions) {
+    return embedding;
+  }
+  if (embedding.length > targetDimensions) {
+    return embedding.slice(0, targetDimensions);
+  }
+  const padded = new Array(targetDimensions).fill(0);
+  padded.splice(0, embedding.length, ...embedding);
+  return padded;
 }
-/**
- * Prepare text for embedding by combining multiple fields
- */
 function prepareTextForEmbedding(fields) {
-    const parts = [];
-    if (fields.title)
-        parts.push(fields.title);
-    if (fields.description)
-        parts.push(fields.description);
-    if (fields.tags && fields.tags.length > 0) {
-        parts.push(`Tags: ${fields.tags.join(', ')}`);
-    }
-    if (fields.content)
-        parts.push(fields.content);
-    return parts.filter(Boolean).join('\n\n');
+  const parts = [];
+  if (fields.title) parts.push(fields.title);
+  if (fields.description) parts.push(fields.description);
+  if (fields.tags && fields.tags.length > 0) {
+    parts.push(`Tags: ${fields.tags.join(", ")}`);
+  }
+  if (fields.content) parts.push(fields.content);
+  return parts.filter(Boolean).join("\n\n");
 }
 
-/**
- * Content indexer for markdown and other formats
- */
-/**
- * Index markdown content from a directory
- */
 async function indexContent(options) {
-    const { client, contentPath, embeddingOptions = {}, fileExtensions = ['.md', '.markdown'], exclude = ['node_modules', '.git', 'dist', 'build'], tableName = 'articles', onProgress } = options;
-    // Find all content files
-    const files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
-    if (files.length === 0) {
-        console.warn(`No files found in ${contentPath}`);
-        return { success: 0, failed: 0, total: 0 };
+  const {
+    client,
+    contentPath,
+    embeddingOptions = {},
+    fileExtensions = [".md", ".markdown"],
+    exclude = ["node_modules", ".git", "dist", "build"],
+    tableName = "articles",
+    onProgress
+  } = options;
+  const files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
+  if (files.length === 0) {
+    console.warn(`No files found in ${contentPath}`);
+    return { success: 0, failed: 0, total: 0 };
+  }
+  await client.execute(`DELETE FROM ${tableName}`);
+  let success = 0;
+  let failed = 0;
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+    if (onProgress) {
+      onProgress(i + 1, files.length, file.relativePath);
     }
-    // Clear existing content
-    await client.execute(`DELETE FROM ${tableName}`);
-    // Process each file
-    let success = 0;
-    let failed = 0;
-    for (let i = 0; i < files.length; i++) {
-        const file = files[i];
-        if (onProgress) {
-            onProgress(i + 1, files.length, file.relativePath);
-        }
-        try {
-            const document = await processFile(file, embeddingOptions);
-            await insertDocument(client, document, tableName);
-            success++;
-        }
-        catch (error) {
-            console.error(`Failed to index ${file.relativePath}:`, error);
-            failed++;
-        }
+    try {
+      const document = await processFile(file, embeddingOptions);
+      await insertDocument(client, document, tableName);
+      success++;
+    } catch (error) {
+      console.error(`Failed to index ${file.relativePath}:`, error);
+      failed++;
     }
-    return { success, failed, total: files.length };
+  }
+  return { success, failed, total: files.length };
 }
-/**
- * Find all files matching extensions
- */
 async function findFiles(dir, baseDir, extensions, exclude) {
-    const files = [];
-    const entries = await readdir(dir, { withFileTypes: true });
-    for (const entry of entries) {
-        const fullPath = join(dir, entry.name);
-        if (entry.isDirectory()) {
-            if (!entry.name.startsWith('.') && !exclude.includes(entry.name)) {
-                const subFiles = await findFiles(fullPath, baseDir, extensions, exclude);
-                files.push(...subFiles);
-            }
-        }
-        else if (extensions.includes(extname(entry.name))) {
-            const relativePath = relative(baseDir, fullPath);
-            const folder = dirname(relativePath);
-            files.push({
-                fullPath,
-                relativePath,
-                folder: folder === '.' ? 'root' : folder
-            });
-        }
+  const files = [];
+  const entries = await readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (!entry.name.startsWith(".") && !exclude.includes(entry.name)) {
+        const subFiles = await findFiles(fullPath, baseDir, extensions, exclude);
+        files.push(...subFiles);
+      }
+    } else if (extensions.includes(extname(entry.name))) {
+      const relativePath = relative(baseDir, fullPath);
+      const folder = dirname(relativePath);
+      files.push({
+        fullPath,
+        relativePath,
+        folder: folder === "." ? "root" : folder
+      });
     }
-    return files;
+  }
+  return files;
 }
-/**
- * Process a single file into an indexed document
- */
 async function processFile(file, embeddingOptions) {
-    const content = await readFile(file.fullPath, 'utf-8');
-    const { data: frontMatter, content: markdown } = matter(content);
-    // Generate slug from relative path
-    const slug = file.relativePath
-        .replace(/\.(md|markdown)$/, '')
-        .replace(/\\/g, '/');
-    // Extract metadata
-    const title = frontMatter.title || file.relativePath
-        .split('/').pop()
-        ?.replace(/\.(md|markdown)$/, '')
-        .replace(/-/g, ' ') || 'Untitled';
-    const tags = Array.isArray(frontMatter.tags) ? frontMatter.tags : [];
-    // Generate embedding
-    const embeddingText = prepareTextForEmbedding({
-        title,
-        description: frontMatter.description,
-        content: markdown,
-        tags
-    });
-    const embedding = await generateEmbedding(embeddingText, embeddingOptions);
-    return {
-        slug,
-        title,
-        content: markdown,
-        folder: file.folder,
-        tags,
-        embedding,
-        metadata: frontMatter
-    };
+  const content = await readFile(file.fullPath, "utf-8");
+  const { data: frontMatter, content: markdown } = matter(content);
+  const slug = file.relativePath.replace(/\.(md|markdown)$/, "").replace(/\\/g, "/");
+  const title = frontMatter.title || file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
+  const tags = Array.isArray(frontMatter.tags) ? frontMatter.tags : [];
+  const embeddingText = prepareTextForEmbedding({
+    title,
+    description: frontMatter.description,
+    content: markdown,
+    tags
+  });
+  const embedding = await generateEmbedding(embeddingText, embeddingOptions);
+  return {
+    slug,
+    title,
+    content: markdown,
+    folder: file.folder,
+    tags,
+    embedding,
+    metadata: frontMatter
+  };
 }
-/**
- * Insert document into database
- */
 async function insertDocument(client, document, tableName) {
-    await client.execute({
-        sql: `INSERT INTO ${tableName}
+  await client.execute({
+    sql: `INSERT INTO ${tableName}
           (slug, title, content, folder, tags, embedding, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, vector(?), datetime('now'), datetime('now'))`,
-        args: [
-            document.slug,
-            document.title,
-            document.content,
-            document.folder,
-            JSON.stringify(document.tags),
-            JSON.stringify(document.embedding)
-        ]
-    });
+    args: [
+      document.slug,
+      document.title,
+      document.content,
+      document.folder,
+      JSON.stringify(document.tags),
+      JSON.stringify(document.embedding)
+    ]
+  });
 }
-/**
- * Create the articles table if it doesn't exist
- */
-async function createTable(client, tableName = 'articles', dimensions = 768) {
-    await client.execute(`
+async function createTable(client, tableName = "articles", dimensions = 768) {
+  await client.execute(`
     CREATE TABLE IF NOT EXISTS ${tableName} (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       slug TEXT UNIQUE NOT NULL,
@@ -251,33 +220,31 @@ async function createTable(client, tableName = 'articles', dimensions = 768) {
       updated_at TEXT NOT NULL
     )
   `);
-    await client.execute(`
+  await client.execute(`
     CREATE INDEX IF NOT EXISTS ${tableName}_embedding_idx
     ON ${tableName}(libsql_vector_idx(embedding))
   `);
-    await client.execute(`
+  await client.execute(`
     CREATE INDEX IF NOT EXISTS ${tableName}_folder_idx
     ON ${tableName}(folder)
   `);
-    await client.execute(`
+  await client.execute(`
     CREATE INDEX IF NOT EXISTS ${tableName}_slug_idx
     ON ${tableName}(slug)
   `);
 }
 
-/**
- * Vector search functionality
- */
-/**
- * Perform semantic search using vector similarity
- */
 async function search(options) {
-    const { client, query, limit = 10, tableName = 'articles', embeddingOptions = {} } = options;
-    // Generate embedding for query
-    const queryEmbedding = await generateEmbedding(query, embeddingOptions);
-    // Perform vector search
-    const results = await client.execute({
-        sql: `
+  const {
+    client,
+    query,
+    limit = 10,
+    tableName = "articles",
+    embeddingOptions = {}
+  } = options;
+  const queryEmbedding = await generateEmbedding(query, embeddingOptions);
+  const results = await client.execute({
+    sql: `
       SELECT
         id,
         slug,
@@ -292,98 +259,85 @@ async function search(options) {
       ORDER BY distance
       LIMIT ?
     `,
-        args: [JSON.stringify(queryEmbedding), limit]
-    });
-    // Parse and format results
-    return results.rows.map(row => ({
-        id: row.id,
-        slug: row.slug,
-        title: row.title,
-        content: row.content,
-        folder: row.folder,
-        tags: JSON.parse(row.tags || '[]'),
-        distance: row.distance,
-        created_at: row.created_at
-    }));
+    args: [JSON.stringify(queryEmbedding), limit]
+  });
+  return results.rows.map((row) => ({
+    id: row.id,
+    slug: row.slug,
+    title: row.title,
+    content: row.content,
+    folder: row.folder,
+    tags: JSON.parse(row.tags || "[]"),
+    distance: row.distance,
+    created_at: row.created_at
+  }));
 }
-/**
- * Get all articles (for building static pages, navigation, etc.)
- */
-async function getAllArticles(client, tableName = 'articles') {
-    const results = await client.execute(`
+async function getAllArticles(client, tableName = "articles") {
+  const results = await client.execute(`
     SELECT id, slug, title, folder, tags, created_at, updated_at
     FROM ${tableName}
     ORDER BY title
   `);
-    return results.rows.map(row => ({
-        id: row.id,
-        slug: row.slug,
-        title: row.title,
-        folder: row.folder,
-        tags: JSON.parse(row.tags || '[]'),
-        created_at: row.created_at,
-        updated_at: row.updated_at
-    }));
+  return results.rows.map((row) => ({
+    id: row.id,
+    slug: row.slug,
+    title: row.title,
+    folder: row.folder,
+    tags: JSON.parse(row.tags || "[]"),
+    created_at: row.created_at,
+    updated_at: row.updated_at
+  }));
 }
-/**
- * Get a single article by slug
- */
-async function getArticleBySlug(client, slug, tableName = 'articles') {
-    const results = await client.execute({
-        sql: `
+async function getArticleBySlug(client, slug, tableName = "articles") {
+  const results = await client.execute({
+    sql: `
       SELECT id, slug, title, content, folder, tags, created_at, updated_at
       FROM ${tableName}
       WHERE slug = ?
       LIMIT 1
     `,
-        args: [slug]
-    });
-    if (results.rows.length === 0) {
-        return null;
-    }
-    const row = results.rows[0];
-    return {
-        id: row.id,
-        slug: row.slug,
-        title: row.title,
-        content: row.content,
-        folder: row.folder,
-        tags: JSON.parse(row.tags || '[]'),
-        created_at: row.created_at,
-        updated_at: row.updated_at
-    };
+    args: [slug]
+  });
+  if (results.rows.length === 0) {
+    return null;
+  }
+  const row = results.rows[0];
+  return {
+    id: row.id,
+    slug: row.slug,
+    title: row.title,
+    content: row.content,
+    folder: row.folder,
+    tags: JSON.parse(row.tags || "[]"),
+    created_at: row.created_at,
+    updated_at: row.updated_at
+  };
 }
-/**
- * Get articles by folder
- */
-async function getArticlesByFolder(client, folder, tableName = 'articles') {
-    const results = await client.execute({
-        sql: `
+async function getArticlesByFolder(client, folder, tableName = "articles") {
+  const results = await client.execute({
+    sql: `
       SELECT id, slug, title, folder, tags
       FROM ${tableName}
       WHERE folder = ?
       ORDER BY title
     `,
-        args: [folder]
-    });
-    return results.rows.map(row => ({
-        id: row.id,
-        slug: row.slug,
-        title: row.title,
-        folder: row.folder,
-        tags: JSON.parse(row.tags || '[]')
-    }));
+    args: [folder]
+  });
+  return results.rows.map((row) => ({
+    id: row.id,
+    slug: row.slug,
+    title: row.title,
+    folder: row.folder,
+    tags: JSON.parse(row.tags || "[]")
+  }));
 }
-/**
- * Get all unique folders
- */
-async function getFolders(client, tableName = 'articles') {
-    const results = await client.execute(`
+async function getFolders(client, tableName = "articles") {
+  const results = await client.execute(`
     SELECT DISTINCT folder
     FROM ${tableName}
     ORDER BY folder
   `);
-    return results.rows.map(row => row.folder);
+  return results.rows.map((row) => row.folder);
 }
 
 export { createTable, generateEmbedding, getAllArticles, getArticleBySlug, getArticlesByFolder, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search };