libsql-search 0.1.3 → 0.1.4

package/README.md

@@ -1,5 +1,11 @@
 # libsql-search
 
+[![npm version](https://img.shields.io/npm/v/libsql-search.svg)](https://www.npmjs.com/package/libsql-search)
+[![JSR](https://jsr.io/badges/@logan/libsql-search)](https://jsr.io/@logan/libsql-search)
+[![npm downloads](https://img.shields.io/npm/dm/libsql-search.svg)](https://www.npmjs.com/package/libsql-search)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![CI](https://github.com/llbbl/libsql-search/actions/workflows/ci.yml/badge.svg)](https://github.com/llbbl/libsql-search/actions/workflows/ci.yml)
+
 Semantic search for static sites using libSQL/Turso with multi-provider embeddings.
 
 Add AI-powered vector search to your Astro, Next.js, or any static site with minimal configuration. Index markdown content, generate embeddings locally or via API, and provide lightning-fast semantic search to your users.
@@ -27,7 +33,7 @@ pnpm add libsql-search @libsql/client
 
 **JSR:**
 ```bash
-deno add @llbbl/libsql-search
+deno add @logan/libsql-search
 ```
 
 ## Quick Start

package/dist/index.cjs

@@ -0,0 +1,354 @@
+'use strict';
+
+var promises = require('fs/promises');
+var path = require('path');
+var matter = require('gray-matter');
+
+const providerCache = {};
+function getEnvironmentVariable(name) {
+  const runtime = globalThis;
+  const nodeValue = runtime.process?.env?.[name];
+  if (nodeValue) {
+    return nodeValue;
+  }
+  try {
+    return runtime.Deno?.env?.get?.(name);
+  } catch {
+    return void 0;
+  }
+}
+async function getLocalEmbeddingModel() {
+  if (!providerCache.local) {
+    console.log("Loading local embedding model (Xenova/all-MiniLM-L6-v2)...");
+    const { pipeline } = await import('@xenova/transformers');
+    providerCache.local = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2");
+    console.log("Local model loaded successfully");
+  }
+  return providerCache.local;
+}
+async function generateEmbedding(text, options = {}) {
+  const {
+    provider = "local",
+    apiKey,
+    dimensions = 768,
+    maxLength = 8e3
+  } = options;
+  const truncated = text.substring(0, maxLength);
+  switch (provider) {
+    case "local":
+      return generateLocalEmbedding(truncated, dimensions);
+    case "gemini":
+      return generateGeminiEmbedding(truncated, apiKey);
+    case "openai":
+      return generateOpenAIEmbedding(truncated, apiKey, dimensions);
+    default:
+      throw new Error(`Unknown embedding provider: ${provider}`);
+  }
+}
+async function generateLocalEmbedding(text, targetDimensions) {
+  const model = await getLocalEmbeddingModel();
+  const output = await model(text, {
+    pooling: "mean",
+    normalize: true
+  });
+  const embedding = Array.from(output.data);
+  return padEmbedding(embedding, targetDimensions);
+}
+async function generateGeminiEmbedding(text, apiKey) {
+  const key = apiKey || getEnvironmentVariable("GEMINI_API_KEY");
+  if (!key) {
+    throw new Error("GEMINI_API_KEY is required for Gemini embeddings");
+  }
+  if (!providerCache.gemini) {
+    const { GoogleGenerativeAI } = await import('@google/generative-ai');
+    const genAI = new GoogleGenerativeAI(key);
+    providerCache.gemini = genAI.getGenerativeModel({ model: "text-embedding-004" });
+  }
+  const result = await providerCache.gemini.embedContent(text);
+  return result.embedding.values;
+}
+async function generateOpenAIEmbedding(text, apiKey, dimensions = 1536) {
+  const key = apiKey || getEnvironmentVariable("OPENAI_API_KEY");
+  if (!key) {
+    throw new Error("OPENAI_API_KEY is required for OpenAI embeddings");
+  }
+  const model = dimensions <= 1536 ? "text-embedding-3-small" : "text-embedding-3-large";
+  const response = await fetch("https://api.openai.com/v1/embeddings", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Authorization": `Bearer ${key}`
+    },
+    body: JSON.stringify({
+      input: text,
+      model,
+      dimensions
+    })
+  });
+  if (!response.ok) {
+    const error = await response.text();
+    throw new Error(`OpenAI API error: ${error}`);
+  }
+  const data = await response.json();
+  return data.data[0].embedding;
+}
+function padEmbedding(embedding, targetDimensions) {
+  if (embedding.length === targetDimensions) {
+    return embedding;
+  }
+  if (embedding.length > targetDimensions) {
+    return embedding.slice(0, targetDimensions);
+  }
+  const padded = new Array(targetDimensions).fill(0);
+  padded.splice(0, embedding.length, ...embedding);
+  return padded;
+}
+function prepareTextForEmbedding(fields) {
+  const parts = [];
+  if (fields.title) parts.push(fields.title);
+  if (fields.description) parts.push(fields.description);
+  if (fields.tags && fields.tags.length > 0) {
+    parts.push(`Tags: ${fields.tags.join(", ")}`);
+  }
+  if (fields.content) parts.push(fields.content);
+  return parts.filter(Boolean).join("\n\n");
+}
+
+async function indexContent(options) {
+  const {
+    client,
+    contentPath,
+    embeddingOptions = {},
+    fileExtensions = [".md", ".markdown"],
+    exclude = ["node_modules", ".git", "dist", "build"],
+    tableName = "articles",
+    onProgress
+  } = options;
+  const files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
+  if (files.length === 0) {
+    console.warn(`No files found in ${contentPath}`);
+    return { success: 0, failed: 0, total: 0 };
+  }
+  await client.execute(`DELETE FROM ${tableName}`);
+  let success = 0;
+  let failed = 0;
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+    if (onProgress) {
+      onProgress(i + 1, files.length, file.relativePath);
+    }
+    try {
+      const document = await processFile(file, embeddingOptions);
+      await insertDocument(client, document, tableName);
+      success++;
+    } catch (error) {
+      console.error(`Failed to index ${file.relativePath}:`, error);
+      failed++;
+    }
+  }
+  return { success, failed, total: files.length };
+}
+async function findFiles(dir, baseDir, extensions, exclude) {
+  const files = [];
+  const entries = await promises.readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (!entry.name.startsWith(".") && !exclude.includes(entry.name)) {
+        const subFiles = await findFiles(fullPath, baseDir, extensions, exclude);
+        files.push(...subFiles);
+      }
+    } else if (extensions.includes(path.extname(entry.name))) {
+      const relativePath = path.relative(baseDir, fullPath);
+      const folder = path.dirname(relativePath);
+      files.push({
+        fullPath,
+        relativePath,
+        folder: folder === "." ? "root" : folder
+      });
+    }
+  }
+  return files;
+}
+async function processFile(file, embeddingOptions) {
+  const content = await promises.readFile(file.fullPath, "utf-8");
+  const { data: frontMatter, content: markdown } = matter(content);
+  const slug = file.relativePath.replace(/\.(md|markdown)$/, "").replace(/\\/g, "/");
+  const title = frontMatter.title || file.relativePath.split("/").pop()?.replace(/\.(md|markdown)$/, "").replace(/-/g, " ") || "Untitled";
+  const tags = Array.isArray(frontMatter.tags) ? frontMatter.tags : [];
+  const embeddingText = prepareTextForEmbedding({
+    title,
+    description: frontMatter.description,
+    content: markdown,
+    tags
+  });
+  const embedding = await generateEmbedding(embeddingText, embeddingOptions);
+  return {
+    slug,
+    title,
+    content: markdown,
+    folder: file.folder,
+    tags,
+    embedding,
+    metadata: frontMatter
+  };
+}
+async function insertDocument(client, document, tableName) {
+  await client.execute({
+    sql: `INSERT INTO ${tableName}
+          (slug, title, content, folder, tags, embedding, created_at, updated_at)
+          VALUES (?, ?, ?, ?, ?, vector(?), datetime('now'), datetime('now'))`,
+    args: [
+      document.slug,
+      document.title,
+      document.content,
+      document.folder,
+      JSON.stringify(document.tags),
+      JSON.stringify(document.embedding)
+    ]
+  });
+}
+async function createTable(client, tableName = "articles", dimensions = 768) {
+  await client.execute(`
+    CREATE TABLE IF NOT EXISTS ${tableName} (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      slug TEXT UNIQUE NOT NULL,
+      title TEXT NOT NULL,
+      content TEXT NOT NULL,
+      folder TEXT NOT NULL DEFAULT 'root',
+      tags TEXT DEFAULT '[]',
+      embedding F32_BLOB(${dimensions}),
+      created_at TEXT NOT NULL,
+      updated_at TEXT NOT NULL
+    )
+  `);
+  await client.execute(`
+    CREATE INDEX IF NOT EXISTS ${tableName}_embedding_idx
+    ON ${tableName}(libsql_vector_idx(embedding))
+  `);
+  await client.execute(`
+    CREATE INDEX IF NOT EXISTS ${tableName}_folder_idx
+    ON ${tableName}(folder)
+  `);
+  await client.execute(`
+    CREATE INDEX IF NOT EXISTS ${tableName}_slug_idx
+    ON ${tableName}(slug)
+  `);
+}
+
+async function search(options) {
+  const {
+    client,
+    query,
+    limit = 10,
+    tableName = "articles",
+    embeddingOptions = {}
+  } = options;
+  const queryEmbedding = await generateEmbedding(query, embeddingOptions);
+  const results = await client.execute({
+    sql: `
+      SELECT
+        id,
+        slug,
+        title,
+        content,
+        folder,
+        tags,
+        created_at,
+        vector_distance_cos(embedding, vector(?)) as distance
+      FROM ${tableName}
+      WHERE embedding IS NOT NULL
+      ORDER BY distance
+      LIMIT ?
+    `,
+    args: [JSON.stringify(queryEmbedding), limit]
+  });
+  return results.rows.map((row) => ({
+    id: row.id,
+    slug: row.slug,
+    title: row.title,
+    content: row.content,
+    folder: row.folder,
+    tags: JSON.parse(row.tags || "[]"),
+    distance: row.distance,
+    created_at: row.created_at
+  }));
+}
+async function getAllArticles(client, tableName = "articles") {
+  const results = await client.execute(`
+    SELECT id, slug, title, folder, tags, created_at, updated_at
+    FROM ${tableName}
+    ORDER BY title
+  `);
+  return results.rows.map((row) => ({
+    id: row.id,
+    slug: row.slug,
+    title: row.title,
+    folder: row.folder,
+    tags: JSON.parse(row.tags || "[]"),
+    created_at: row.created_at,
+    updated_at: row.updated_at
+  }));
+}
+async function getArticleBySlug(client, slug, tableName = "articles") {
+  const results = await client.execute({
+    sql: `
+      SELECT id, slug, title, content, folder, tags, created_at, updated_at
+      FROM ${tableName}
+      WHERE slug = ?
+      LIMIT 1
+    `,
+    args: [slug]
+  });
+  if (results.rows.length === 0) {
+    return null;
+  }
+  const row = results.rows[0];
+  return {
+    id: row.id,
+    slug: row.slug,
+    title: row.title,
+    content: row.content,
+    folder: row.folder,
+    tags: JSON.parse(row.tags || "[]"),
+    created_at: row.created_at,
+    updated_at: row.updated_at
+  };
+}
+async function getArticlesByFolder(client, folder, tableName = "articles") {
+  const results = await client.execute({
+    sql: `
+      SELECT id, slug, title, folder, tags
+      FROM ${tableName}
+      WHERE folder = ?
+      ORDER BY title
+    `,
+    args: [folder]
+  });
+  return results.rows.map((row) => ({
+    id: row.id,
+    slug: row.slug,
+    title: row.title,
+    folder: row.folder,
+    tags: JSON.parse(row.tags || "[]")
+  }));
+}
+async function getFolders(client, tableName = "articles") {
+  const results = await client.execute(`
+    SELECT DISTINCT folder
+    FROM ${tableName}
+    ORDER BY folder
+  `);
+  return results.rows.map((row) => row.folder);
+}
+
+exports.createTable = createTable;
+exports.generateEmbedding = generateEmbedding;
+exports.getAllArticles = getAllArticles;
+exports.getArticleBySlug = getArticleBySlug;
+exports.getArticlesByFolder = getArticlesByFolder;
+exports.getFolders = getFolders;
+exports.indexContent = indexContent;
+exports.padEmbedding = padEmbedding;
+exports.prepareTextForEmbedding = prepareTextForEmbedding;
+exports.search = search;

package/dist/index.d.ts

@@ -1,9 +1,5 @@
 import { Client } from '@libsql/client';
 
-/**
- * Multi-provider embedding generation
- * Supports local (Xenova), Gemini, and OpenAI
- */
 type EmbeddingProvider = 'local' | 'gemini' | 'openai';
 interface EmbeddingOptions {
     provider?: EmbeddingProvider;