libsql-search 0.1.2

package/dist/index.esm.js

@@ -0,0 +1,389 @@
+import { pipeline } from '@xenova/transformers';
+import { readdir, readFile } from 'fs/promises';
+import { join, extname, relative, dirname } from 'path';
+import matter from 'gray-matter';
+
+/**
+ * Multi-provider embedding generation
+ * Supports local (Xenova), Gemini, and OpenAI
+ */
+const providerCache = {};
+/**
+ * Generate embeddings using the specified provider
+ */
+async function generateEmbedding(text, options = {}) {
+    const { provider = 'local', apiKey, dimensions = 768, maxLength = 8000 } = options;
+    const truncated = text.substring(0, maxLength);
+    switch (provider) {
+        case 'local':
+            return generateLocalEmbedding(truncated, dimensions);
+        case 'gemini':
+            return generateGeminiEmbedding(truncated, apiKey);
+        case 'openai':
+            return generateOpenAIEmbedding(truncated, apiKey, dimensions);
+        default:
+            throw new Error(`Unknown embedding provider: ${provider}`);
+    }
+}
+/**
+ * Generate embeddings using local model (Xenova/all-MiniLM-L6-v2)
+ * Returns 384 dimensions, padded to target dimensions
+ */
+async function generateLocalEmbedding(text, targetDimensions) {
+    if (!providerCache.local) {
+        console.log('Loading local embedding model (Xenova/all-MiniLM-L6-v2)...');
+        providerCache.local = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+        console.log('Local model loaded successfully');
+    }
+    const output = await providerCache.local(text, {
+        pooling: 'mean',
+        normalize: true
+    });
+    const embedding = Array.from(output.data);
+    return padEmbedding(embedding, targetDimensions);
+}
+/**
+ * Generate embeddings using Google Gemini API
+ * Returns 768 dimensions natively
+ */
+async function generateGeminiEmbedding(text, apiKey) {
+    const key = apiKey || process.env.GEMINI_API_KEY;
+    if (!key) {
+        throw new Error('GEMINI_API_KEY is required for Gemini embeddings');
+    }
+    if (!providerCache.gemini) {
+        // Dynamic import to keep it optional
+        const { GoogleGenerativeAI } = await import('@google/generative-ai');
+        const genAI = new GoogleGenerativeAI(key);
+        providerCache.gemini = genAI.getGenerativeModel({ model: 'text-embedding-004' });
+    }
+    const result = await providerCache.gemini.embedContent(text);
+    return result.embedding.values;
+}
+/**
+ * Generate embeddings using OpenAI API
+ * Supports text-embedding-3-small (1536 dims) and text-embedding-3-large (3072 dims)
+ */
+async function generateOpenAIEmbedding(text, apiKey, dimensions = 1536) {
+    const key = apiKey || process.env.OPENAI_API_KEY;
+    if (!key) {
+        throw new Error('OPENAI_API_KEY is required for OpenAI embeddings');
+    }
+    const model = dimensions <= 1536 ? 'text-embedding-3-small' : 'text-embedding-3-large';
+    const response = await fetch('https://api.openai.com/v1/embeddings', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${key}`
+        },
+        body: JSON.stringify({
+            input: text,
+            model,
+            dimensions
+        })
+    });
+    if (!response.ok) {
+        const error = await response.text();
+        throw new Error(`OpenAI API error: ${error}`);
+    }
+    const data = await response.json();
+    return data.data[0].embedding;
+}
+/**
+ * Pad or truncate embedding to target dimensions
+ */
+function padEmbedding(embedding, targetDimensions) {
+    if (embedding.length === targetDimensions) {
+        return embedding;
+    }
+    if (embedding.length > targetDimensions) {
+        return embedding.slice(0, targetDimensions);
+    }
+    const padded = new Array(targetDimensions).fill(0);
+    padded.splice(0, embedding.length, ...embedding);
+    return padded;
+}
+/**
+ * Prepare text for embedding by combining multiple fields
+ */
+function prepareTextForEmbedding(fields) {
+    const parts = [];
+    if (fields.title)
+        parts.push(fields.title);
+    if (fields.description)
+        parts.push(fields.description);
+    if (fields.tags && fields.tags.length > 0) {
+        parts.push(`Tags: ${fields.tags.join(', ')}`);
+    }
+    if (fields.content)
+        parts.push(fields.content);
+    return parts.filter(Boolean).join('\n\n');
+}
+
+/**
+ * Content indexer for markdown and other formats
+ */
+/**
+ * Index markdown content from a directory
+ */
+async function indexContent(options) {
+    const { client, contentPath, embeddingOptions = {}, fileExtensions = ['.md', '.markdown'], exclude = ['node_modules', '.git', 'dist', 'build'], tableName = 'articles', onProgress } = options;
+    // Find all content files
+    const files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
+    if (files.length === 0) {
+        console.warn(`No files found in ${contentPath}`);
+        return { success: 0, failed: 0, total: 0 };
+    }
+    // Clear existing content
+    await client.execute(`DELETE FROM ${tableName}`);
+    // Process each file
+    let success = 0;
+    let failed = 0;
+    for (let i = 0; i < files.length; i++) {
+        const file = files[i];
+        if (onProgress) {
+            onProgress(i + 1, files.length, file.relativePath);
+        }
+        try {
+            const document = await processFile(file, embeddingOptions);
+            await insertDocument(client, document, tableName);
+            success++;
+        }
+        catch (error) {
+            console.error(`Failed to index ${file.relativePath}:`, error);
+            failed++;
+        }
+    }
+    return { success, failed, total: files.length };
+}
+/**
+ * Find all files matching extensions
+ */
+async function findFiles(dir, baseDir, extensions, exclude) {
+    const files = [];
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const fullPath = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            if (!entry.name.startsWith('.') && !exclude.includes(entry.name)) {
+                const subFiles = await findFiles(fullPath, baseDir, extensions, exclude);
+                files.push(...subFiles);
+            }
+        }
+        else if (extensions.includes(extname(entry.name))) {
+            const relativePath = relative(baseDir, fullPath);
+            const folder = dirname(relativePath);
+            files.push({
+                fullPath,
+                relativePath,
+                folder: folder === '.' ? 'root' : folder
+            });
+        }
+    }
+    return files;
+}
+/**
+ * Process a single file into an indexed document
+ */
+async function processFile(file, embeddingOptions) {
+    const content = await readFile(file.fullPath, 'utf-8');
+    const { data: frontMatter, content: markdown } = matter(content);
+    // Generate slug from relative path
+    const slug = file.relativePath
+        .replace(/\.(md|markdown)$/, '')
+        .replace(/\\/g, '/');
+    // Extract metadata
+    const title = frontMatter.title || file.relativePath
+        .split('/').pop()
+        ?.replace(/\.(md|markdown)$/, '')
+        .replace(/-/g, ' ') || 'Untitled';
+    const tags = Array.isArray(frontMatter.tags) ? frontMatter.tags : [];
+    // Generate embedding
+    const embeddingText = prepareTextForEmbedding({
+        title,
+        description: frontMatter.description,
+        content: markdown,
+        tags
+    });
+    const embedding = await generateEmbedding(embeddingText, embeddingOptions);
+    return {
+        slug,
+        title,
+        content: markdown,
+        folder: file.folder,
+        tags,
+        embedding,
+        metadata: frontMatter
+    };
+}
+/**
+ * Insert document into database
+ */
+async function insertDocument(client, document, tableName) {
+    await client.execute({
+        sql: `INSERT INTO ${tableName}
+          (slug, title, content, folder, tags, embedding, created_at, updated_at)
+          VALUES (?, ?, ?, ?, ?, vector(?), datetime('now'), datetime('now'))`,
+        args: [
+            document.slug,
+            document.title,
+            document.content,
+            document.folder,
+            JSON.stringify(document.tags),
+            JSON.stringify(document.embedding)
+        ]
+    });
+}
+/**
+ * Create the articles table if it doesn't exist
+ */
+async function createTable(client, tableName = 'articles', dimensions = 768) {
+    await client.execute(`
+    CREATE TABLE IF NOT EXISTS ${tableName} (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      slug TEXT UNIQUE NOT NULL,
+      title TEXT NOT NULL,
+      content TEXT NOT NULL,
+      folder TEXT NOT NULL DEFAULT 'root',
+      tags TEXT DEFAULT '[]',
+      embedding F32_BLOB(${dimensions}),
+      created_at TEXT NOT NULL,
+      updated_at TEXT NOT NULL
+    )
+  `);
+    await client.execute(`
+    CREATE INDEX IF NOT EXISTS ${tableName}_embedding_idx
+    ON ${tableName}(libsql_vector_idx(embedding))
+  `);
+    await client.execute(`
+    CREATE INDEX IF NOT EXISTS ${tableName}_folder_idx
+    ON ${tableName}(folder)
+  `);
+    await client.execute(`
+    CREATE INDEX IF NOT EXISTS ${tableName}_slug_idx
+    ON ${tableName}(slug)
+  `);
+}
+
+/**
+ * Vector search functionality
+ */
+/**
+ * Perform semantic search using vector similarity
+ */
+async function search(options) {
+    const { client, query, limit = 10, tableName = 'articles', embeddingOptions = {} } = options;
+    // Generate embedding for query
+    const queryEmbedding = await generateEmbedding(query, embeddingOptions);
+    // Perform vector search
+    const results = await client.execute({
+        sql: `
+      SELECT
+        id,
+        slug,
+        title,
+        content,
+        folder,
+        tags,
+        created_at,
+        vector_distance_cos(embedding, vector(?)) as distance
+      FROM ${tableName}
+      WHERE embedding IS NOT NULL
+      ORDER BY distance
+      LIMIT ?
+    `,
+        args: [JSON.stringify(queryEmbedding), limit]
+    });
+    // Parse and format results
+    return results.rows.map(row => ({
+        id: row.id,
+        slug: row.slug,
+        title: row.title,
+        content: row.content,
+        folder: row.folder,
+        tags: JSON.parse(row.tags || '[]'),
+        distance: row.distance,
+        created_at: row.created_at
+    }));
+}
+/**
+ * Get all articles (for building static pages, navigation, etc.)
+ */
+async function getAllArticles(client, tableName = 'articles') {
+    const results = await client.execute(`
+    SELECT id, slug, title, folder, tags, created_at, updated_at
+    FROM ${tableName}
+    ORDER BY title
+  `);
+    return results.rows.map(row => ({
+        id: row.id,
+        slug: row.slug,
+        title: row.title,
+        folder: row.folder,
+        tags: JSON.parse(row.tags || '[]'),
+        created_at: row.created_at,
+        updated_at: row.updated_at
+    }));
+}
+/**
+ * Get a single article by slug
+ */
+async function getArticleBySlug(client, slug, tableName = 'articles') {
+    const results = await client.execute({
+        sql: `
+      SELECT id, slug, title, content, folder, tags, created_at, updated_at
+      FROM ${tableName}
+      WHERE slug = ?
+      LIMIT 1
+    `,
+        args: [slug]
+    });
+    if (results.rows.length === 0) {
+        return null;
+    }
+    const row = results.rows[0];
+    return {
+        id: row.id,
+        slug: row.slug,
+        title: row.title,
+        content: row.content,
+        folder: row.folder,
+        tags: JSON.parse(row.tags || '[]'),
+        created_at: row.created_at,
+        updated_at: row.updated_at
+    };
+}
+/**
+ * Get articles by folder
+ */
+async function getArticlesByFolder(client, folder, tableName = 'articles') {
+    const results = await client.execute({
+        sql: `
+      SELECT id, slug, title, folder, tags
+      FROM ${tableName}
+      WHERE folder = ?
+      ORDER BY title
+    `,
+        args: [folder]
+    });
+    return results.rows.map(row => ({
+        id: row.id,
+        slug: row.slug,
+        title: row.title,
+        folder: row.folder,
+        tags: JSON.parse(row.tags || '[]')
+    }));
+}
+/**
+ * Get all unique folders
+ */
+async function getFolders(client, tableName = 'articles') {
+    const results = await client.execute(`
+    SELECT DISTINCT folder
+    FROM ${tableName}
+    ORDER BY folder
+  `);
+    return results.rows.map(row => row.folder);
+}
+
+export { createTable, generateEmbedding, getAllArticles, getArticleBySlug, getArticlesByFolder, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search };