libsql-search 0.1.2

package/dist/index.js

@@ -0,0 +1,400 @@
+'use strict';
+
+var transformers = require('@xenova/transformers');
+var promises = require('fs/promises');
+var path = require('path');
+var matter = require('gray-matter');
+
+/**
+ * Multi-provider embedding generation
+ * Supports local (Xenova), Gemini, and OpenAI
+ */
+const providerCache = {};
+/**
+ * Generate embeddings using the specified provider
+ */
+async function generateEmbedding(text, options = {}) {
+    const { provider = 'local', apiKey, dimensions = 768, maxLength = 8000 } = options;
+    const truncated = text.substring(0, maxLength);
+    switch (provider) {
+        case 'local':
+            return generateLocalEmbedding(truncated, dimensions);
+        case 'gemini':
+            return generateGeminiEmbedding(truncated, apiKey);
+        case 'openai':
+            return generateOpenAIEmbedding(truncated, apiKey, dimensions);
+        default:
+            throw new Error(`Unknown embedding provider: ${provider}`);
+    }
+}
+/**
+ * Generate embeddings using local model (Xenova/all-MiniLM-L6-v2)
+ * Returns 384 dimensions, padded to target dimensions
+ */
+async function generateLocalEmbedding(text, targetDimensions) {
+    if (!providerCache.local) {
+        console.log('Loading local embedding model (Xenova/all-MiniLM-L6-v2)...');
+        providerCache.local = await transformers.pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+        console.log('Local model loaded successfully');
+    }
+    const output = await providerCache.local(text, {
+        pooling: 'mean',
+        normalize: true
+    });
+    const embedding = Array.from(output.data);
+    return padEmbedding(embedding, targetDimensions);
+}
+/**
+ * Generate embeddings using Google Gemini API
+ * Returns 768 dimensions natively
+ */
+async function generateGeminiEmbedding(text, apiKey) {
+    const key = apiKey || process.env.GEMINI_API_KEY;
+    if (!key) {
+        throw new Error('GEMINI_API_KEY is required for Gemini embeddings');
+    }
+    if (!providerCache.gemini) {
+        // Dynamic import to keep it optional
+        const { GoogleGenerativeAI } = await import('@google/generative-ai');
+        const genAI = new GoogleGenerativeAI(key);
+        providerCache.gemini = genAI.getGenerativeModel({ model: 'text-embedding-004' });
+    }
+    const result = await providerCache.gemini.embedContent(text);
+    return result.embedding.values;
+}
+/**
+ * Generate embeddings using OpenAI API
+ * Supports text-embedding-3-small (1536 dims) and text-embedding-3-large (3072 dims)
+ */
+async function generateOpenAIEmbedding(text, apiKey, dimensions = 1536) {
+    const key = apiKey || process.env.OPENAI_API_KEY;
+    if (!key) {
+        throw new Error('OPENAI_API_KEY is required for OpenAI embeddings');
+    }
+    const model = dimensions <= 1536 ? 'text-embedding-3-small' : 'text-embedding-3-large';
+    const response = await fetch('https://api.openai.com/v1/embeddings', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${key}`
+        },
+        body: JSON.stringify({
+            input: text,
+            model,
+            dimensions
+        })
+    });
+    if (!response.ok) {
+        const error = await response.text();
+        throw new Error(`OpenAI API error: ${error}`);
+    }
+    const data = await response.json();
+    return data.data[0].embedding;
+}
+/**
+ * Pad or truncate embedding to target dimensions
+ */
+function padEmbedding(embedding, targetDimensions) {
+    if (embedding.length === targetDimensions) {
+        return embedding;
+    }
+    if (embedding.length > targetDimensions) {
+        return embedding.slice(0, targetDimensions);
+    }
+    const padded = new Array(targetDimensions).fill(0);
+    padded.splice(0, embedding.length, ...embedding);
+    return padded;
+}
+/**
+ * Prepare text for embedding by combining multiple fields
+ */
+function prepareTextForEmbedding(fields) {
+    const parts = [];
+    if (fields.title)
+        parts.push(fields.title);
+    if (fields.description)
+        parts.push(fields.description);
+    if (fields.tags && fields.tags.length > 0) {
+        parts.push(`Tags: ${fields.tags.join(', ')}`);
+    }
+    if (fields.content)
+        parts.push(fields.content);
+    return parts.filter(Boolean).join('\n\n');
+}
+
+/**
+ * Content indexer for markdown and other formats
+ */
+/**
+ * Index markdown content from a directory
+ */
+async function indexContent(options) {
+    const { client, contentPath, embeddingOptions = {}, fileExtensions = ['.md', '.markdown'], exclude = ['node_modules', '.git', 'dist', 'build'], tableName = 'articles', onProgress } = options;
+    // Find all content files
+    const files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
+    if (files.length === 0) {
+        console.warn(`No files found in ${contentPath}`);
+        return { success: 0, failed: 0, total: 0 };
+    }
+    // Clear existing content
+    await client.execute(`DELETE FROM ${tableName}`);
+    // Process each file
+    let success = 0;
+    let failed = 0;
+    for (let i = 0; i < files.length; i++) {
+        const file = files[i];
+        if (onProgress) {
+            onProgress(i + 1, files.length, file.relativePath);
+        }
+        try {
+            const document = await processFile(file, embeddingOptions);
+            await insertDocument(client, document, tableName);
+            success++;
+        }
+        catch (error) {
+            console.error(`Failed to index ${file.relativePath}:`, error);
+            failed++;
+        }
+    }
+    return { success, failed, total: files.length };
+}
+/**
+ * Find all files matching extensions
+ */
+async function findFiles(dir, baseDir, extensions, exclude) {
+    const files = [];
+    const entries = await promises.readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            if (!entry.name.startsWith('.') && !exclude.includes(entry.name)) {
+                const subFiles = await findFiles(fullPath, baseDir, extensions, exclude);
+                files.push(...subFiles);
+            }
+        }
+        else if (extensions.includes(path.extname(entry.name))) {
+            const relativePath = path.relative(baseDir, fullPath);
+            const folder = path.dirname(relativePath);
+            files.push({
+                fullPath,
+                relativePath,
+                folder: folder === '.' ? 'root' : folder
+            });
+        }
+    }
+    return files;
+}
+/**
+ * Process a single file into an indexed document
+ */
+async function processFile(file, embeddingOptions) {
+    const content = await promises.readFile(file.fullPath, 'utf-8');
+    const { data: frontMatter, content: markdown } = matter(content);
+    // Generate slug from relative path
+    const slug = file.relativePath
+        .replace(/\.(md|markdown)$/, '')
+        .replace(/\\/g, '/');
+    // Extract metadata
+    const title = frontMatter.title || file.relativePath
+        .split('/').pop()
+        ?.replace(/\.(md|markdown)$/, '')
+        .replace(/-/g, ' ') || 'Untitled';
+    const tags = Array.isArray(frontMatter.tags) ? frontMatter.tags : [];
+    // Generate embedding
+    const embeddingText = prepareTextForEmbedding({
+        title,
+        description: frontMatter.description,
+        content: markdown,
+        tags
+    });
+    const embedding = await generateEmbedding(embeddingText, embeddingOptions);
+    return {
+        slug,
+        title,
+        content: markdown,
+        folder: file.folder,
+        tags,
+        embedding,
+        metadata: frontMatter
+    };
+}
+/**
+ * Insert document into database
+ */
+async function insertDocument(client, document, tableName) {
+    await client.execute({
+        sql: `INSERT INTO ${tableName}
+          (slug, title, content, folder, tags, embedding, created_at, updated_at)
+          VALUES (?, ?, ?, ?, ?, vector(?), datetime('now'), datetime('now'))`,
+        args: [
+            document.slug,
+            document.title,
+            document.content,
+            document.folder,
+            JSON.stringify(document.tags),
+            JSON.stringify(document.embedding)
+        ]
+    });
+}
+/**
+ * Create the articles table if it doesn't exist
+ */
+async function createTable(client, tableName = 'articles', dimensions = 768) {
+    await client.execute(`
+    CREATE TABLE IF NOT EXISTS ${tableName} (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      slug TEXT UNIQUE NOT NULL,
+      title TEXT NOT NULL,
+      content TEXT NOT NULL,
+      folder TEXT NOT NULL DEFAULT 'root',
+      tags TEXT DEFAULT '[]',
+      embedding F32_BLOB(${dimensions}),
+      created_at TEXT NOT NULL,
+      updated_at TEXT NOT NULL
+    )
+  `);
+    await client.execute(`
+    CREATE INDEX IF NOT EXISTS ${tableName}_embedding_idx
+    ON ${tableName}(libsql_vector_idx(embedding))
+  `);
+    await client.execute(`
+    CREATE INDEX IF NOT EXISTS ${tableName}_folder_idx
+    ON ${tableName}(folder)
+  `);
+    await client.execute(`
+    CREATE INDEX IF NOT EXISTS ${tableName}_slug_idx
+    ON ${tableName}(slug)
+  `);
+}
+
+/**
+ * Vector search functionality
+ */
+/**
+ * Perform semantic search using vector similarity
+ */
+async function search(options) {
+    const { client, query, limit = 10, tableName = 'articles', embeddingOptions = {} } = options;
+    // Generate embedding for query
+    const queryEmbedding = await generateEmbedding(query, embeddingOptions);
+    // Perform vector search
+    const results = await client.execute({
+        sql: `
+      SELECT
+        id,
+        slug,
+        title,
+        content,
+        folder,
+        tags,
+        created_at,
+        vector_distance_cos(embedding, vector(?)) as distance
+      FROM ${tableName}
+      WHERE embedding IS NOT NULL
+      ORDER BY distance
+      LIMIT ?
+    `,
+        args: [JSON.stringify(queryEmbedding), limit]
+    });
+    // Parse and format results
+    return results.rows.map(row => ({
+        id: row.id,
+        slug: row.slug,
+        title: row.title,
+        content: row.content,
+        folder: row.folder,
+        tags: JSON.parse(row.tags || '[]'),
+        distance: row.distance,
+        created_at: row.created_at
+    }));
+}
+/**
+ * Get all articles (for building static pages, navigation, etc.)
+ */
+async function getAllArticles(client, tableName = 'articles') {
+    const results = await client.execute(`
+    SELECT id, slug, title, folder, tags, created_at, updated_at
+    FROM ${tableName}
+    ORDER BY title
+  `);
+    return results.rows.map(row => ({
+        id: row.id,
+        slug: row.slug,
+        title: row.title,
+        folder: row.folder,
+        tags: JSON.parse(row.tags || '[]'),
+        created_at: row.created_at,
+        updated_at: row.updated_at
+    }));
+}
+/**
+ * Get a single article by slug
+ */
+async function getArticleBySlug(client, slug, tableName = 'articles') {
+    const results = await client.execute({
+        sql: `
+      SELECT id, slug, title, content, folder, tags, created_at, updated_at
+      FROM ${tableName}
+      WHERE slug = ?
+      LIMIT 1
+    `,
+        args: [slug]
+    });
+    if (results.rows.length === 0) {
+        return null;
+    }
+    const row = results.rows[0];
+    return {
+        id: row.id,
+        slug: row.slug,
+        title: row.title,
+        content: row.content,
+        folder: row.folder,
+        tags: JSON.parse(row.tags || '[]'),
+        created_at: row.created_at,
+        updated_at: row.updated_at
+    };
+}
+/**
+ * Get articles by folder
+ */
+async function getArticlesByFolder(client, folder, tableName = 'articles') {
+    const results = await client.execute({
+        sql: `
+      SELECT id, slug, title, folder, tags
+      FROM ${tableName}
+      WHERE folder = ?
+      ORDER BY title
+    `,
+        args: [folder]
+    });
+    return results.rows.map(row => ({
+        id: row.id,
+        slug: row.slug,
+        title: row.title,
+        folder: row.folder,
+        tags: JSON.parse(row.tags || '[]')
+    }));
+}
+/**
+ * Get all unique folders
+ */
+async function getFolders(client, tableName = 'articles') {
+    const results = await client.execute(`
+    SELECT DISTINCT folder
+    FROM ${tableName}
+    ORDER BY folder
+  `);
+    return results.rows.map(row => row.folder);
+}
+
+exports.createTable = createTable;
+exports.generateEmbedding = generateEmbedding;
+exports.getAllArticles = getAllArticles;
+exports.getArticleBySlug = getArticleBySlug;
+exports.getArticlesByFolder = getArticlesByFolder;
+exports.getFolders = getFolders;
+exports.indexContent = indexContent;
+exports.padEmbedding = padEmbedding;
+exports.prepareTextForEmbedding = prepareTextForEmbedding;
+exports.search = search;

package/package.json

@@ -0,0 +1,74 @@
+{
+  "name": "libsql-search",
+  "version": "0.1.2",
+  "description": "Semantic search for static sites using libSQL/Turso with multi-provider embeddings",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.esm.js",
+      "require": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "keywords": [
+    "libsql",
+    "turso",
+    "vector-search",
+    "semantic-search",
+    "embeddings",
+    "static-sites",
+    "markdown",
+    "astro",
+    "nextjs"
+  ],
+  "author": "llbbl",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/llbbl/libsql-search.git"
+  },
+  "bugs": {
+    "url": "https://github.com/llbbl/libsql-search/issues"
+  },
+  "homepage": "https://github.com/llbbl/libsql-search#readme",
+  "engines": {
+    "node": "^20.19.0 || >=22.12.0"
+  },
+  "peerDependencies": {
+    "@libsql/client": "^0.15.0"
+  },
+  "dependencies": {
+    "@xenova/transformers": "^2.17.2",
+    "gray-matter": "^4.0.3"
+  },
+  "optionalDependencies": {
+    "@google/generative-ai": "^0.24.1"
+  },
+  "devDependencies": {
+    "@libsql/client": "^0.15.15",
+    "@rollup/plugin-commonjs": "^28.0.7",
+    "@rollup/plugin-node-resolve": "^16.0.3",
+    "@rollup/plugin-typescript": "^12.1.4",
+    "@types/node": "^20.19.21",
+    "@vitest/ui": "^3.2.4",
+    "rollup": "^4.52.4",
+    "rollup-plugin-dts": "^6.2.3",
+    "tslib": "^2.8.1",
+    "typescript": "^5.9.3",
+    "vitest": "^3.2.4"
+  },
+  "scripts": {
+    "build": "rollup -c",
+    "dev": "rollup -c --watch",
+    "test": "vitest",
+    "test:ui": "vitest --ui",
+    "test:run": "vitest run",
+    "test:coverage": "vitest run --coverage"
+  }
+}