libsql-search 0.1.4 → 0.1.6

package/dist/index.cjs

@@ -114,6 +114,34 @@ function prepareTextForEmbedding(fields) {
   return parts.filter(Boolean).join("\n\n");
 }
 
+const SQL_IDENTIFIER_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
+const DEFAULT_SEARCH_LIMIT = 10;
+const MAX_SEARCH_LIMIT = 100;
+function validateSqlIdentifier(identifier, name = "identifier") {
+  if (typeof identifier !== "string" || !SQL_IDENTIFIER_PATTERN.test(identifier)) {
+    throw new Error(
+      `Invalid SQL ${name}: expected an ASCII identifier matching ${SQL_IDENTIFIER_PATTERN.toString()}`
+    );
+  }
+  return identifier;
+}
+function quoteSqlIdentifier(identifier, name) {
+  const validated = validateSqlIdentifier(identifier, name);
+  return `"${validated.replace(/"/g, '""')}"`;
+}
+function normalizeSearchLimit(limit = DEFAULT_SEARCH_LIMIT) {
+  if (typeof limit !== "number" || !Number.isFinite(limit) || !Number.isInteger(limit) || limit < 1 || limit > MAX_SEARCH_LIMIT) {
+    throw new Error(`Invalid search limit: expected an integer from 1 to ${MAX_SEARCH_LIMIT}`);
+  }
+  return limit;
+}
+function normalizeVectorDimensions(dimensions) {
+  if (typeof dimensions !== "number" || !Number.isFinite(dimensions) || !Number.isInteger(dimensions) || dimensions < 1) {
+    throw new Error("Invalid vector dimensions: expected a positive integer");
+  }
+  return dimensions;
+}
+
 async function indexContent(options) {
   const {
     client,
@@ -124,12 +152,13 @@ async function indexContent(options) {
     tableName = "articles",
     onProgress
   } = options;
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
   if (files.length === 0) {
     console.warn(`No files found in ${contentPath}`);
     return { success: 0, failed: 0, total: 0 };
   }
-  await client.execute(`DELETE FROM ${tableName}`);
+  await client.execute(`DELETE FROM ${quotedTableName}`);
   let success = 0;
   let failed = 0;
   for (let i = 0; i < files.length; i++) {
@@ -139,7 +168,7 @@ async function indexContent(options) {
     }
     try {
       const document = await processFile(file, embeddingOptions);
-      await insertDocument(client, document, tableName);
+      await insertDocument(client, document, quotedTableName);
       success++;
     } catch (error) {
       console.error(`Failed to index ${file.relativePath}:`, error);
@@ -193,9 +222,9 @@ async function processFile(file, embeddingOptions) {
     metadata: frontMatter
   };
 }
-async function insertDocument(client, document, tableName) {
+async function insertDocument(client, document, quotedTableName) {
   await client.execute({
-    sql: `INSERT INTO ${tableName}
+    sql: `INSERT INTO ${quotedTableName}
           (slug, title, content, folder, tags, embedding, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, vector(?), datetime('now'), datetime('now'))`,
     args: [
@@ -209,30 +238,35 @@ async function insertDocument(client, document, tableName) {
   });
 }
 async function createTable(client, tableName = "articles", dimensions = 768) {
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
+  const vectorDimensions = normalizeVectorDimensions(dimensions);
+  const quotedEmbeddingIndexName = quoteSqlIdentifier(`${tableName}_embedding_idx`, "embedding index name");
+  const quotedFolderIndexName = quoteSqlIdentifier(`${tableName}_folder_idx`, "folder index name");
+  const quotedSlugIndexName = quoteSqlIdentifier(`${tableName}_slug_idx`, "slug index name");
   await client.execute(`
-    CREATE TABLE IF NOT EXISTS ${tableName} (
+    CREATE TABLE IF NOT EXISTS ${quotedTableName} (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       slug TEXT UNIQUE NOT NULL,
       title TEXT NOT NULL,
       content TEXT NOT NULL,
       folder TEXT NOT NULL DEFAULT 'root',
       tags TEXT DEFAULT '[]',
-      embedding F32_BLOB(${dimensions}),
+      embedding F32_BLOB(${vectorDimensions}),
       created_at TEXT NOT NULL,
       updated_at TEXT NOT NULL
     )
   `);
   await client.execute(`
-    CREATE INDEX IF NOT EXISTS ${tableName}_embedding_idx
-    ON ${tableName}(libsql_vector_idx(embedding))
+    CREATE INDEX IF NOT EXISTS ${quotedEmbeddingIndexName}
+    ON ${quotedTableName}(libsql_vector_idx(embedding))
   `);
   await client.execute(`
-    CREATE INDEX IF NOT EXISTS ${tableName}_folder_idx
-    ON ${tableName}(folder)
+    CREATE INDEX IF NOT EXISTS ${quotedFolderIndexName}
+    ON ${quotedTableName}(folder)
   `);
   await client.execute(`
-    CREATE INDEX IF NOT EXISTS ${tableName}_slug_idx
-    ON ${tableName}(slug)
+    CREATE INDEX IF NOT EXISTS ${quotedSlugIndexName}
+    ON ${quotedTableName}(slug)
   `);
 }
 
@@ -244,6 +278,8 @@ async function search(options) {
     tableName = "articles",
     embeddingOptions = {}
   } = options;
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
+  const resultLimit = normalizeSearchLimit(limit);
   const queryEmbedding = await generateEmbedding(query, embeddingOptions);
   const results = await client.execute({
     sql: `
@@ -256,12 +292,12 @@ async function search(options) {
         tags,
         created_at,
         vector_distance_cos(embedding, vector(?)) as distance
-      FROM ${tableName}
+      FROM ${quotedTableName}
       WHERE embedding IS NOT NULL
       ORDER BY distance
       LIMIT ?
     `,
-    args: [JSON.stringify(queryEmbedding), limit]
+    args: [JSON.stringify(queryEmbedding), resultLimit]
   });
   return results.rows.map((row) => ({
     id: row.id,
@@ -275,9 +311,10 @@ async function search(options) {
   }));
 }
 async function getAllArticles(client, tableName = "articles") {
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const results = await client.execute(`
     SELECT id, slug, title, folder, tags, created_at, updated_at
-    FROM ${tableName}
+    FROM ${quotedTableName}
     ORDER BY title
   `);
   return results.rows.map((row) => ({
@@ -291,10 +328,11 @@ async function getAllArticles(client, tableName = "articles") {
   }));
 }
 async function getArticleBySlug(client, slug, tableName = "articles") {
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const results = await client.execute({
     sql: `
       SELECT id, slug, title, content, folder, tags, created_at, updated_at
-      FROM ${tableName}
+      FROM ${quotedTableName}
       WHERE slug = ?
       LIMIT 1
     `,
@@ -316,10 +354,11 @@ async function getArticleBySlug(client, slug, tableName = "articles") {
   };
 }
 async function getArticlesByFolder(client, folder, tableName = "articles") {
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const results = await client.execute({
     sql: `
       SELECT id, slug, title, folder, tags
-      FROM ${tableName}
+      FROM ${quotedTableName}
       WHERE folder = ?
       ORDER BY title
     `,
@@ -334,9 +373,10 @@ async function getArticlesByFolder(client, folder, tableName = "articles") {
   }));
 }
 async function getFolders(client, tableName = "articles") {
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const results = await client.execute(`
     SELECT DISTINCT folder
-    FROM ${tableName}
+    FROM ${quotedTableName}
     ORDER BY folder
   `);
   return results.rows.map((row) => row.folder);

package/dist/index.esm.js

@@ -112,6 +112,34 @@ function prepareTextForEmbedding(fields) {
   return parts.filter(Boolean).join("\n\n");
 }
 
+const SQL_IDENTIFIER_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
+const DEFAULT_SEARCH_LIMIT = 10;
+const MAX_SEARCH_LIMIT = 100;
+function validateSqlIdentifier(identifier, name = "identifier") {
+  if (typeof identifier !== "string" || !SQL_IDENTIFIER_PATTERN.test(identifier)) {
+    throw new Error(
+      `Invalid SQL ${name}: expected an ASCII identifier matching ${SQL_IDENTIFIER_PATTERN.toString()}`
+    );
+  }
+  return identifier;
+}
+function quoteSqlIdentifier(identifier, name) {
+  const validated = validateSqlIdentifier(identifier, name);
+  return `"${validated.replace(/"/g, '""')}"`;
+}
+function normalizeSearchLimit(limit = DEFAULT_SEARCH_LIMIT) {
+  if (typeof limit !== "number" || !Number.isFinite(limit) || !Number.isInteger(limit) || limit < 1 || limit > MAX_SEARCH_LIMIT) {
+    throw new Error(`Invalid search limit: expected an integer from 1 to ${MAX_SEARCH_LIMIT}`);
+  }
+  return limit;
+}
+function normalizeVectorDimensions(dimensions) {
+  if (typeof dimensions !== "number" || !Number.isFinite(dimensions) || !Number.isInteger(dimensions) || dimensions < 1) {
+    throw new Error("Invalid vector dimensions: expected a positive integer");
+  }
+  return dimensions;
+}
+
 async function indexContent(options) {
   const {
     client,
@@ -122,12 +150,13 @@ async function indexContent(options) {
     tableName = "articles",
     onProgress
   } = options;
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const files = await findFiles(contentPath, contentPath, fileExtensions, exclude);
   if (files.length === 0) {
     console.warn(`No files found in ${contentPath}`);
     return { success: 0, failed: 0, total: 0 };
   }
-  await client.execute(`DELETE FROM ${tableName}`);
+  await client.execute(`DELETE FROM ${quotedTableName}`);
   let success = 0;
   let failed = 0;
   for (let i = 0; i < files.length; i++) {
@@ -137,7 +166,7 @@ async function indexContent(options) {
     }
     try {
       const document = await processFile(file, embeddingOptions);
-      await insertDocument(client, document, tableName);
+      await insertDocument(client, document, quotedTableName);
       success++;
     } catch (error) {
       console.error(`Failed to index ${file.relativePath}:`, error);
@@ -191,9 +220,9 @@ async function processFile(file, embeddingOptions) {
     metadata: frontMatter
   };
 }
-async function insertDocument(client, document, tableName) {
+async function insertDocument(client, document, quotedTableName) {
   await client.execute({
-    sql: `INSERT INTO ${tableName}
+    sql: `INSERT INTO ${quotedTableName}
           (slug, title, content, folder, tags, embedding, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, vector(?), datetime('now'), datetime('now'))`,
     args: [
@@ -207,30 +236,35 @@ async function insertDocument(client, document, tableName) {
   });
 }
 async function createTable(client, tableName = "articles", dimensions = 768) {
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
+  const vectorDimensions = normalizeVectorDimensions(dimensions);
+  const quotedEmbeddingIndexName = quoteSqlIdentifier(`${tableName}_embedding_idx`, "embedding index name");
+  const quotedFolderIndexName = quoteSqlIdentifier(`${tableName}_folder_idx`, "folder index name");
+  const quotedSlugIndexName = quoteSqlIdentifier(`${tableName}_slug_idx`, "slug index name");
   await client.execute(`
-    CREATE TABLE IF NOT EXISTS ${tableName} (
+    CREATE TABLE IF NOT EXISTS ${quotedTableName} (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       slug TEXT UNIQUE NOT NULL,
       title TEXT NOT NULL,
       content TEXT NOT NULL,
       folder TEXT NOT NULL DEFAULT 'root',
       tags TEXT DEFAULT '[]',
-      embedding F32_BLOB(${dimensions}),
+      embedding F32_BLOB(${vectorDimensions}),
       created_at TEXT NOT NULL,
       updated_at TEXT NOT NULL
     )
   `);
   await client.execute(`
-    CREATE INDEX IF NOT EXISTS ${tableName}_embedding_idx
-    ON ${tableName}(libsql_vector_idx(embedding))
+    CREATE INDEX IF NOT EXISTS ${quotedEmbeddingIndexName}
+    ON ${quotedTableName}(libsql_vector_idx(embedding))
   `);
   await client.execute(`
-    CREATE INDEX IF NOT EXISTS ${tableName}_folder_idx
-    ON ${tableName}(folder)
+    CREATE INDEX IF NOT EXISTS ${quotedFolderIndexName}
+    ON ${quotedTableName}(folder)
   `);
   await client.execute(`
-    CREATE INDEX IF NOT EXISTS ${tableName}_slug_idx
-    ON ${tableName}(slug)
+    CREATE INDEX IF NOT EXISTS ${quotedSlugIndexName}
+    ON ${quotedTableName}(slug)
   `);
 }
 
@@ -242,6 +276,8 @@ async function search(options) {
     tableName = "articles",
     embeddingOptions = {}
   } = options;
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
+  const resultLimit = normalizeSearchLimit(limit);
   const queryEmbedding = await generateEmbedding(query, embeddingOptions);
   const results = await client.execute({
     sql: `
@@ -254,12 +290,12 @@ async function search(options) {
         tags,
         created_at,
         vector_distance_cos(embedding, vector(?)) as distance
-      FROM ${tableName}
+      FROM ${quotedTableName}
       WHERE embedding IS NOT NULL
       ORDER BY distance
       LIMIT ?
     `,
-    args: [JSON.stringify(queryEmbedding), limit]
+    args: [JSON.stringify(queryEmbedding), resultLimit]
   });
   return results.rows.map((row) => ({
     id: row.id,
@@ -273,9 +309,10 @@ async function search(options) {
   }));
 }
 async function getAllArticles(client, tableName = "articles") {
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const results = await client.execute(`
     SELECT id, slug, title, folder, tags, created_at, updated_at
-    FROM ${tableName}
+    FROM ${quotedTableName}
     ORDER BY title
   `);
   return results.rows.map((row) => ({
@@ -289,10 +326,11 @@ async function getAllArticles(client, tableName = "articles") {
   }));
 }
 async function getArticleBySlug(client, slug, tableName = "articles") {
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const results = await client.execute({
     sql: `
       SELECT id, slug, title, content, folder, tags, created_at, updated_at
-      FROM ${tableName}
+      FROM ${quotedTableName}
       WHERE slug = ?
       LIMIT 1
     `,
@@ -314,10 +352,11 @@ async function getArticleBySlug(client, slug, tableName = "articles") {
   };
 }
 async function getArticlesByFolder(client, folder, tableName = "articles") {
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const results = await client.execute({
     sql: `
       SELECT id, slug, title, folder, tags
-      FROM ${tableName}
+      FROM ${quotedTableName}
       WHERE folder = ?
       ORDER BY title
     `,
@@ -332,9 +371,10 @@ async function getArticlesByFolder(client, folder, tableName = "articles") {
   }));
 }
 async function getFolders(client, tableName = "articles") {
+  const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const results = await client.execute(`
     SELECT DISTINCT folder
-    FROM ${tableName}
+    FROM ${quotedTableName}
     ORDER BY folder
   `);
   return results.rows.map((row) => row.folder);

package/docs/API.md

@@ -0,0 +1,180 @@
+# API Reference
+
+## Exports
+
+`libsql-search` exports:
+
+- `createTable`
+- `indexContent`
+- `search`
+- `getAllArticles`
+- `getArticleBySlug`
+- `getArticlesByFolder`
+- `getFolders`
+- `generateEmbedding`
+- `padEmbedding`
+- `prepareTextForEmbedding`
+
+It also exports these types:
+
+- `EmbeddingProvider`
+- `EmbeddingOptions`
+- `IndexerOptions`
+- `IndexedDocument`
+- `SearchOptions`
+- `SearchResult`
+
+## `createTable(client, tableName?, dimensions?)`
+
+Creates the table and supporting indexes used by search.
+
+```ts
+await createTable(client, "articles", 768);
+```
+
+Defaults:
+
+- `tableName`: `"articles"`
+- `dimensions`: `768`
+
+`tableName` must be an ASCII SQLite identifier matching
+`[A-Za-z_][A-Za-z0-9_]*`. Valid identifiers are quoted internally, so reserved
+words such as `"select"` are safe to use. `dimensions` must be a positive
+integer.
+
+The created schema includes:
+
+- `id` primary key
+- `slug`
+- `title`
+- `content`
+- `folder`
+- `tags`
+- `embedding`
+- `created_at`
+- `updated_at`
+
+## `indexContent(options)`
+
+Indexes Markdown files from a directory on disk.
+
+```ts
+interface IndexerOptions {
+  client: Client;
+  contentPath: string;
+  embeddingOptions?: EmbeddingOptions;
+  fileExtensions?: string[];
+  exclude?: string[];
+  tableName?: string;
+  onProgress?: (current: number, total: number, file: string) => void;
+}
+```
+
+Defaults:
+
+- `fileExtensions`: [".md", ".markdown"]
+- `exclude`: ["node_modules", ".git", "dist", "build"]
+- `tableName`: `"articles"`
+
+`tableName` follows the same identifier policy as `createTable()`.
+
+Return shape:
+
+```ts
+{
+  success: number;
+  failed: number;
+  total: number;
+}
+```
+
+Behavior notes:
+
+- `indexContent()` deletes existing rows in the target table before rebuilding
+- frontmatter `title`, `description`, and `tags` are folded into the embedding
+  text
+- if a file has no frontmatter title, the filename becomes the title
+
+## `search(options)`
+
+Generates a query embedding and performs vector similarity search.
+
+```ts
+interface SearchOptions {
+  client: Client;
+  query: string;
+  limit?: number;
+  tableName?: string;
+  embeddingOptions?: EmbeddingOptions;
+}
+```
+
+Defaults:
+
+- `limit`: `10`
+- `tableName`: `"articles"`
+
+`limit` must be an integer from `1` through `100`; invalid values are rejected
+before query embedding generation. `tableName` follows the same identifier
+policy as `createTable()`.
+
+Result shape:
+
+```ts
+interface SearchResult {
+  id: number;
+  slug: string;
+  title: string;
+  content: string;
+  folder: string;
+  tags: string[];
+  distance: number;
+  created_at: string;
+}
+```
+
+Lower `distance` values are better matches.
+
+## Article Retrieval Helpers
+
+### `getAllArticles(client, tableName?)`
+
+Returns all indexed articles ordered by title.
+
+### `getArticleBySlug(client, slug, tableName?)`
+
+Returns one article or `null`.
+
+### `getArticlesByFolder(client, folder, tableName?)`
+
+Returns articles in a specific folder.
+
+### `getFolders(client, tableName?)`
+
+Returns distinct folder names from the index.
+
+All article retrieval helpers validate `tableName` before executing SQL.
+
+## Embedding Helpers
+
+### `generateEmbedding(text, options?)`
+
+Generates an embedding for arbitrary text using the selected provider.
+
+### `padEmbedding(embedding, targetDimensions)`
+
+Pads or truncates an embedding array to the requested length.
+
+### `prepareTextForEmbedding(fields)`
+
+Combines title, description, tags, and content into the text sent to the
+embedding model.
+
+```ts
+const text = prepareTextForEmbedding({
+  title: "My Article",
+  description: "How semantic search works",
+  tags: ["search", "turso"],
+  content: "# Content",
+});
+```

package/docs/INDEXING.md

@@ -0,0 +1,70 @@
+# Indexing And Operations
+
+## Content Shape
+
+`indexContent()` walks a directory tree, reads Markdown files, parses
+frontmatter with `gray-matter`, and stores:
+
+- `slug`
+- `title`
+- `content`
+- `folder`
+- `tags`
+- `embedding`
+
+The slug is derived from the file path relative to `contentPath`.
+
+## Rebuild Behavior
+
+`indexContent()` clears the target table before rebuilding:
+
+```ts
+await indexContent({
+  client,
+  contentPath: "./content",
+  tableName: "articles",
+  embeddingOptions: {
+    provider: "local",
+    dimensions: 768,
+  },
+});
+```
+
+That keeps the implementation simple, but it also means a failed rebuild can
+leave the index partially repopulated.
+
+## Quality Guidelines
+
+- include descriptive frontmatter titles
+- add meaningful `tags` when they help retrieval
+- use the same embedding provider and dimensions at index and query time
+- keep `maxLength` intentional if your content is very large
+- start with a smaller search `limit` and tune from real query behavior
+
+## Build Integration
+
+Many projects wire indexing into a dedicated script and call it before their
+site build:
+
+```json
+{
+  "scripts": {
+    "index": "node ./scripts/index.js",
+    "build": "pnpm index && astro build"
+  }
+}
+```
+
+## Table Names
+
+`tableName` must be an ASCII SQLite identifier matching
+`[A-Za-z_][A-Za-z0-9_]*`. Valid names are quoted internally for table and index
+SQL, so reserved words such as `"select"` work safely. Invalid names fail before
+database calls or embedding generation.
+
+## Runtime Notes
+
+- local embeddings may download a model on the first run
+- Node users need `@libsql/client` installed alongside the package
+- the repository validates both the npm package build and `deno check`, but the
+  indexing flow itself still depends on filesystem access