libsql-search 0.9.1 → 0.10.1

package/README.md

@@ -31,6 +31,8 @@ deno add jsr:@logan/libsql-search npm:@libsql/client@^0.17.0
 
 For npm usage, the package requires Node `>=22.12.0`.
 
+`@tursodatabase/database` is supported as an **optional** peer, behind the separate `libsql-search/turso` entry point. It is experimental and exact-search-only, because Turso Database has no ANN vector index. Nothing is installed or resolved for it unless you opt in — see the [Turso Database backend guide](./docs/TURSO.md).
+
 **On npm/pnpm**, the peer range is `@libsql/client ^0.15.0 || ^0.17.0`. Both lines are supported: every behavior this package depends on — `vector_top_k()`'s result shape, the vector index error wording that `search()` matches on, and transactional `batch()` rollback — is identical across them, so an existing `0.15.x` install does not have to move. There is no `0.16.x` line upstream, which is why the range is a disjunction rather than a span. The packaged build is smoke-tested against both arms on every release, at the newest release each arm admits (currently `0.15.15` and `0.17.4`).
 
 **On JSR/Deno the range does not apply to you.** `deno.json` declares no dependency on `@libsql/client` — this package imports only its *types* — so the client you `deno add` separately is constrained by nothing on our side, and a plain `deno add npm:@libsql/client` will silently take whatever is newest, including a future major we have never tested. Deno also cannot express our range: `npm:@libsql/client@^0.15.0 || ^0.17.0` is a parse error, as is any `>=`/`<` span. Pin an arm yourself instead:

package/dist/index.cjs

@@ -833,6 +833,48 @@ function normalizeVectorDimensions(dimensions) {
   return dimensions;
 }
 
+function isDatabaseAdapter(client) {
+  return typeof client === "object" && client !== null && client.libsqlSearchAdapter === true;
+}
+function resolveDatabase(client) {
+  if (isDatabaseAdapter(client)) {
+    assertCompleteAdapter(client);
+    return client;
+  }
+  return createLibsqlAdapter(client);
+}
+const ADAPTER_METHODS = ["executeDdl", "executeQuery", "executeAtomicWrite"];
+function assertCompleteAdapter(adapter) {
+  const missing = ADAPTER_METHODS.filter(
+    (method) => typeof adapter[method] !== "function"
+  );
+  if (missing.length > 0) {
+    throw new TypeError(
+      `This client is marked as a libsql-search database adapter but is missing ${missing.join(", ")}. The usual cause is two different versions of libsql-search resolved in one dependency tree, so the adapter was built by a different copy of the package than the one calling it. Deduplicate libsql-search, or build the adapter from the same copy you call.`
+    );
+  }
+}
+function createLibsqlAdapter(client) {
+  return {
+    libsqlSearchAdapter: true,
+    backend: "libsql",
+    supportsVectorIndex: true,
+    async executeDdl(sql) {
+      await client.execute(sql);
+    },
+    async executeQuery(sql, args) {
+      const result = args === void 0 ? await client.execute(sql) : await client.execute({ sql, args });
+      return result.rows;
+    },
+    async executeAtomicWrite(statements) {
+      const batch = statements.map(
+        (statement) => statement.args === void 0 ? statement.sql : statement
+      );
+      await client.batch(batch, "write");
+    }
+  };
+}
+
 class IndexingError extends Error {
   phase;
   failures;
@@ -855,6 +897,7 @@ async function indexContent(options) {
     failurePolicy = "abort",
     allowEmptyIndex = false
   } = options;
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   let files;
   try {
@@ -928,7 +971,7 @@ async function indexContent(options) {
     );
   }
   const success = documents.length;
-  await replaceIndex(client, quotedTableName, documents, failures);
+  await replaceIndex(database, quotedTableName, documents, failures);
   return {
     success,
     failed: failures.length,
@@ -974,11 +1017,13 @@ async function buildDocument(file, embeddingOptions) {
     return { ok: false, stage: "parse", error: toError(error) };
   }
   let embedding;
+  let serializedEmbedding;
   try {
     embedding = await generateEmbedding(parsed.embeddingText, {
       ...embeddingOptions,
       intent: embeddingOptions.intent ?? "document"
     });
+    serializedEmbedding = JSON.stringify(embedding);
   } catch (error) {
     return { ok: false, stage: "embed", error: toError(error) };
   }
@@ -992,6 +1037,7 @@ async function buildDocument(file, embeddingOptions) {
       tags: parsed.tags,
       serializedTags: parsed.serializedTags,
       embedding,
+      serializedEmbedding,
       metadata: parsed.metadata
     }
   };
@@ -1041,8 +1087,8 @@ function fallbackTitle(file) {
 function describeType(value) {
   return Array.isArray(value) ? "array" : typeof value;
 }
-async function replaceIndex(client, quotedTableName, documents, failures) {
-  const statements = [`DELETE FROM ${quotedTableName}`];
+async function replaceIndex(database, quotedTableName, documents, failures) {
+  const statements = [{ sql: `DELETE FROM ${quotedTableName}` }];
   try {
     for (let i = 0; i < documents.length; i++) {
       const document = documents[i];
@@ -1055,7 +1101,7 @@ async function replaceIndex(client, quotedTableName, documents, failures) {
     documents.length = 0;
   }
   try {
-    await client.batch(statements, "write");
+    await database.executeAtomicWrite(statements);
   } catch (error) {
     throw new IndexingError(
       `Failed to replace the contents of ${quotedTableName}. The transaction was rolled back and the existing index was left unchanged.`,
@@ -1076,7 +1122,7 @@ function createInsertStatement(document, quotedTableName) {
       document.content,
       document.folder,
       document.serializedTags,
-      JSON.stringify(document.embedding)
+      document.serializedEmbedding
     ]
   };
 }
@@ -1084,12 +1130,13 @@ function toError(error) {
   return error instanceof Error ? error : new Error(String(error));
 }
 async function createTable(client, tableName = "articles", dimensions = 384) {
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const vectorDimensions = normalizeVectorDimensions(dimensions);
   const quotedEmbeddingIndexName = quoteSqlIdentifier(`${tableName}_embedding_idx`, "embedding index name");
   const quotedFolderIndexName = quoteSqlIdentifier(`${tableName}_folder_idx`, "folder index name");
   const quotedSlugIndexName = quoteSqlIdentifier(`${tableName}_slug_idx`, "slug index name");
-  await client.execute(`
+  await database.executeDdl(`
     CREATE TABLE IF NOT EXISTS ${quotedTableName} (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       slug TEXT UNIQUE NOT NULL,
@@ -1102,15 +1149,17 @@ async function createTable(client, tableName = "articles", dimensions = 384) {
       updated_at TEXT NOT NULL
     )
   `);
-  await client.execute(`
-    CREATE INDEX IF NOT EXISTS ${quotedEmbeddingIndexName}
-    ON ${quotedTableName}(libsql_vector_idx(embedding))
-  `);
-  await client.execute(`
+  if (database.supportsVectorIndex) {
+    await database.executeDdl(`
+      CREATE INDEX IF NOT EXISTS ${quotedEmbeddingIndexName}
+      ON ${quotedTableName}(libsql_vector_idx(embedding))
+    `);
+  }
+  await database.executeDdl(`
     CREATE INDEX IF NOT EXISTS ${quotedFolderIndexName}
     ON ${quotedTableName}(folder)
   `);
-  await client.execute(`
+  await database.executeDdl(`
     CREATE INDEX IF NOT EXISTS ${quotedSlugIndexName}
     ON ${quotedTableName}(slug)
   `);
@@ -1133,6 +1182,7 @@ async function search(options) {
     candidates,
     exact = false
   } = options;
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const resultLimit = normalizeSearchLimit(limit);
   const embeddingIndexName = validateSqlIdentifier(
@@ -1145,20 +1195,21 @@ async function search(options) {
     intent: embeddingOptions.intent ?? "query"
   });
   const queryVector = JSON.stringify(queryEmbedding);
-  const results = exact ? await executeExactSearch(client, quotedTableName, queryVector, resultLimit) : await executeIndexSearch(
-    client,
+  const useExactSearch = exact || !database.supportsVectorIndex;
+  const rows = useExactSearch ? await executeExactSearch(database, quotedTableName, queryVector, resultLimit) : await executeIndexSearch(
+    database,
     quotedTableName,
     embeddingIndexName,
     queryVector,
     candidateCount,
     resultLimit
   );
-  return results.rows.map(toSearchResult);
+  return rows.map(toSearchResult);
 }
-async function executeIndexSearch(client, quotedTableName, embeddingIndexName, queryVector, candidateCount, resultLimit) {
+async function executeIndexSearch(database, quotedTableName, embeddingIndexName, queryVector, candidateCount, resultLimit) {
   try {
-    return await client.execute({
-      sql: `
+    return await database.executeQuery(
+      `
         SELECT
           ${RESULT_COLUMNS},
           vector_distance_cos(a.embedding, vector(:queryVector)) as distance
@@ -1167,20 +1218,20 @@ async function executeIndexSearch(client, quotedTableName, embeddingIndexName, q
         ORDER BY distance, a.id
         LIMIT :resultLimit
       `,
-      args: {
+      {
         queryVector,
         indexName: embeddingIndexName,
         candidates: candidateCount,
         resultLimit
       }
-    });
+    );
   } catch (error) {
     throw wrapIndexPathError(error, embeddingIndexName);
   }
 }
-async function executeExactSearch(client, quotedTableName, queryVector, resultLimit) {
-  return client.execute({
-    sql: `
+async function executeExactSearch(database, quotedTableName, queryVector, resultLimit) {
+  return database.executeQuery(
+    `
       SELECT
         ${RESULT_COLUMNS},
         vector_distance_cos(a.embedding, vector(:queryVector)) as distance
@@ -1189,8 +1240,8 @@ async function executeExactSearch(client, quotedTableName, queryVector, resultLi
       ORDER BY distance, a.id
       LIMIT :resultLimit
     `,
-    args: { queryVector, resultLimit }
-  });
+    { queryVector, resultLimit }
+  );
 }
 function wrapIndexPathError(error, embeddingIndexName) {
   if (!(error instanceof Error)) {
@@ -1223,13 +1274,14 @@ function toSearchResult(row) {
   };
 }
 async function getAllArticles(client, tableName = "articles") {
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
-  const results = await client.execute(`
+  const rows = await database.executeQuery(`
     SELECT id, slug, title, folder, tags, created_at, updated_at
     FROM ${quotedTableName}
     ORDER BY title
   `);
-  return results.rows.map((row) => ({
+  return rows.map((row) => ({
     id: row.id,
     slug: row.slug,
     title: row.title,
@@ -1240,20 +1292,21 @@ async function getAllArticles(client, tableName = "articles") {
   }));
 }
 async function getArticleBySlug(client, slug, tableName = "articles") {
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
-  const results = await client.execute({
-    sql: `
+  const rows = await database.executeQuery(
+    `
       SELECT id, slug, title, content, folder, tags, created_at, updated_at
       FROM ${quotedTableName}
       WHERE slug = ?
       LIMIT 1
     `,
-    args: [slug]
-  });
-  if (results.rows.length === 0) {
+    [slug]
+  );
+  if (rows.length === 0) {
     return null;
   }
-  const row = results.rows[0];
+  const row = rows[0];
   return {
     id: row.id,
     slug: row.slug,
@@ -1266,17 +1319,18 @@ async function getArticleBySlug(client, slug, tableName = "articles") {
   };
 }
 async function getArticlesByFolder(client, folder, tableName = "articles") {
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
-  const results = await client.execute({
-    sql: `
+  const rows = await database.executeQuery(
+    `
       SELECT id, slug, title, folder, tags
       FROM ${quotedTableName}
       WHERE folder = ?
       ORDER BY title
     `,
-    args: [folder]
-  });
-  return results.rows.map((row) => ({
+    [folder]
+  );
+  return rows.map((row) => ({
     id: row.id,
     slug: row.slug,
     title: row.title,
@@ -1285,13 +1339,14 @@ async function getArticlesByFolder(client, folder, tableName = "articles") {
   }));
 }
 async function getFolders(client, tableName = "articles") {
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
-  const results = await client.execute(`
+  const rows = await database.executeQuery(`
     SELECT DISTINCT folder
     FROM ${quotedTableName}
     ORDER BY folder
   `);
-  return results.rows.map((row) => row.folder);
+  return rows.map((row) => row.folder);
 }
 
 exports.DEFAULT_SEARCH_CANDIDATE_MULTIPLIER = DEFAULT_SEARCH_CANDIDATE_MULTIPLIER;

package/dist/index.d.ts

@@ -78,6 +78,84 @@ declare function prepareTextForEmbedding(fields: {
     [key: string]: unknown;
 }): string;
 
+/**
+ * Database backend boundary.
+ *
+ * Every SQL call this library makes goes through {@link DatabaseAdapter}. The
+ * `@libsql/client` adapter is built here, because that client is the declared
+ * peer dependency and the only backend the main entry point knows about. Other
+ * backends supply their own adapter from their own entry point, so the main
+ * bundle never imports, resolves, or references their package.
+ */
+
+/**
+ * The narrow database contract the indexer, search, and retrieval helpers use.
+ *
+ * This interface is deliberately self-contained: it names no other type from
+ * this package and no type from any backend package. That keeps the declaration
+ * emitted into every entry point's `.d.ts` structurally identical, so an adapter
+ * built by one entry point is assignable to the parameter types of another
+ * without the two entry points sharing a nominal symbol.
+ *
+ * Do not brand this with a `unique symbol`. Each bundled `.d.ts` declares its
+ * own copy of the interface, and a `unique symbol` would make those copies
+ * nominally distinct — an adapter from the `libsql-search/turso` entry point
+ * would stop being assignable to `SearchOptions['client']`.
+ */
+interface DatabaseAdapter {
+    /**
+     * Marker set by this package's adapter factories.
+     *
+     * Public functions accept either a raw `@libsql/client` `Client` or an
+     * adapter, and this property is how they tell the two apart. It is a plain
+     * property rather than structural method sniffing on purpose: guessing a
+     * backend from its method names would risk picking the wrong atomicity
+     * primitive, and the two backends disagree about which primitive is
+     * transactional.
+     */
+    readonly libsqlSearchAdapter: true;
+    /** Which backend this adapter drives. Informational; behavior keys off the capability flags. */
+    readonly backend: string;
+    /**
+     * Whether the backend can build a `libsql_vector_idx()` index and query it
+     * through `vector_top_k()`.
+     *
+     * When false, `createTable()` skips the vector index and `search()` runs the
+     * exact full-scan path automatically. Both are required, not optimizations:
+     * on a backend without the index, `CREATE INDEX ... libsql_vector_idx(...)`
+     * is a parse error and `vector_top_k` is not a table.
+     */
+    readonly supportsVectorIndex: boolean;
+    /** Run a schema statement that returns no rows. */
+    executeDdl(sql: string): Promise<void>;
+    /**
+     * Run one statement and return its rows as plain column-keyed objects.
+     *
+     * The bind-value union is spelled out inline rather than named, so that this
+     * interface stays free of any other symbol. It is narrower than `unknown` on
+     * purpose: it is the only compile-time check that a caller is binding
+     * something a SQLite driver can actually store.
+     */
+    executeQuery(sql: string, args?: Readonly<Record<string, string | number | bigint | Uint8Array | null>> | ReadonlyArray<string | number | bigint | Uint8Array | null>): Promise<Array<Record<string, unknown>>>;
+    /**
+     * Apply every statement atomically: either all of them commit, or none do.
+     *
+     * The whole "a failed rebuild leaves the previous index intact" guarantee
+     * rests on this method. A backend whose batch primitive is not transactional
+     * must implement this with an explicit transaction instead.
+     */
+    executeAtomicWrite(statements: ReadonlyArray<{
+        sql: string;
+        args?: ReadonlyArray<string | number | bigint | Uint8Array | null>;
+    }>): Promise<void>;
+}
+/**
+ * A client accepted by this library's public functions: either the
+ * `@libsql/client` client, or an adapter built by one of this package's
+ * adapter factories.
+ */
+type DatabaseClient = Client | DatabaseAdapter;
+
 /**
  * Content indexer for markdown and other formats
  */
@@ -105,7 +183,11 @@ interface IndexFailure {
     error: Error;
 }
 interface IndexerOptions {
-    client: Client;
+    /**
+     * A `@libsql/client` client, or an adapter from one of this package's
+     * backend entry points, such as `tursoAdapter()` from `libsql-search/turso`.
+     */
+    client: DatabaseClient;
     contentPath: string;
     embeddingOptions?: EmbeddingOptions;
     fileExtensions?: string[];
@@ -157,9 +239,14 @@ declare class IndexingError extends Error {
  */
 declare function indexContent(options: IndexerOptions): Promise<IndexResult>;
 /**
- * Create the articles table if it doesn't exist
+ * Create the articles table if it doesn't exist.
+ *
+ * Creates the table, the vector index, the folder index, and the slug index.
+ * The vector index is created only on backends that support
+ * `libsql_vector_idx()`; on Turso Database, which does not, everything else is
+ * still created and `search()` falls back to the exact full scan.
  */
-declare function createTable(client: Client, tableName?: string, dimensions?: number): Promise<void>;
+declare function createTable(client: DatabaseClient, tableName?: string, dimensions?: number): Promise<void>;
 
 /**
  * Multiplier applied to `limit` when deriving the default candidate count.
@@ -187,7 +274,11 @@ declare const MAX_SEARCH_CANDIDATES: number;
  */
 
 interface SearchOptions {
-    client: Client;
+    /**
+     * A `@libsql/client` client, or an adapter from one of this package's
+     * backend entry points, such as `tursoAdapter()` from `libsql-search/turso`.
+     */
+    client: DatabaseClient;
     query: string;
     limit?: number;
     tableName?: string;
@@ -206,6 +297,10 @@ interface SearchOptions {
     /**
      * Bypass the vector index and score every row instead. Exact but linear in
      * table size. Defaults to `false`.
+     *
+     * Backends with no vector index take this path whether or not it is set: on
+     * Turso Database `vector_top_k()` does not exist, so there is nothing to fall
+     * back from.
      */
     exact?: boolean;
 }
@@ -231,12 +326,16 @@ interface SearchResult {
  * deterministic even though the candidate set is not.
  *
  * Pass `exact: true` for a guaranteed-exact full scan.
+ *
+ * On a backend with no ANN vector index — Turso Database, reached through
+ * `tursoAdapter()` — the exact path is the only path, and is selected
+ * automatically without `exact: true`.
  */
 declare function search(options: SearchOptions): Promise<SearchResult[]>;
 /**
  * Get all articles (for building static pages, navigation, etc.)
  */
-declare function getAllArticles(client: Client, tableName?: string): Promise<Array<{
+declare function getAllArticles(client: DatabaseClient, tableName?: string): Promise<Array<{
     id: number;
     slug: string;
     title: string;
@@ -248,7 +347,7 @@ declare function getAllArticles(client: Client, tableName?: string): Promise<Arr
 /**
  * Get a single article by slug
  */
-declare function getArticleBySlug(client: Client, slug: string, tableName?: string): Promise<{
+declare function getArticleBySlug(client: DatabaseClient, slug: string, tableName?: string): Promise<{
     id: number;
     slug: string;
     title: string;
@@ -261,7 +360,7 @@ declare function getArticleBySlug(client: Client, slug: string, tableName?: stri
 /**
  * Get articles by folder
  */
-declare function getArticlesByFolder(client: Client, folder: string, tableName?: string): Promise<Array<{
+declare function getArticlesByFolder(client: DatabaseClient, folder: string, tableName?: string): Promise<Array<{
     id: number;
     slug: string;
     title: string;
@@ -271,7 +370,7 @@ declare function getArticlesByFolder(client: Client, folder: string, tableName?:
 /**
  * Get all unique folders
  */
-declare function getFolders(client: Client, tableName?: string): Promise<string[]>;
+declare function getFolders(client: DatabaseClient, tableName?: string): Promise<string[]>;
 
 export { DEFAULT_SEARCH_CANDIDATE_MULTIPLIER, IndexingError, MAX_SEARCH_CANDIDATES, MIN_SEARCH_CANDIDATES, createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };
 export type { EmbeddingBatchBehavior, EmbeddingBatchItem, EmbeddingBatchItemResult, EmbeddingBatchMode, EmbeddingBatchResult, EmbeddingIntent, EmbeddingOptions, EmbeddingProvider, EmbeddingProviderClient, EmbeddingProviderMetadata, EmbeddingRequestOptions, IndexFailure, IndexFailurePolicy, IndexFailureStage, IndexResult, IndexedDocument, IndexerOptions, IndexingErrorPhase, SearchOptions, SearchResult };