libsql-search 0.9.0 → 0.10.0

package/dist/index.esm.js

@@ -831,6 +831,48 @@ function normalizeVectorDimensions(dimensions) {
   return dimensions;
 }
 
+function isDatabaseAdapter(client) {
+  return typeof client === "object" && client !== null && client.libsqlSearchAdapter === true;
+}
+function resolveDatabase(client) {
+  if (isDatabaseAdapter(client)) {
+    assertCompleteAdapter(client);
+    return client;
+  }
+  return createLibsqlAdapter(client);
+}
+const ADAPTER_METHODS = ["executeDdl", "executeQuery", "executeAtomicWrite"];
+function assertCompleteAdapter(adapter) {
+  const missing = ADAPTER_METHODS.filter(
+    (method) => typeof adapter[method] !== "function"
+  );
+  if (missing.length > 0) {
+    throw new TypeError(
+      `This client is marked as a libsql-search database adapter but is missing ${missing.join(", ")}. The usual cause is two different versions of libsql-search resolved in one dependency tree, so the adapter was built by a different copy of the package than the one calling it. Deduplicate libsql-search, or build the adapter from the same copy you call.`
+    );
+  }
+}
+function createLibsqlAdapter(client) {
+  return {
+    libsqlSearchAdapter: true,
+    backend: "libsql",
+    supportsVectorIndex: true,
+    async executeDdl(sql) {
+      await client.execute(sql);
+    },
+    async executeQuery(sql, args) {
+      const result = args === void 0 ? await client.execute(sql) : await client.execute({ sql, args });
+      return result.rows;
+    },
+    async executeAtomicWrite(statements) {
+      const batch = statements.map(
+        (statement) => statement.args === void 0 ? statement.sql : statement
+      );
+      await client.batch(batch, "write");
+    }
+  };
+}
+
 class IndexingError extends Error {
   phase;
   failures;
@@ -853,6 +895,7 @@ async function indexContent(options) {
     failurePolicy = "abort",
     allowEmptyIndex = false
   } = options;
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   let files;
   try {
@@ -926,7 +969,7 @@ async function indexContent(options) {
     );
   }
   const success = documents.length;
-  await replaceIndex(client, quotedTableName, documents, failures);
+  await replaceIndex(database, quotedTableName, documents, failures);
   return {
     success,
     failed: failures.length,
@@ -1039,8 +1082,8 @@ function fallbackTitle(file) {
 function describeType(value) {
   return Array.isArray(value) ? "array" : typeof value;
 }
-async function replaceIndex(client, quotedTableName, documents, failures) {
-  const statements = [`DELETE FROM ${quotedTableName}`];
+async function replaceIndex(database, quotedTableName, documents, failures) {
+  const statements = [{ sql: `DELETE FROM ${quotedTableName}` }];
   try {
     for (let i = 0; i < documents.length; i++) {
       const document = documents[i];
@@ -1053,7 +1096,7 @@ async function replaceIndex(client, quotedTableName, documents, failures) {
     documents.length = 0;
   }
   try {
-    await client.batch(statements, "write");
+    await database.executeAtomicWrite(statements);
   } catch (error) {
     throw new IndexingError(
       `Failed to replace the contents of ${quotedTableName}. The transaction was rolled back and the existing index was left unchanged.`,
@@ -1082,12 +1125,13 @@ function toError(error) {
   return error instanceof Error ? error : new Error(String(error));
 }
 async function createTable(client, tableName = "articles", dimensions = 384) {
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const vectorDimensions = normalizeVectorDimensions(dimensions);
   const quotedEmbeddingIndexName = quoteSqlIdentifier(`${tableName}_embedding_idx`, "embedding index name");
   const quotedFolderIndexName = quoteSqlIdentifier(`${tableName}_folder_idx`, "folder index name");
   const quotedSlugIndexName = quoteSqlIdentifier(`${tableName}_slug_idx`, "slug index name");
-  await client.execute(`
+  await database.executeDdl(`
     CREATE TABLE IF NOT EXISTS ${quotedTableName} (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       slug TEXT UNIQUE NOT NULL,
@@ -1100,15 +1144,17 @@ async function createTable(client, tableName = "articles", dimensions = 384) {
       updated_at TEXT NOT NULL
     )
   `);
-  await client.execute(`
-    CREATE INDEX IF NOT EXISTS ${quotedEmbeddingIndexName}
-    ON ${quotedTableName}(libsql_vector_idx(embedding))
-  `);
-  await client.execute(`
+  if (database.supportsVectorIndex) {
+    await database.executeDdl(`
+      CREATE INDEX IF NOT EXISTS ${quotedEmbeddingIndexName}
+      ON ${quotedTableName}(libsql_vector_idx(embedding))
+    `);
+  }
+  await database.executeDdl(`
     CREATE INDEX IF NOT EXISTS ${quotedFolderIndexName}
     ON ${quotedTableName}(folder)
   `);
-  await client.execute(`
+  await database.executeDdl(`
     CREATE INDEX IF NOT EXISTS ${quotedSlugIndexName}
     ON ${quotedTableName}(slug)
   `);
@@ -1131,6 +1177,7 @@ async function search(options) {
     candidates,
     exact = false
   } = options;
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
   const resultLimit = normalizeSearchLimit(limit);
   const embeddingIndexName = validateSqlIdentifier(
@@ -1143,20 +1190,21 @@ async function search(options) {
     intent: embeddingOptions.intent ?? "query"
   });
   const queryVector = JSON.stringify(queryEmbedding);
-  const results = exact ? await executeExactSearch(client, quotedTableName, queryVector, resultLimit) : await executeIndexSearch(
-    client,
+  const useExactSearch = exact || !database.supportsVectorIndex;
+  const rows = useExactSearch ? await executeExactSearch(database, quotedTableName, queryVector, resultLimit) : await executeIndexSearch(
+    database,
     quotedTableName,
     embeddingIndexName,
     queryVector,
     candidateCount,
     resultLimit
   );
-  return results.rows.map(toSearchResult);
+  return rows.map(toSearchResult);
 }
-async function executeIndexSearch(client, quotedTableName, embeddingIndexName, queryVector, candidateCount, resultLimit) {
+async function executeIndexSearch(database, quotedTableName, embeddingIndexName, queryVector, candidateCount, resultLimit) {
   try {
-    return await client.execute({
-      sql: `
+    return await database.executeQuery(
+      `
         SELECT
           ${RESULT_COLUMNS},
           vector_distance_cos(a.embedding, vector(:queryVector)) as distance
@@ -1165,20 +1213,20 @@ async function executeIndexSearch(client, quotedTableName, embeddingIndexName, q
         ORDER BY distance, a.id
         LIMIT :resultLimit
       `,
-      args: {
+      {
         queryVector,
         indexName: embeddingIndexName,
         candidates: candidateCount,
         resultLimit
       }
-    });
+    );
   } catch (error) {
     throw wrapIndexPathError(error, embeddingIndexName);
   }
 }
-async function executeExactSearch(client, quotedTableName, queryVector, resultLimit) {
-  return client.execute({
-    sql: `
+async function executeExactSearch(database, quotedTableName, queryVector, resultLimit) {
+  return database.executeQuery(
+    `
       SELECT
         ${RESULT_COLUMNS},
         vector_distance_cos(a.embedding, vector(:queryVector)) as distance
@@ -1187,8 +1235,8 @@ async function executeExactSearch(client, quotedTableName, queryVector, resultLi
       ORDER BY distance, a.id
       LIMIT :resultLimit
     `,
-    args: { queryVector, resultLimit }
-  });
+    { queryVector, resultLimit }
+  );
 }
 function wrapIndexPathError(error, embeddingIndexName) {
   if (!(error instanceof Error)) {
@@ -1221,13 +1269,14 @@ function toSearchResult(row) {
   };
 }
 async function getAllArticles(client, tableName = "articles") {
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
-  const results = await client.execute(`
+  const rows = await database.executeQuery(`
     SELECT id, slug, title, folder, tags, created_at, updated_at
     FROM ${quotedTableName}
     ORDER BY title
   `);
-  return results.rows.map((row) => ({
+  return rows.map((row) => ({
     id: row.id,
     slug: row.slug,
     title: row.title,
@@ -1238,20 +1287,21 @@ async function getAllArticles(client, tableName = "articles") {
   }));
 }
 async function getArticleBySlug(client, slug, tableName = "articles") {
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
-  const results = await client.execute({
-    sql: `
+  const rows = await database.executeQuery(
+    `
       SELECT id, slug, title, content, folder, tags, created_at, updated_at
       FROM ${quotedTableName}
       WHERE slug = ?
       LIMIT 1
     `,
-    args: [slug]
-  });
-  if (results.rows.length === 0) {
+    [slug]
+  );
+  if (rows.length === 0) {
     return null;
   }
-  const row = results.rows[0];
+  const row = rows[0];
   return {
     id: row.id,
     slug: row.slug,
@@ -1264,17 +1314,18 @@ async function getArticleBySlug(client, slug, tableName = "articles") {
   };
 }
 async function getArticlesByFolder(client, folder, tableName = "articles") {
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
-  const results = await client.execute({
-    sql: `
+  const rows = await database.executeQuery(
+    `
       SELECT id, slug, title, folder, tags
       FROM ${quotedTableName}
       WHERE folder = ?
       ORDER BY title
     `,
-    args: [folder]
-  });
-  return results.rows.map((row) => ({
+    [folder]
+  );
+  return rows.map((row) => ({
     id: row.id,
     slug: row.slug,
     title: row.title,
@@ -1283,13 +1334,14 @@ async function getArticlesByFolder(client, folder, tableName = "articles") {
   }));
 }
 async function getFolders(client, tableName = "articles") {
+  const database = resolveDatabase(client);
   const quotedTableName = quoteSqlIdentifier(tableName, "tableName");
-  const results = await client.execute(`
+  const rows = await database.executeQuery(`
     SELECT DISTINCT folder
     FROM ${quotedTableName}
     ORDER BY folder
   `);
-  return results.rows.map((row) => row.folder);
+  return rows.map((row) => row.folder);
 }
 
 export { DEFAULT_SEARCH_CANDIDATE_MULTIPLIER, IndexingError, MAX_SEARCH_CANDIDATES, MIN_SEARCH_CANDIDATES, createEmbeddingProvider, createTable, generateEmbedding, generateEmbeddings, getAllArticles, getArticleBySlug, getArticlesByFolder, getEmbeddingProviderMetadata, getFolders, indexContent, padEmbedding, prepareTextForEmbedding, search, validateEmbeddingBatch };

package/dist/turso.cjs

@@ -0,0 +1,96 @@
+'use strict';
+
+function tursoAdapter(database) {
+  assertTursoDatabase(database);
+  return {
+    libsqlSearchAdapter: true,
+    backend: "turso",
+    /**
+     * Turso Database ships the vector functions but no ANN index. This is a
+     * design direction rather than a gap: the project shipped SIMD-accelerated
+     * exact search instead, and the DiskANN port remains an open backlog item.
+     */
+    supportsVectorIndex: false,
+    async executeDdl(sql) {
+      await database.exec(sql);
+    },
+    async executeQuery(sql, args) {
+      const statement = database.prepare(sql);
+      try {
+        const rows = await (args === void 0 ? statement.all() : statement.all(args));
+        return rows;
+      } finally {
+        closeStatement(statement);
+      }
+    },
+    /**
+     * Replace the table contents inside one explicit transaction.
+     *
+     * `batch()` is deliberately not used. It is not transactional on this
+     * backend: a batch that fails part way through leaves the statements before
+     * the failure applied, which would silently turn a failed rebuild into a
+     * half-destroyed index. `BEGIN IMMEDIATE` takes the write lock up front and
+     * `ROLLBACK` restores the previous rows exactly.
+     *
+     * Statements are prepared once per distinct SQL string and rebound per row.
+     * An index rebuild is one `DELETE` followed by N identical `INSERT`s, so
+     * this turns N+1 prepares into exactly 2 regardless of corpus size.
+     */
+    async executeAtomicWrite(statements) {
+      const preparedBySql = /* @__PURE__ */ new Map();
+      const prepareOnce = (sql) => {
+        let prepared = preparedBySql.get(sql);
+        if (prepared === void 0) {
+          prepared = database.prepare(sql);
+          preparedBySql.set(sql, prepared);
+        }
+        return prepared;
+      };
+      await database.exec("BEGIN IMMEDIATE");
+      try {
+        for (const statement of statements) {
+          const prepared = prepareOnce(statement.sql);
+          await (statement.args === void 0 ? prepared.run() : prepared.run(statement.args));
+        }
+        await database.exec("COMMIT");
+      } catch (error) {
+        try {
+          await database.exec("ROLLBACK");
+        } catch (rollbackError) {
+          warnOnFailedRollback(rollbackError);
+        }
+        throw error;
+      } finally {
+        for (const prepared of preparedBySql.values()) {
+          closeStatement(prepared);
+        }
+        preparedBySql.clear();
+      }
+    }
+  };
+}
+function closeStatement(statement) {
+  try {
+    statement.close?.();
+  } catch {
+  }
+}
+const NO_ACTIVE_TRANSACTION_PATTERN = /no transaction is active/i;
+function warnOnFailedRollback(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  if (NO_ACTIVE_TRANSACTION_PATTERN.test(message)) {
+    return;
+  }
+  console.warn(
+    `[libsql-search] ROLLBACK failed after a failed index replacement: ${message}. This database handle may still be inside an open transaction, in which case a later write on it fails with "cannot start a transaction within a transaction". Reconnect the handle before reusing it.`
+  );
+}
+function assertTursoDatabase(database) {
+  if (typeof database !== "object" || database === null || typeof database.exec !== "function" || typeof database.prepare !== "function") {
+    throw new TypeError(
+      "tursoAdapter() expects a connected @tursodatabase/database handle with exec() and prepare() methods. connect() returns a Promise, so remember to await it."
+    );
+  }
+}
+
+exports.tursoAdapter = tursoAdapter;

package/dist/turso.d.ts

@@ -0,0 +1,156 @@
+/**
+ * Database backend boundary.
+ *
+ * Every SQL call this library makes goes through {@link DatabaseAdapter}. The
+ * `@libsql/client` adapter is built here, because that client is the declared
+ * peer dependency and the only backend the main entry point knows about. Other
+ * backends supply their own adapter from their own entry point, so the main
+ * bundle never imports, resolves, or references their package.
+ */
+
+/**
+ * The narrow database contract the indexer, search, and retrieval helpers use.
+ *
+ * This interface is deliberately self-contained: it names no other type from
+ * this package and no type from any backend package. That keeps the declaration
+ * emitted into every entry point's `.d.ts` structurally identical, so an adapter
+ * built by one entry point is assignable to the parameter types of another
+ * without the two entry points sharing a nominal symbol.
+ *
+ * Do not brand this with a `unique symbol`. Each bundled `.d.ts` declares its
+ * own copy of the interface, and a `unique symbol` would make those copies
+ * nominally distinct — an adapter from the `libsql-search/turso` entry point
+ * would stop being assignable to `SearchOptions['client']`.
+ */
+interface DatabaseAdapter {
+    /**
+     * Marker set by this package's adapter factories.
+     *
+     * Public functions accept either a raw `@libsql/client` `Client` or an
+     * adapter, and this property is how they tell the two apart. It is a plain
+     * property rather than structural method sniffing on purpose: guessing a
+     * backend from its method names would risk picking the wrong atomicity
+     * primitive, and the two backends disagree about which primitive is
+     * transactional.
+     */
+    readonly libsqlSearchAdapter: true;
+    /** Which backend this adapter drives. Informational; behavior keys off the capability flags. */
+    readonly backend: string;
+    /**
+     * Whether the backend can build a `libsql_vector_idx()` index and query it
+     * through `vector_top_k()`.
+     *
+     * When false, `createTable()` skips the vector index and `search()` runs the
+     * exact full-scan path automatically. Both are required, not optimizations:
+     * on a backend without the index, `CREATE INDEX ... libsql_vector_idx(...)`
+     * is a parse error and `vector_top_k` is not a table.
+     */
+    readonly supportsVectorIndex: boolean;
+    /** Run a schema statement that returns no rows. */
+    executeDdl(sql: string): Promise<void>;
+    /**
+     * Run one statement and return its rows as plain column-keyed objects.
+     *
+     * The bind-value union is spelled out inline rather than named, so that this
+     * interface stays free of any other symbol. It is narrower than `unknown` on
+     * purpose: it is the only compile-time check that a caller is binding
+     * something a SQLite driver can actually store.
+     */
+    executeQuery(sql: string, args?: Readonly<Record<string, string | number | bigint | Uint8Array | null>> | ReadonlyArray<string | number | bigint | Uint8Array | null>): Promise<Array<Record<string, unknown>>>;
+    /**
+     * Apply every statement atomically: either all of them commit, or none do.
+     *
+     * The whole "a failed rebuild leaves the previous index intact" guarantee
+     * rests on this method. A backend whose batch primitive is not transactional
+     * must implement this with an explicit transaction instead.
+     */
+    executeAtomicWrite(statements: ReadonlyArray<{
+        sql: string;
+        args?: ReadonlyArray<string | number | bigint | Uint8Array | null>;
+    }>): Promise<void>;
+}
+
+/**
+ * Turso Database adapter — `libsql-search/turso`.
+ *
+ * **Experimental, and exact-search only.** See `docs/TURSO.md`.
+ *
+ * This entry point exists so that the main entry point never mentions
+ * `@tursodatabase/database`. Nothing here imports it either: the database
+ * handle is accepted through the structural {@link TursoDatabase} type, so
+ * neither `tsc`, nor Deno, nor a bundler is ever asked to resolve the
+ * native package on behalf of a user who does not use it.
+ *
+ * ```ts
+ * import { connect } from '@tursodatabase/database';
+ * import { tursoAdapter } from 'libsql-search/turso';
+ * import { createTable, indexContent, search } from 'libsql-search';
+ *
+ * const client = tursoAdapter(await connect('./local.db'));
+ *
+ * await createTable(client);
+ * await indexContent({ client, contentPath: './content' });
+ * const results = await search({ client, query: 'vector search' });
+ * ```
+ *
+ * @module libsql-search/turso
+ */
+
+/**
+ * The prepared-statement surface this adapter uses, as
+ * `@tursodatabase/database` exposes it.
+ *
+ * Every method returns a Promise on this backend, but they are typed as
+ * possibly-synchronous and always awaited, so a synchronous driver with the
+ * same shape works too.
+ */
+interface TursoStatement {
+    run(args?: unknown): unknown;
+    all(args?: unknown): unknown;
+    /**
+     * Release the native statement. Optional: a driver that manages statement
+     * lifetime itself may omit it.
+     *
+     * Not closing leaks roughly 10 KB of native memory per prepare on
+     * `@tursodatabase/database`, which the garbage collector does not reclaim
+     * because it is not JavaScript heap. A server calling `search()` per request
+     * grows without bound until the process is killed.
+     */
+    close?(): unknown;
+}
+/**
+ * The database-handle surface this adapter uses, matching the object returned
+ * by `connect()` from `@tursodatabase/database`.
+ *
+ * Declared structurally on purpose. Importing the real type would make
+ * `@tursodatabase/database` a hard resolution target for anyone who type-checks
+ * this package, including the many users who only ever touch `@libsql/client`.
+ */
+interface TursoDatabase {
+    exec(sql: string): unknown;
+    prepare(sql: string): TursoStatement;
+}
+/**
+ * Wrap a `@tursodatabase/database` handle so this library's functions can use
+ * it.
+ *
+ * The returned value is passed as `client` to `createTable()`,
+ * `indexContent()`, `search()`, and the retrieval helpers.
+ *
+ * Two behaviors differ from `@libsql/client`, and both are forced by the
+ * backend rather than chosen here:
+ *
+ * - **No ANN vector index.** Turso Database implements the vector *functions*
+ *   but not `libsql_vector_idx()` or `vector_top_k()`, so `createTable()` skips
+ *   the vector index and `search()` always runs the exact full scan. Search is
+ *   therefore O(rows) on this backend.
+ * - **Transactions, not batches.** Turso's `batch()` is not atomic — a failing
+ *   batch can leave earlier statements committed — so the index replacement
+ *   runs inside an explicit `BEGIN IMMEDIATE` / `COMMIT` transaction, which is.
+ *   This preserves the guarantee that a failed rebuild leaves the previous
+ *   index intact.
+ */
+declare function tursoAdapter(database: TursoDatabase): DatabaseAdapter;
+
+export { tursoAdapter };
+export type { DatabaseAdapter, TursoDatabase, TursoStatement };

package/dist/turso.esm.js

@@ -0,0 +1,94 @@
+function tursoAdapter(database) {
+  assertTursoDatabase(database);
+  return {
+    libsqlSearchAdapter: true,
+    backend: "turso",
+    /**
+     * Turso Database ships the vector functions but no ANN index. This is a
+     * design direction rather than a gap: the project shipped SIMD-accelerated
+     * exact search instead, and the DiskANN port remains an open backlog item.
+     */
+    supportsVectorIndex: false,
+    async executeDdl(sql) {
+      await database.exec(sql);
+    },
+    async executeQuery(sql, args) {
+      const statement = database.prepare(sql);
+      try {
+        const rows = await (args === void 0 ? statement.all() : statement.all(args));
+        return rows;
+      } finally {
+        closeStatement(statement);
+      }
+    },
+    /**
+     * Replace the table contents inside one explicit transaction.
+     *
+     * `batch()` is deliberately not used. It is not transactional on this
+     * backend: a batch that fails part way through leaves the statements before
+     * the failure applied, which would silently turn a failed rebuild into a
+     * half-destroyed index. `BEGIN IMMEDIATE` takes the write lock up front and
+     * `ROLLBACK` restores the previous rows exactly.
+     *
+     * Statements are prepared once per distinct SQL string and rebound per row.
+     * An index rebuild is one `DELETE` followed by N identical `INSERT`s, so
+     * this turns N+1 prepares into exactly 2 regardless of corpus size.
+     */
+    async executeAtomicWrite(statements) {
+      const preparedBySql = /* @__PURE__ */ new Map();
+      const prepareOnce = (sql) => {
+        let prepared = preparedBySql.get(sql);
+        if (prepared === void 0) {
+          prepared = database.prepare(sql);
+          preparedBySql.set(sql, prepared);
+        }
+        return prepared;
+      };
+      await database.exec("BEGIN IMMEDIATE");
+      try {
+        for (const statement of statements) {
+          const prepared = prepareOnce(statement.sql);
+          await (statement.args === void 0 ? prepared.run() : prepared.run(statement.args));
+        }
+        await database.exec("COMMIT");
+      } catch (error) {
+        try {
+          await database.exec("ROLLBACK");
+        } catch (rollbackError) {
+          warnOnFailedRollback(rollbackError);
+        }
+        throw error;
+      } finally {
+        for (const prepared of preparedBySql.values()) {
+          closeStatement(prepared);
+        }
+        preparedBySql.clear();
+      }
+    }
+  };
+}
+function closeStatement(statement) {
+  try {
+    statement.close?.();
+  } catch {
+  }
+}
+const NO_ACTIVE_TRANSACTION_PATTERN = /no transaction is active/i;
+function warnOnFailedRollback(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  if (NO_ACTIVE_TRANSACTION_PATTERN.test(message)) {
+    return;
+  }
+  console.warn(
+    `[libsql-search] ROLLBACK failed after a failed index replacement: ${message}. This database handle may still be inside an open transaction, in which case a later write on it fails with "cannot start a transaction within a transaction". Reconnect the handle before reusing it.`
+  );
+}
+function assertTursoDatabase(database) {
+  if (typeof database !== "object" || database === null || typeof database.exec !== "function" || typeof database.prepare !== "function") {
+    throw new TypeError(
+      "tursoAdapter() expects a connected @tursodatabase/database handle with exec() and prepare() methods. connect() returns a Promise, so remember to await it."
+    );
+  }
+}
+
+export { tursoAdapter };