libsql-search 0.7.0 → 0.8.0

package/docs/INTEGRATIONS.md

@@ -1,7 +1,120 @@
 # Integration Examples
 
-These examples show the current exported API wired into typical server-side
-routes. They are intentionally small so you can adapt them to your app.
+These examples show the current exported API wired into common server-side flows. Start with the shared provider flow, then adapt the framework snippets to your app.
+
+## Shared Provider Flow
+
+Use one provider preset, one dimension count, and one table name per embedding space.
+
+```ts
+import { createClient } from "@libsql/client";
+import { createTable, indexContent, search } from "libsql-search";
+
+const client = createClient({
+  url: process.env.TURSO_DB_URL!,
+  authToken: process.env.TURSO_AUTH_TOKEN!,
+});
+
+const providerConfig = {
+  tableName: "articles_openai_1536",
+  dimensions: 1536,
+  embeddingOptions: {
+    provider: "openai" as const,
+    apiKey: process.env.OPENAI_API_KEY,
+    dimensions: 1536,
+  },
+};
+
+await createTable(client, providerConfig.tableName, providerConfig.dimensions);
+
+await indexContent({
+  client,
+  contentPath: "./content",
+  tableName: providerConfig.tableName,
+  embeddingOptions: {
+    ...providerConfig.embeddingOptions,
+    intent: "document",
+  },
+});
+
+const results = await search({
+  client,
+  tableName: providerConfig.tableName,
+  query: "deployment checklist",
+  limit: 5,
+  embeddingOptions: {
+    ...providerConfig.embeddingOptions,
+    intent: "query",
+  },
+});
+```
+
+## Provider Presets
+
+These presets keep credentials out of the source file while making dimensions and table names explicit. Hosted presets send content to external providers and may incur provider charges when you run indexing or search.
+
+```ts
+const providerPresets = {
+  local: {
+    tableName: "articles_local_384",
+    dimensions: 384,
+    embeddingOptions: {
+      provider: "local" as const,
+    },
+  },
+  cloudflare: {
+    tableName: "articles_cf_bgem3_1024",
+    dimensions: 1024,
+    embeddingOptions: {
+      provider: "cloudflare" as const,
+      accountId: process.env.CLOUDFLARE_ACCOUNT_ID,
+      apiToken: process.env.CLOUDFLARE_API_TOKEN,
+      dimensions: 1024,
+    },
+  },
+  mistral: {
+    tableName: "articles_mistral_1024",
+    dimensions: 1024,
+    embeddingOptions: {
+      provider: "mistral" as const,
+      apiKey: process.env.MISTRAL_API_KEY,
+      dimensions: 1024,
+    },
+  },
+  gemini: {
+    tableName: "articles_gemini_3072",
+    dimensions: 3072,
+    embeddingOptions: {
+      provider: "gemini" as const,
+      apiKey: process.env.GEMINI_API_KEY,
+      dimensions: 3072,
+    },
+  },
+  openai: {
+    tableName: "articles_openai_1536",
+    dimensions: 1536,
+    embeddingOptions: {
+      provider: "openai" as const,
+      apiKey: process.env.OPENAI_API_KEY,
+      dimensions: 1536,
+    },
+  },
+  openaiCompatible: {
+    tableName: "articles_tei_1024",
+    dimensions: 1024,
+    embeddingOptions: {
+      provider: "openai-compatible" as const,
+      baseUrl: process.env.EMBEDDING_BASE_URL,
+      model: process.env.EMBEDDING_MODEL,
+      dimensions: 1024,
+      apiKey: process.env.EMBEDDING_API_KEY,
+      batchSize: 32,
+    },
+  },
+} as const;
+```
+
+Pick one preset and use its `tableName` and `dimensions` all the way through `createTable()`, `indexContent()`, and `search()`. Do not point two providers or two dimension counts at the same table.
 
 ## Astro Search Endpoint
 
@@ -24,8 +137,10 @@ export const POST: APIRoute = async ({ request }) => {
     client,
     query,
     limit,
+    tableName: "articles_local_384",
     embeddingOptions: {
       provider: "local",
+      intent: "query",
     },
   });
 
@@ -47,14 +162,14 @@ const client = createClient({
 });
 
 export async function getStaticPaths() {
-  const articles = await getAllArticles(client);
+  const articles = await getAllArticles(client, "articles_local_384");
 
   return articles.map((article) => ({
     params: { slug: article.slug },
   }));
 }
 
-const article = await getArticleBySlug(client, "guides/getting-started");
+const article = await getArticleBySlug(client, "guides/getting-started", "articles_local_384");
 ```
 
 ## Next.js Route Handler
@@ -76,8 +191,10 @@ export async function POST(request: NextRequest) {
     client,
     query,
     limit,
+    tableName: "articles_local_384",
     embeddingOptions: {
       provider: "local",
+      intent: "query",
     },
   });
 
@@ -97,7 +214,7 @@ const client = createClient({
 });
 
 export async function generateStaticParams() {
-  const articles = await getAllArticles(client);
+  const articles = await getAllArticles(client, "articles_local_384");
 
   return articles.map((article) => ({
     slug: article.slug,
@@ -106,7 +223,7 @@ export async function generateStaticParams() {
 
 export default async function Page({ params }: { params: Promise<{ slug: string }> }) {
   const { slug } = await params;
-  const article = await getArticleBySlug(client, slug);
+  const article = await getArticleBySlug(client, slug, "articles_local_384");
 
   return <article>{article?.title}</article>;
 }
@@ -114,8 +231,6 @@ export default async function Page({ params }: { params: Promise<{ slug: string
 
 ## Build-Time Index Script
 
-A small script is usually enough to rebuild the index before a site build.
-
 ```ts
 import { createClient } from "@libsql/client";
 import { createTable, indexContent } from "libsql-search";
@@ -125,48 +240,113 @@ const client = createClient({
   authToken: process.env.TURSO_AUTH_TOKEN!,
 });
 
-const embeddingProvider =
-  process.env.EMBEDDING_PROVIDER as
-    | "local"
-    | "cloudflare"
-    | "mistral"
-    | "gemini"
-    | "openai"
-    | "openai-compatible"
-    | undefined;
-
-const dimensionsByProvider = {
-  local: 384,
-  cloudflare: 1024,
-  mistral: 1024,
-  gemini: 3072,
-  openai: 1536,
+const providerPresets = {
+  local: {
+    tableName: "articles_local_384",
+    dimensions: 384,
+    embeddingOptions: {
+      provider: "local" as const,
+    },
+  },
+  cloudflare: {
+    tableName: "articles_cf_bgem3_1024",
+    dimensions: 1024,
+    embeddingOptions: {
+      provider: "cloudflare" as const,
+      accountId: process.env.CLOUDFLARE_ACCOUNT_ID,
+      apiToken: process.env.CLOUDFLARE_API_TOKEN,
+      dimensions: 1024,
+    },
+  },
+  mistral: {
+    tableName: "articles_mistral_1024",
+    dimensions: 1024,
+    embeddingOptions: {
+      provider: "mistral" as const,
+      apiKey: process.env.MISTRAL_API_KEY,
+      dimensions: 1024,
+    },
+  },
+  gemini: {
+    tableName: "articles_gemini_3072",
+    dimensions: 3072,
+    embeddingOptions: {
+      provider: "gemini" as const,
+      apiKey: process.env.GEMINI_API_KEY,
+      dimensions: 3072,
+    },
+  },
+  openai: {
+    tableName: "articles_openai_1536",
+    dimensions: 1536,
+    embeddingOptions: {
+      provider: "openai" as const,
+      apiKey: process.env.OPENAI_API_KEY,
+      dimensions: 1536,
+    },
+  },
+  "openai-compatible": {
+    tableName: "articles_tei_1024",
+    dimensions: 1024,
+    embeddingOptions: {
+      provider: "openai-compatible" as const,
+      baseUrl: process.env.EMBEDDING_BASE_URL,
+      model: process.env.EMBEDDING_MODEL,
+      dimensions: 1024,
+      apiKey: process.env.EMBEDDING_API_KEY,
+      batchSize: 32,
+    },
+  },
 } as const;
 
-const embeddingDimensions =
-  embeddingProvider === "openai-compatible"
-    ? Number(process.env.EMBEDDING_DIMENSIONS)
-    : dimensionsByProvider[embeddingProvider ?? "local"];
+const provider = process.env.EMBEDDING_PROVIDER ?? "local";
+
+if (!(provider in providerPresets)) {
+  throw new Error(
+    `Unknown EMBEDDING_PROVIDER: ${provider}. Expected one of ${Object.keys(providerPresets).join(", ")}`
+  );
+}
+
+const preset = providerPresets[provider as keyof typeof providerPresets];
 
-await createTable(client, "articles", embeddingDimensions);
+await createTable(client, preset.tableName, preset.dimensions);
 
 await indexContent({
   client,
   contentPath: "./content",
+  tableName: preset.tableName,
   embeddingOptions: {
-    provider: embeddingProvider,
-    baseUrl: process.env.EMBEDDING_BASE_URL,
-    model: process.env.EMBEDDING_MODEL,
-    dimensions: embeddingDimensions,
+    ...preset.embeddingOptions,
+    intent: "document",
   },
 });
 ```
 
-Pair this with your framework build command so indexed content and deployed code
-stay in sync.
+## CI-safe Provider Coverage
+
+Keep routine CI on mocks only:
+
+```ts
+import { vi } from "vitest";
+import { generateEmbeddings } from "libsql-search";
+
+vi.stubGlobal("fetch", vi.fn().mockResolvedValue({
+  ok: true,
+  headers: new Headers(),
+  json: async () => ({
+    data: [
+      { index: 0, embedding: [1, 2] },
+      { index: 1, embedding: [3, 4] },
+    ],
+  }),
+}));
+
+await generateEmbeddings(["doc one", "doc two"], {
+  provider: "openai-compatible",
+  baseUrl: "https://tei.example.internal/v1",
+  model: "tei-model",
+  dimensions: 2,
+});
+```
 
-When you use `openai-compatible`, keep `EMBEDDING_BASE_URL` as trusted
-server-side configuration, not user request input. Recreate the vector table or
-rebuild into a separate table whenever the provider, endpoint, model, or
-dimension count changes; stored vectors and query vectors must come from the
-same embedding space.
+The repository test suite should not require real provider credentials. See [Testing guidance](./TESTING.md) for local model mocks, Gemini SDK mocks, and validation-before-network assertions.

package/docs/MIGRATIONS.md

@@ -0,0 +1,176 @@
+# Migration And Reindexing
+
+Changing an embedding provider is not just a config flip. Stored vectors and query vectors must stay in the same embedding space for search quality to hold.
+
+## What The Table Stores
+
+`createTable(client, tableName, dimensions)` creates an `embedding` column with the exact Turso/libSQL vector type:
+
+```sql
+embedding F32_BLOB(dimensions)
+```
+
+That width is fixed by the table schema. `CREATE TABLE IF NOT EXISTS` does not resize an existing vector column.
+
+References:
+
+- [Turso AI and embeddings](https://docs.turso.tech/features/ai-and-embeddings)
+- [Gemini model versions](https://ai.google.dev/gemini-api/docs/embeddings#model-versions)
+
+## Rules
+
+- if dimensions change, create a new table or recreate the old table, then fully re-embed
+- if dimensions stay the same but provider, model, endpoint, model revision, pooling, normalization, or input formatting changes, fully reindex anyway
+- never mix two embedding spaces in one table
+- prefer a parallel table migration because `indexContent()` replaces the whole target table, so an in-place rebuild leaves no way back to the old vectors
+
+In practice, this means:
+
+- `384 local` and `1024 Mistral` can never share a table
+- `1024 Cloudflare` and `1024 Mistral` still need separate rebuilds because equal width does not make the vectors compatible
+- a custom endpoint change at the same width still needs a new table because the model or serving stack may have changed
+
+## Safe Migration Pattern
+
+1. Pick a new table name that makes the provider and dimensions obvious.
+2. Create that table with the target width.
+3. Reindex all content into the new table.
+4. Run search quality checks against the new table.
+5. Switch application reads and writes to the new table.
+6. Retire the old table in a separate cleanup step.
+
+Step 3 is all or nothing. `indexContent()` throws `IndexingError` and leaves the target table untouched when a file or the replacement transaction fails, so a failed migration step can be retried without cleanup. See [Indexing and operational behavior](./INDEXING.md) for `failurePolicy` and `allowEmptyIndex`.
+
+Example:
+
+```ts
+await createTable(client, "articles_gemini2_3072", 3072);
+
+await indexContent({
+  client,
+  contentPath: "./content",
+  tableName: "articles_gemini2_3072",
+  embeddingOptions: {
+    provider: "gemini",
+    apiKey: process.env.GEMINI_API_KEY,
+    dimensions: 3072,
+    intent: "document",
+  },
+});
+```
+
+Then query the new table explicitly during validation:
+
+```ts
+const results = await search({
+  client,
+  tableName: "articles_gemini2_3072",
+  query: "deployment checklist",
+  embeddingOptions: {
+    provider: "gemini",
+    apiKey: process.env.GEMINI_API_KEY,
+    dimensions: 3072,
+    intent: "query",
+  },
+});
+```
+
+## Common Migration Paths
+
+| From | To | Why a rebuild is required | Recommended table move |
+| --- | --- | --- | --- |
+| Legacy padded local `768` | Native local `384` | Old tables stored `384` model values plus zero padding; current local provider is a native `384`-dimension space | Build into `articles_local_384`, validate, then retire the legacy table |
+| Any `768` space | Any `1024` space | Width changes from `F32_BLOB(768)` to `F32_BLOB(1024)` | Create a new `1024` table and reindex |
+| Cloudflare `1024` | Mistral `1024` | Width stays the same, but provider/model space changes | Use a parallel `1024` table such as `articles_mistral_1024` |
+| Mistral `1024` | Cloudflare `1024` | Same reason in reverse | Use a parallel `1024` table such as `articles_cf_bgem3_1024` |
+| Any `1024` space | OpenAI `1536` | Width changes and model changes | Create `articles_openai_1536` and reindex |
+| OpenAI `1536` | OpenAI or Gemini `3072` | Width changes to `3072` | Create a new `3072` table and reindex |
+| Gemini legacy `text-embedding-004` at `768` | Gemini `gemini-embedding-2` at `768` | Same width, but the upstream model changed and the adapter now distinguishes document/query formatting | Build a parallel `articles_gemini2_768` table |
+| Any custom endpoint/model | Any other custom endpoint/model, same width or different width | Endpoint, model, or serving revision may change the embedding space even when dimensions match | Always create a new table named for the target provider/model and reindex |
+
+## Scenario Notes
+
+### Legacy Local `768` To Native Local `384`
+
+Earlier local migrations sometimes relied on zero padding to fit a `768`-wide table. The current local adapter emits the model's native `384` dimensions and rejects any other local dimension count.
+
+Safe path:
+
+```ts
+await createTable(client, "articles_local_384", 384);
+```
+
+Reindex into `articles_local_384`; do not keep writing new local vectors into the legacy padded table.
+
+### `768` To `1024`
+
+Any move from `768` dimensions to `1024` dimensions changes the schema width. Examples include a legacy local table moving to Cloudflare or Mistral.
+
+```ts
+await createTable(client, "articles_mistral_1024", 1024);
+```
+
+### Cloudflare `1024` To Mistral `1024`
+
+This is the main same-width example. The schema width can stay `1024`, but the vector space still changes.
+
+```ts
+await createTable(client, "articles_cf_bgem3_1024", 1024);
+await createTable(client, "articles_mistral_1024", 1024);
+```
+
+Keep both tables side by side during validation. Do not clear the Cloudflare table until the Mistral table is validated in production-like queries.
+
+### `1024` To OpenAI `1536`
+
+```ts
+await createTable(client, "articles_openai_1536", 1536);
+```
+
+This is both a width change and a provider/model change.
+
+### `1536` To `3072`
+
+Applies when moving from OpenAI `1536` to OpenAI `3072`, or from OpenAI `1536` to Gemini `3072`.
+
+```ts
+await createTable(client, "articles_openai_3072", 3072);
+await createTable(client, "articles_gemini_3072", 3072);
+```
+
+Pick one target space and validate it before switching reads.
+
+### Gemini Older Model `768` To Gemini 2 `768`
+
+Even if you stay at `768`, rebuild because `text-embedding-004` has been replaced by `gemini-embedding-2`, and the current adapter uses different text formatting for document versus query intent.
+
+```ts
+await createTable(client, "articles_gemini2_768", 768);
+```
+
+### Custom Endpoint Or Model Change
+
+Treat any `openai-compatible` move as a new embedding space:
+
+- TEI model upgrade
+- base URL change
+- serving stack change
+- pooling or normalization change
+- gateway rewrite that changes input formatting
+
+Example:
+
+```ts
+await createTable(client, "articles_tei_bge_1024_v2", 1024);
+```
+
+## Quality Checks Before Cutover
+
+Run a small set of real queries against both old and new tables before switching:
+
+- high-value navigation queries
+- acronym or product-name queries
+- queries that depend on tags or frontmatter wording
+- long-tail queries that previously returned useful content
+
+If the new table looks worse, keep the old table live while you inspect chunk size, source content, provider choice, and dimensionality.