libsql-search 0.1.4 → 0.1.6

package/docs/INTEGRATIONS.md

@@ -0,0 +1,143 @@
+# Integration Examples
+
+These examples show the current exported API wired into typical server-side
+routes. They are intentionally small so you can adapt them to your app.
+
+## Astro Search Endpoint
+
+```ts
+import type { APIRoute } from "astro";
+import { createClient } from "@libsql/client";
+import { search } from "libsql-search";
+
+export const prerender = false;
+
+const client = createClient({
+  url: import.meta.env.TURSO_DB_URL,
+  authToken: import.meta.env.TURSO_AUTH_TOKEN,
+});
+
+export const POST: APIRoute = async ({ request }) => {
+  const { query, limit = 10 } = await request.json();
+
+  const results = await search({
+    client,
+    query,
+    limit,
+    embeddingOptions: {
+      provider: "local",
+      dimensions: 768,
+    },
+  });
+
+  return new Response(JSON.stringify({ results }), {
+    headers: { "Content-Type": "application/json" },
+  });
+};
+```
+
+## Astro Static Paths
+
+```ts
+import { createClient } from "@libsql/client";
+import { getAllArticles, getArticleBySlug } from "libsql-search";
+
+const client = createClient({
+  url: import.meta.env.TURSO_DB_URL,
+  authToken: import.meta.env.TURSO_AUTH_TOKEN,
+});
+
+export async function getStaticPaths() {
+  const articles = await getAllArticles(client);
+
+  return articles.map((article) => ({
+    params: { slug: article.slug },
+  }));
+}
+
+const article = await getArticleBySlug(client, "guides/getting-started");
+```
+
+## Next.js Route Handler
+
+```ts
+import { createClient } from "@libsql/client";
+import { search } from "libsql-search";
+import { NextRequest } from "next/server";
+
+const client = createClient({
+  url: process.env.TURSO_DB_URL!,
+  authToken: process.env.TURSO_AUTH_TOKEN!,
+});
+
+export async function POST(request: NextRequest) {
+  const { query, limit = 10 } = await request.json();
+
+  const results = await search({
+    client,
+    query,
+    limit,
+    embeddingOptions: {
+      provider: "local",
+      dimensions: 768,
+    },
+  });
+
+  return Response.json({ results });
+}
+```
+
+## Next.js Static Params
+
+```tsx
+import { createClient } from "@libsql/client";
+import { getAllArticles, getArticleBySlug } from "libsql-search";
+
+const client = createClient({
+  url: process.env.TURSO_DB_URL!,
+  authToken: process.env.TURSO_AUTH_TOKEN!,
+});
+
+export async function generateStaticParams() {
+  const articles = await getAllArticles(client);
+
+  return articles.map((article) => ({
+    slug: article.slug,
+  }));
+}
+
+export default async function Page({ params }: { params: Promise<{ slug: string }> }) {
+  const { slug } = await params;
+  const article = await getArticleBySlug(client, slug);
+
+  return <article>{article?.title}</article>;
+}
+```
+
+## Build-Time Index Script
+
+A small script is usually enough to rebuild the index before a site build.
+
+```ts
+import { createClient } from "@libsql/client";
+import { createTable, indexContent } from "libsql-search";
+
+const client = createClient({
+  url: process.env.TURSO_DB_URL!,
+  authToken: process.env.TURSO_AUTH_TOKEN!,
+});
+
+await createTable(client, "articles", 768);
+
+await indexContent({
+  client,
+  contentPath: "./content",
+  embeddingOptions: {
+    provider: process.env.EMBEDDING_PROVIDER as "local" | "gemini" | "openai" | undefined,
+    dimensions: 768,
+  },
+});
+```
+
+Pair this with your framework build command so indexed content and deployed code
+stay in sync.

package/docs/PROVIDERS.md

@@ -0,0 +1,100 @@
+# Embedding Providers
+
+`libsql-search` currently supports three embedding providers:
+
+- `local`
+- `gemini`
+- `openai`
+
+Use the same provider and dimensions for both indexing and querying. A mismatch
+between stored vectors and query vectors will break search quality or fail at
+query time.
+
+## Shared Options
+
+```ts
+interface EmbeddingOptions {
+  provider?: "local" | "gemini" | "openai";
+  apiKey?: string;
+  dimensions?: number;
+  maxLength?: number;
+}
+```
+
+- `provider` defaults to `"local"`
+- `dimensions` defaults to `768`
+- `maxLength` defaults to `8000`
+- `apiKey` is optional in code, but required for hosted providers unless the
+  matching environment variable is available
+
+## Local
+
+Provider value: `local`
+
+The local provider loads `Xenova/all-MiniLM-L6-v2` through
+`@xenova/transformers`.
+
+```ts
+embeddingOptions: {
+  provider: "local",
+  dimensions: 768,
+}
+```
+
+Notes:
+
+- the model emits 384 dimensions and `libsql-search` pads or truncates to your
+  requested size
+- the first run downloads the model and can take longer on a fresh machine
+- no API key is required
+
+## Gemini
+
+Provider value: `gemini`
+
+Gemini uses Google `text-embedding-004`.
+
+```ts
+embeddingOptions: {
+  provider: "gemini",
+  apiKey: process.env.GEMINI_API_KEY,
+}
+```
+
+Behavior:
+
+- if `apiKey` is omitted, the library reads `GEMINI_API_KEY`
+- Gemini returns 768 dimensions natively
+- the current implementation does not expose model selection
+
+## OpenAI
+
+Provider value: `openai`
+
+OpenAI uses `text-embedding-3-small` when `dimensions <= 1536` and
+`text-embedding-3-large` when `dimensions > 1536`.
+
+```ts
+embeddingOptions: {
+  provider: "openai",
+  apiKey: process.env.OPENAI_API_KEY,
+  dimensions: 1536,
+}
+```
+
+Behavior:
+
+- if `apiKey` is omitted, the library reads `OPENAI_API_KEY`
+- the request sends the `dimensions` value to the OpenAI embeddings API
+- use the same dimension count in `createTable()`
+
+## Dimension Guidelines
+
+- `768` is the easiest cross-provider target in the current implementation
+- local embeddings are padded from 384 to your target size
+- Gemini stays at 768
+- OpenAI can be used at 1536 or 3072, or another supported OpenAI dimension
+  value you explicitly set
+
+If you switch provider or dimensions for an existing table, recreate the table
+or rebuild the index into a separate table so stored vectors stay consistent.

package/docs/README.md

@@ -0,0 +1,16 @@
+# Documentation
+
+This directory holds the longer-form reference material for `libsql-search`.
+Start with the page that matches the job you are doing:
+
+- [Provider guide](./PROVIDERS.md): local, Gemini, and OpenAI embedding options,
+  dimensions, and API key behavior
+- [API reference](./API.md): exported functions, option shapes, and result data
+- [Integration examples](./INTEGRATIONS.md): Astro and Next.js server-side usage
+- [Indexing and operations](./INDEXING.md): content layout, rebuild scripts,
+  search quality tips, and indexing gotchas
+- [Troubleshooting](./TROUBLESHOOTING.md): known install/runtime issues
+- [Releasing](./RELEASING.md): maintainer release workflow
+
+For the shortest first-use path, go back to the repository
+[README](../README.md).

package/docs/RELEASING.md

@@ -0,0 +1,91 @@
+# Releasing
+
+Normal releases are automatic. Merge a reviewed PR to `main`; the `Publish
+Package` workflow decides the next stable SemVer version, creates any required
+manifest-only release commit, creates an annotated `vX.Y.Z` tag, publishes npm
+and JSR, then creates GitHub Release notes after both registries succeed.
+
+## Automatic release flow
+
+1. Every push to `main` starts `.github/workflows/publish.yml`.
+2. The workflow ignores self-generated `chore(release): ...` commits.
+3. A serialized release-writer job fetches the current `origin/main` after it
+   has the release slot. If `origin/main` moved beyond the triggering commit,
+   that older run exits and the newer run handles the accumulated changes.
+4. The release planner compares commits since the highest stable `vX.Y.Z` tag
+   merged into the release commit. Higher tags that are not reachable from
+   `main` are ignored so a stray or divergent tag cannot make the default
+   release line jump. Commits whose changed files are confined to `docs/**` or
+   `.github/**` do not count as release-eligible and do not affect the bump. If
+   the newest `main` commit is docs-only but an earlier untagged code or README
+   commit is still pending, that newest run releases the accumulated eligible
+   changes. Breaking changes bump major, `feat:` bumps minor, and all other
+   eligible commits bump patch.
+5. `package.json`, `jsr.json`, and `deno.json` are synchronized to the chosen
+   version. If they already match the chosen version, no release commit is
+   created.
+6. The workflow validates the candidate, creates an annotated tag, and atomically
+   pushes the release commit plus tag.
+7. npm publishes `libsql-search` through trusted publishing OIDC with the GitHub
+   environment named `npm`.
+8. JSR publishes `@logan/libsql-search` through a separate OIDC job.
+9. GitHub Release notes are generated only after both registry jobs complete.
+
+The current bootstrap case is supported: if the latest tag is `v0.1.3` and the
+manifests already say `0.1.4`, the first qualifying `main` release tags and
+publishes `v0.1.4` without creating an empty release commit.
+
+## Manual overrides
+
+Manual version and tag commands are overrides, not the default path. Use them
+only when you intentionally need to publish a specific already-reviewed `main`
+commit.
+
+1. Synchronize `package.json`, `jsr.json`, and `deno.json` to the target
+   `X.Y.Z` version.
+2. Merge that manifest change to `main`.
+3. Create annotated tag `vX.Y.Z` on the merged `main` commit.
+4. Push the tag.
+
+The same `Publish Package` workflow validates manual tags before publishing. It
+checks that the tag target is on `origin/main`, that the tag and manifests agree,
+and that the remote tag still resolves to the checked-out commit immediately
+before each registry publish.
+
+## Recovery dispatch
+
+Use the guarded workflow dispatch path when an existing `vX.Y.Z` tag needs a
+registry or GitHub Release recovery without creating a new version. Example:
+
+```bash
+gh workflow run publish.yml --ref main -f release_tag=v0.1.4
+```
+
+Dispatch validates that `release_tag` is strict `vX.Y.Z`, fetches the remote tag,
+peels it to a commit, checks that commit is on `origin/main`, checks manifests
+against the tag, and then publishes from the tag target SHA. Unlike a tag-push
+run, dispatch does not require the tag target to equal the workflow dispatch
+SHA; this allows a newer `main` workflow fix to recover an older release tag.
+
+npm and GitHub Release recovery steps remain idempotent and skip when the
+version or release already exists. JSR recovery always invokes
+`pnpm dlx jsr@0.14.3 publish`; do not use `deno info jsr:...` as an existence
+gate, because it can resolve successfully even when the public JSR package
+version metadata is not actually published.
+
+## GitHub and registry settings
+
+- npm trusted publishing must point at workflow filename `publish.yml` and use
+  the GitHub Actions environment named `npm`.
+- The workflow does not use `NPM_TOKEN`.
+- JSR publishing uses its own OIDC job and does not depend on the npm
+  environment.
+- The repository settings must allow GitHub Actions to push the generated
+  `chore(release): ...` commit to `main`; otherwise the automatic release job
+  will validate successfully and then fail at the atomic push step.
+- Configure a repository ruleset or tag protection rule for `v*` tags.
+- Restrict the `npm` environment to the `main` deployment branch for automatic
+  releases and `v*` tags for intentional manual overrides.
+
+GitHub Releases are an after-publish record. They are not a prerequisite for npm
+or JSR publishing.

package/docs/TROUBLESHOOTING-SHARP.md

@@ -0,0 +1,64 @@
+# Troubleshooting: Transitive `sharp` Install Errors
+
+`libsql-search` does not directly depend on `sharp`. If you see an install error
+mentioning `sharp`, it is coming from another dependency in your application or
+toolchain.
+
+This page exists because the error can show up in environments that also use
+`libsql-search`, and it is easy to misattribute the failure to this package.
+
+## Typical Error
+
+```text
+Cannot find module '../build/Release/sharp-*.node'
+```
+
+Or:
+
+```text
+Error: Something went wrong installing the "sharp" module
+```
+
+## Why It Happens
+
+With pnpm, native packages may need explicit build-script approval. If the
+relevant install script is blocked, the native binary is never downloaded or
+built.
+
+## What To Do
+
+First inspect which build scripts pnpm blocked:
+
+```bash
+pnpm ignored-builds
+```
+
+Then approve the package that is actually failing and reinstall:
+
+```bash
+pnpm approve-builds
+pnpm install
+```
+
+In the interactive `pnpm approve-builds` prompt, select `sharp` if that is the
+package reporting the native-module failure.
+
+For a committed repository-level fix, you can also allow the package explicitly
+in `pnpm-workspace.yaml` with `onlyBuiltDependencies`.
+
+## Relation To `libsql-search`
+
+- local embeddings use `@xenova/transformers`
+- the first local embedding run may download a model at runtime
+- that runtime model download is separate from a pnpm native-module install
+  failure
+
+## Verification
+
+After reinstalling, rerun the command that originally failed. If your app uses
+`sharp` directly, verify that import in your own project context.
+
+## Additional Resources
+
+- [pnpm approve-builds](https://pnpm.io/10.x/cli/approve-builds)
+- [Sharp installation docs](https://sharp.pixelplumbing.com/install)

package/docs/TROUBLESHOOTING.md

@@ -0,0 +1,12 @@
+# Troubleshooting
+
+Use the page that matches the failure mode:
+
+- [Sharp native module issues](./TROUBLESHOOTING-SHARP.md)
+
+Common operational checks:
+
+- verify you called `createTable()` before indexing or searching
+- verify the table dimension matches the embedding dimension in your code
+- verify the same provider is used for indexing and querying
+- verify hosted providers have `GEMINI_API_KEY` or `OPENAI_API_KEY` available

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libsql-search",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "Semantic search for static sites using libSQL/Turso with multi-provider embeddings",
   "type": "module",
   "packageManager": "pnpm@10.34.5",
@@ -15,7 +15,8 @@
   },
   "files": [
     "dist",
-    "README.md"
+    "README.md",
+    "docs"
   ],
   "scripts": {
     "build": "tsc --noEmit && rollup -c && rollup -c rollup.dts.config.js",