@lunora/ai 1.0.0-alpha.59 → 1.0.0-alpha.60

package/dist/rag/index.d.mts

@@ -8,6 +8,129 @@ import { E as EmbeddingModelInput, a as LunoraAi } from "../packem_shared/types.
  * @experimental
  */
 declare const fixedWindowChunks: (text: string, size: number, overlap: number) => ReadonlyArray<string>;
+/**
+ * Options shared by the character-budgeted chunkers. `overlap` is a **floor**
+ * in characters: enough trailing atoms are carried into the next window to
+ * cover it, so the realised overlap lands on an atom boundary and is usually a
+ * little larger.
+ */
+interface ChunkerOptions {
+  /** Overlap floor between adjacent chunks, in characters. Default 200. Must be `< size`. */
+  overlap?: number;
+  /** Maximum chunk size, in characters. Default 1000. */
+  size?: number;
+}
+/**
+ * Options for {@link tokenChunker}.
+ *
+ * `countTokens` is **required and injected**: a real token count needs the
+ * model's tokenizer, and `@lunora/ai` will not add one as a dependency or
+ * pretend a characters-per-token constant is a token count — a "token-aware"
+ * chunker built on an estimate is a character chunker with extra steps, and it
+ * silently overshoots on code and non-Latin scripts, which is exactly where the
+ * budget matters. Pass `js-tiktoken`, `gpt-tokenizer`, or your provider's
+ * counter.
+ */
+interface TokenChunkerOptions {
+  /** Token counter for the target model — e.g. `(text) => encoding.encode(text).length`. */
+  countTokens: (text: string) => number;
+  /** Maximum chunk size, in tokens. Default 256. */
+  maxTokens?: number;
+  /** Overlap floor between adjacent chunks, in tokens. Default 0. Must be `< maxTokens`. */
+  overlapTokens?: number;
+}
+/**
+ * Split on sentence boundaries, then greedily pack whole sentences into
+ * `size`-bounded chunks. Prefer this over the fixed window for prose: chunks
+ * start and end where the author ended a thought, which is what the embedding
+ * model was trained on.
+ * @experimental
+ */
+declare const sentenceChunker: (options?: ChunkerOptions) => ((text: string) => ReadonlyArray<string>);
+/**
+ * Split a Markdown document at its ATX headings, then pack each section's
+ * sentences into `size`-bounded chunks.
+ *
+ * Every emitted chunk is prefixed with its **heading trail** (`# Guide > ##
+ * Auth > ### OAuth`), so a chunk taken from deep inside a long document still
+ * carries the context that says what it is about. That prefix is what makes a
+ * mid-document chunk retrievable by a query naming its section rather than its
+ * prose, and it is why this beats {@link sentenceChunker} on structured docs.
+ * @experimental
+ */
+declare const markdownChunker: (options?: ChunkerOptions) => ((text: string) => ReadonlyArray<string>);
+/**
+ * Pack sentences into chunks bounded by a real **token** count rather than a
+ * character count. Use this when the embedding model's context window is the
+ * binding constraint — a 512-token model silently truncates anything longer, so
+ * the tail of an over-long chunk is embedded as if it were never written.
+ * @experimental
+ */
+declare const tokenChunker: (options: TokenChunkerOptions) => ((text: string) => ReadonlyArray<string>);
+/**
+ * What a store can hold and return. Every field is a hard limit `defineRag`
+ * enforces locally, so a breach fails here — naming the store and the limit —
+ * rather than at the backend with nothing saying why.
+ */
+interface RagVectorStoreCapabilities {
+  /**
+   * Ceiling on embedding dimensionality, or `false` for no limit.
+   *
+   * Vectorize stores at most 1536 at 32-bit precision, which rules out
+   * `text-embedding-3-large` (3072) and Qwen3-Embedding (4096). A store with
+   * no such limit declares `false` and those models just work.
+   */
+  maxDimensions: number | false;
+  /**
+   * Ceiling on a vector's id, in bytes, or `false` for no limit.
+   *
+   * Vectorize allows 64, and chunk ids are derived from the namespace and the
+   * caller's source id — a bucket key like
+   * `handbook/engineering/onboarding/day-one.md` under a uuid namespace is
+   * already past it. Without this the upsert is rejected at the far side with
+   * nothing naming the cause, the same failure {@link RagVectorStoreCapabilities.maxMetadataBytes}
+   * exists to replace.
+   */
+  maxIdBytes: number | false;
+  /**
+   * Ceiling on the serialized metadata object per vector, in bytes, or
+   * `false` for no limit. Covers the whole object — chunk text included when
+   * no `textStore` moves it out.
+   */
+  maxMetadataBytes: number | false;
+  /** Ceiling on `topK` when only indexed metadata is requested (text-store mode). */
+  maxTopK: number;
+  /** Ceiling on `topK` when the query asks for full metadata (the default mode). */
+  maxTopKWithMetadata: number;
+}
+/**
+ * The storage operations `defineRag` needs. Deliberately the same four
+ * operations `RagVectors` already exposes — this is a capability-carrying
+ * wrapper, not a new protocol, so adapting an existing implementation is a
+ * one-liner.
+ */
+interface RagVectorStore {
+  capabilities: RagVectorStoreCapabilities;
+  deleteByIds: (ids: ReadonlyArray<string>, namespace?: string) => Promise<unknown>;
+  getByIds: (ids: ReadonlyArray<string>, namespace?: string) => Promise<ReadonlyArray<RagVectorRecord>>;
+  query: (input: RagVectorQueryInput) => Promise<RagVectorMatches>;
+  upsert: (input: RagVectorUpsertInput) => Promise<unknown>;
+}
+/**
+ * Vectorize's documented limits.
+ *
+ * `maxTopKWithMetadata` is 50, matching Vectorize V2. **Legacy V1 indexes cap
+ * at 20** and reject a larger `topK` remotely; a binding handle does not expose
+ * its index version, so this cannot branch on it.
+ */
+declare const VECTORIZE_CAPABILITIES: RagVectorStoreCapabilities;
+/**
+ * Wrap a `ctx.vectors` facade (or any {@link RagVectors}) as a store declaring
+ * Vectorize's limits. This is the default when no `store` is configured, so the
+ * behaviour of an existing `defineRag` is unchanged.
+ * @experimental
+ */
+declare const vectorizeStore: (vectors: RagVectors, indexName: string) => RagVectorStore;
 /**
  * `(text) => vector` — the embedder shape `ctx.vectors` accepts on both its
  * write (`upsert`) and read (`query`) inputs. Matches `@lunora/server`'s
@@ -131,12 +254,22 @@ interface RagContext {
    * untraced when it is absent (a hand-built context / test).
    */
   trace?: unknown;
-  vectors: RagVectors;
+  /**
+   * The Vectorize facade backing the default store — an `ActionCtx`'s
+   * `ctx.vectors` satisfies it.
+   *
+   * OPTIONAL, because it is never read when {@link RagConfig.store} is set,
+   * and codegen only emits `ctx.vectors` for a schema that declares a vector
+   * index. Requiring it made "the store that needs no Vectorize" impossible
+   * to type-check in an app with no Vectorize index. Absent with no `store`
+   * configured throws a directed error when the RAG is bound.
+   */
+  vectors?: RagVectors;
 }
 /**
  * Pluggable chunk-text storage. By default chunk text is stored in vector
  * metadata (`__ragText`), which forces `returnMetadata: "all"` on retrieval and
- * caps `topK` at 20 (the Vectorize full-metadata ceiling) — and each vector's
+ * caps `topK` at 50 (the Vectorize full-metadata ceiling) — and each vector's
  * metadata must stay under the ~10 KiB Vectorize cap. Supplying a text store
  * (a DO table, KV, …) moves the text out of metadata: retrieval queries with
  * `returnMetadata: "indexed"` (topK up to 100) and hydrates text by chunk id.
@@ -163,6 +296,16 @@ interface RagTextStore {
 interface StoredRagChunk {
   chunkIndex: number;
   id: string;
+  /**
+   * The caller `metadata` attached to this chunk's source at index time
+   * (internal `__rag*` keys excluded).
+   *
+   * Present so a {@link RagLexicalStore} can evaluate the same metadata
+   * filter the vector leg gets. Without it a lexical store has nothing to
+   * filter on and must fail closed on every filtered query — which made
+   * hybrid search and metadata-based RLS mutually exclusive.
+   */
+  metadata?: Record<string, unknown>;
   sourceId: string;
   text: string;
 }
@@ -235,6 +378,20 @@ interface RagNamedFilter {
   /** The filter expression passed verbatim to Vectorize's `filter` parameter. */
   filter: Record<string, unknown>;
 }
+/**
+ * Re-score retrieved candidates against the query. Returns the chunks in their
+ * new order; may drop chunks. See {@link RagConfig.rerank}.
+ * @experimental
+ */
+type RagReranker = (query: string, chunks: ReadonlyArray<RetrievedChunk>) => Promise<ReadonlyArray<RetrievedChunk>> | ReadonlyArray<RetrievedChunk>;
+/**
+ * Rewrite a query, or expand it into several. See {@link RagConfig.transformQuery}.
+ * @experimental
+ */
+type RagQueryTransform = (query: string, info: {
+  conversationId?: string;
+  namespace?: string;
+}) => Promise<ReadonlyArray<string> | string> | ReadonlyArray<string> | string;
 /**
  * `RagConfig` is part of the experimental `@lunora/ai` API and may change without a major version bump.
  * @experimental
@@ -247,6 +404,36 @@ interface RagConfig {
    * vectors across every tenant.
    */
   allowSharedNamespace?: boolean;
+  /**
+   * Retain up to this many embeddings per bound context, keyed by text, so a
+   * repeated `retrieve()` of the same question does not re-embed it.
+   *
+   * Default 0 (retain nothing beyond one call). Indexing always batches its
+   * embeds regardless of this setting — the batch lives in its own
+   * request-scoped map, released when `index()` returns, so it neither needs
+   * this budget nor evicts what is held in it.
+   *
+   * Sized in entries, not bytes, but budget in bytes: one 1536-dimension
+   * embedding is ~12 KB, so 100 entries is over a megabyte held in the
+   * isolate. Keep it small.
+   */
+  cacheEmbeddings?: number;
+  /**
+   * How many chunks each retrieval leg fetches **before** fusion and
+   * reranking trim the result to `topK`.
+   *
+   * Defaults to `topK * 4` whenever anything downstream reorders — a
+   * `lexicalStore`, a multi-query `transformQuery`, or a `rerank` — and to
+   * plain `topK` otherwise. Bounded by the store ceiling (50 in metadata
+   * mode, 100 with a `textStore`).
+   *
+   * This is the knob that decides how much recall the reordering has to work
+   * with. Fetching only `topK` per leg defeats the point of having two: the
+   * whole reason to run a lexical leg is to surface a chunk the vector leg
+   * ranked *below* `topK`, and it cannot do that if it was never asked for
+   * more than `topK`.
+   */
+  candidates?: number;
   /** Custom chunker; overrides the built-in fixed-window splitter. */
   chunk?: (text: string) => ReadonlyArray<string>;
   /** Overlap (chars) between adjacent chunks. Default 200. Must be < `chunkSize`. */
@@ -294,6 +481,23 @@ interface RagConfig {
   lexicalStore?: RagLexicalStore;
   /** Retrieval depth for the lexical leg of hybrid search. Defaults to the effective `topK`. */
   lexicalTopK?: number;
+  /**
+   * Ceiling on the embedding model's dimensionality, checked once per bound
+   * context against the first embedding actually produced. Defaults to
+   * **1536** — Vectorize's per-vector limit at 32-bit precision.
+   *
+   * The default rules out most current large embedding models
+   * (`text-embedding-3-large` and Gemini embedding at 3072,
+   * Qwen3-Embedding at 4096). Without the check they fail at Vectorize with
+   * nothing naming the cause; with it they fail at the first embed, naming
+   * the ceiling and both escapes — truncate via the provider's Matryoshka
+   * `dimensions` option, or set this to `false` when the index is not
+   * Vectorize-backed.
+   *
+   * `false` disables the check entirely: the right setting for a
+   * {@link RagVectors} implementation with a different (or no) ceiling.
+   */
+  maxEmbeddingDimensions?: number | false;
   /**
    * Enforce tenant isolation: throw (instead of the one-time dev warning)
    * when `index`/`retrieve`/`remove` run without a `namespace`. Recommended
@@ -301,6 +505,23 @@ interface RagConfig {
    * in metadata mode the leaked payload includes raw chunk text.
    */
   requireNamespace?: boolean;
+  /**
+   * Re-score the retrieved candidates against the query before they are
+   * trimmed to `topK`. **Injected, not bundled** — a reranker is a model call,
+   * and `@lunora/ai` takes no provider dependency to make one. Adapt yours
+   * with `scoreReranker`/`batchReranker`, or write the two-line hook yourself.
+   *
+   * This is the standard quality step that vector search alone cannot do:
+   * an embedding is computed without the query, so it cannot know which of
+   * two topically-similar passages actually answers *this* question. A
+   * cross-encoder sees both at once and orders them accordingly.
+   *
+   * Retrieval fetches {@link RagConfig.candidates} chunks, hands them
+   * here, and keeps the first `topK` of whatever comes back — so the hook may
+   * reorder and drop, but its output order is final. Runs after hybrid fusion
+   * and before `chunkContext` expansion.
+   */
+  rerank?: RagReranker;
   /**
    * Row-level-security filter derived from the retrieval identity. Called once
    * per `retrieve()` with {@link RagContext.auth} (the bound ctx's `auth`); the
@@ -319,10 +540,42 @@ interface RagConfig {
    * ```
    */
   rlsFilter?: (auth: unknown) => Promise<Record<string, unknown> | undefined> | Record<string, unknown> | undefined;
+  /**
+   * Back this RAG with a different vector store.
+   *
+   * Called once per bound context with that context, so a store needing
+   * per-request state (a Hyperdrive/pgvector connection from `ctx.sql`, a
+   * shard's own SQLite) can build itself from it. Defaults to wrapping
+   * `context.vectors` as a Vectorize-backed store.
+   *
+   * The store declares its own limits, and `defineRag` reads them instead of
+   * assuming Vectorize's — so a pgvector index is not held to a
+   * 1536-dimension ceiling or a 10 KiB metadata budget it does not have.
+   */
+  store?: (context: RagContext) => RagVectorStore;
   /** Chunk-text storage override — see {@link RagTextStore}. */
   textStore?: RagTextStore;
-  /** Default retrieval depth. Default 5. Capped at 20 (metadata mode) / 100 (text-store mode). */
+  /** Default retrieval depth. Default 5. Capped by the store: 50 (metadata mode) / 100 (text-store mode) on Vectorize. */
   topK?: number;
+  /**
+   * Rewrite or expand the query before it is embedded.
+   *
+   * The raw user query is often the worst possible search string: a
+   * conversational follow-up ("what about the other one?") carries its
+   * meaning in the preceding turns, and a short question shares few terms
+   * with the long passage that answers it.
+   *
+   * Return **one** string to rewrite, or **several** to run multi-query
+   * retrieval — each is embedded and searched independently and the rankings
+   * are fused with RRF, which recovers passages any single phrasing would
+   * miss. Returning the query unchanged is a no-op.
+   *
+   * **Injected, not bundled**, for the same reason as {@link RagConfig.rerank}:
+   * every useful strategy (HyDE, multi-query expansion, follow-up rewriting)
+   * needs a language model, and this package does not pick one for you. The
+   * lexical leg searches the first returned query.
+   */
+  transformQuery?: RagQueryTransform;
 }
 /**
  * `IndexInput` is part of the experimental `@lunora/ai` API and may change without a major version bump.
@@ -356,6 +609,18 @@ interface IndexInput {
     text: string;
     total: number;
   }) => void;
+  /**
+   * Index this source even when its content hash is unchanged.
+   *
+   * The hash short-circuit skips chunking, embedding and every write — which
+   * is what makes a cron re-sync cheap, and also what makes attaching a
+   * `textStore` or `lexicalStore` to an ALREADY-indexed corpus a silent no-op:
+   * the new store is never written, so the keyword leg returns nothing
+   * forever with no error. Set this for the one pass that backfills it.
+   *
+   * It re-embeds, so it is not a setting to leave on.
+   */
+  reindex?: boolean;
   /** The document body to chunk + embed + upsert. */
   text: string;
 }
@@ -415,7 +680,15 @@ interface RetrieveOptions {
     matches: number;
     query: string;
   }) => void;
+  /**
+   * Set `false` to skip {@link RagConfig.rerank} for this call — the escape
+   * hatch for a latency-sensitive path (typeahead, an agent's inner loop)
+   * that cannot afford the extra model round-trip.
+   */
+  rerank?: false;
   topK?: number;
+  /** Set `false` to skip {@link RagConfig.transformQuery} for this call. */
+  transformQuery?: false;
 }
 /**
  * `RetrievedChunk` is part of the experimental `@lunora/ai` API and may change without a major version bump.
@@ -524,15 +797,26 @@ declare const contentHash: (data: BufferSource) => Promise<string>;
  *
  * Each result set contributes `1 / (k + rank)` to each chunk's fused score,
  * where `rank` is 0-based position in the list. The constant `k` (default 60)
- * dampens the influence of high ranks — the standard value from the RRF
- * literature that works well across domains.
+ * dampens the influence of high ranks.
  *
- * The fused list is sorted descending by fused score. Ties are broken by
- * preferring the chunk ranked higher in the vector search result (typically
- * the more semantically accurate of the two methods).
+ * **The returned chunks carry the fused score in `score`**, multiplied by the
+ * chunk's `importance` so source weighting still applies. Writing it back is
+ * what makes the fusion survive: a caller that re-sorts by `score` — as
+ * `retrieve()` does, to apply importance weighting — would otherwise re-order
+ * by the incomparable inputs and discard the ranking this function computed.
+ * Since BM25 is unbounded while cosine is `[0, 1]`, that silently promoted
+ * every lexical-only hit above every vector hit.
+ *
+ * So in hybrid mode `RetrievedChunk.score` is an RRF score (small, ~`1/60`
+ * scale), not a cosine similarity. `retrieve()` applies `minScore` to the
+ * vector leg *before* fusion for exactly this reason — the option is documented
+ * against the cosine scale.
+ *
+ * Ties are broken by preferring the chunk ranked higher in the vector result,
+ * typically the more semantically accurate of the two methods.
  *
  * Callers MUST ensure every chunk in both lists carries a unique, comparable
- * `id` — this is guaranteed by the chunk-id scheme `${sourceId}#${chunkIndex}`.
+ * `id` — guaranteed by the chunk-id scheme `${sourceId}#${chunkIndex}`.
  * @experimental
  */
 declare const hybridRank: (vectorResults: ReadonlyArray<RetrievedChunk>, textResults: ReadonlyArray<RetrievedChunk>, k?: number) => ReadonlyArray<RetrievedChunk>;
@@ -545,16 +829,260 @@ declare const hybridRank: (vectorResults: ReadonlyArray<RetrievedChunk>, textRes
  * {@link RagLexicalStore} (a DO-SQLite inverted index, D1, or an external search
  * service) behind the same seam.
  *
- * Tenant isolation is by `namespace` (each namespace keeps its own index). This
- * store holds **no metadata**, so it cannot evaluate a metadata `filter`
- * (including an `rlsFilter` result): when `search` is called with a non-empty
- * filter it **fails closed** — returns no lexical hits and warns once — rather
- * than risk surfacing a row the filter would exclude. If your RLS is
- * metadata-based (not namespace-based) and you want a lexical leg, fold the RLS
- * dimension into the `namespace` or plug a filter-aware store.
+ * Tenant isolation is by `namespace` (each namespace keeps its own index), and
+ * each chunk's source `metadata` is stored alongside it so `search` evaluates
+ * the **same** metadata predicate the vector leg receives — including an
+ * `rlsFilter` result. A hit the filter excludes never reaches fusion.
+ *
+ * That matters because the filter carries the tenant/RBAC scope: a lexical leg
+ * that ignored it would leak excluded chunk text into the fused result no
+ * matter what the vector leg returned. This store previously had no metadata to
+ * check and so refused every filtered query, which made hybrid search and
+ * metadata-based RLS mutually exclusive.
  * @experimental
  */
 declare const bm25LexicalStore: () => RagLexicalStore;
+/**
+ * True when `metadata` satisfies every clause of `filter`. An empty or absent
+ * filter matches everything; absent metadata satisfies only an empty filter.
+ *
+ * Clauses are ANDed, matching Vectorize.
+ */
+declare const matchesMetadataFilter: (metadata: Record<string, unknown> | undefined, filter: Record<string, unknown> | undefined) => boolean;
+/** Options for {@link scoreReranker}. */
+interface ScoreRerankerOptions {
+  /**
+   * Drop chunks scoring below this threshold, in the scorer's own scale.
+   * Omitted → nothing is dropped and the reranker only reorders.
+   *
+   * Worth setting: reranking's real value is not just promoting the best
+   * passage but *rejecting* the ones vector search only matched topically,
+   * and a chunk that survives to the prompt is a chunk the model may cite.
+   */
+  minScore?: number;
+  /**
+   * Score one `(query, passage)` pair for relevance. Higher is better; the
+   * scale does not matter, only the ordering it induces.
+   *
+   * Called once per candidate chunk. Wrap a Workers AI reranker
+   * (`ctx.ai.run("@cf/baai/bge-reranker-base", …)`), a Cohere/Voyage rerank
+   * endpoint, or an LLM prompted to rate relevance.
+   */
+  score: (query: string, text: string) => Promise<number> | number;
+}
+/**
+ * Batched variant — score every candidate in one call.
+ */
+interface BatchRerankerOptions {
+  /** Drop chunks scoring below this threshold. See {@link ScoreRerankerOptions.minScore}. */
+  minScore?: number;
+  /**
+   * Score all candidates at once, returning one number per passage **in the
+   * same order**. Prefer this over {@link ScoreRerankerOptions.score} when the
+   * provider has a batch endpoint: cross-encoder APIs are priced and rate
+   * limited per request, so N passages in one call beats N calls.
+   *
+   * A result whose length does not match the input is rejected rather than
+   * zipped against the wrong passages.
+   */
+  scoreAll: (query: string, texts: ReadonlyArray<string>) => Promise<ReadonlyArray<number>> | ReadonlyArray<number>;
+}
+/**
+ * Adapt a per-passage relevance scorer into a {@link RagReranker}.
+ *
+ * The scorer is **injected** — `@lunora/ai` takes no provider dependency to
+ * make a model call. Scoring runs with bounded concurrency so a 50-candidate
+ * pool does not fan out 50 simultaneous subrequests.
+ *
+ * ```ts
+ * defineRag({
+ *   index: "docs",
+ *   rerank: scoreReranker({
+ *     score: async (query, text) => {
+ *       const result = await ctx.ai.run("@cf/baai/bge-reranker-base", { query, contexts: [{ text }] });
+ *       return result.response[0].score;
+ *     },
+ *   }),
+ * });
+ * ```
+ * @experimental
+ */
+declare const scoreReranker: (options: ScoreRerankerOptions) => RagReranker;
+/**
+ * Adapt a **batch** relevance scorer into a {@link RagReranker} — one call for
+ * the whole candidate pool. See {@link BatchRerankerOptions.scoreAll}.
+ * @experimental
+ */
+declare const batchReranker: (options: BatchRerankerOptions) => RagReranker;
+/** One object listed by a {@link RagObjectSource}. */
+interface RagSourceObject {
+  /** Content type, when the source knows it. Falls back to the key's extension. */
+  contentType?: string;
+  /** Stable key — becomes the indexed source id. */
+  key: string;
+  /** Metadata attached to every chunk of this object. */
+  metadata?: Record<string, unknown>;
+}
+/**
+ * Where documents come from. Two operations, both injected.
+ *
+ * `list` is an async iterable rather than an array so the caller decides how to
+ * page a large bucket rather than being forced to build one array up front.
+ * `sync` still collects the KEYS it yields — it needs the full current key set
+ * to work out what disappeared — but never more than one object's BODY at a
+ * time, which is where the memory actually is.
+ */
+interface RagObjectSource {
+  /** Fetch one object's raw text. Return `undefined` to skip it (unreadable, unsupported). */
+  get: (object: RagSourceObject) => Promise<string | undefined> | string | undefined;
+  /** Enumerate the objects to index. */
+  list: () => AsyncIterable<RagSourceObject> | Iterable<RagSourceObject>;
+}
+/** Turn a fetched object's raw text into indexable plain text. */
+type RagExtractor = (raw: string, object: RagSourceObject) => Promise<string | undefined> | string | undefined;
+/** Options for {@link defineRagSource}. */
+interface RagSourceOptions {
+  /**
+   * How many objects to process at once. Default 4.
+   *
+   * Each one is a fetch plus an embed plus an upsert, so this multiplies into
+   * the subrequest budget — the default is deliberately low.
+   */
+  concurrency?: number;
+  /**
+   * Extractors keyed by content type (`text/html`, `application/pdf`, …), or
+   * `"*"` as a fallback. A content type with no extractor and no `"*"` entry
+   * is skipped and counted in `skipped` — never indexed as raw bytes, which
+   * would fill the index with markup or binary noise that embeds to nothing
+   * meaningful.
+   */
+  extractors?: Record<string, RagExtractor>;
+  /** Namespace (tenant/shard key) applied to every indexed object. */
+  namespace?: string;
+  /** Called after each object is handled — for progress reporting. */
+  onObject?: (info: {
+    chunks: number;
+    key: string;
+    status: "indexed" | "skipped" | "unchanged";
+  }) => void;
+}
+/** Per-pass options for {@link RagSourceSync.sync}. */
+interface RagSyncPassOptions {
+  /**
+   * The keys this index is believed to already hold. Any of them missing from
+   * this pass's `list()` is deleted from the index.
+   *
+   * Pruning is what makes the index a mirror rather than an append-only pile:
+   * a document removed at the source but left in the index keeps being
+   * retrieved and cited, which is worse than never having indexed it.
+   *
+   * It is the CALLER's set, and required, because nothing else can hold it
+   * honestly. A previous pass's keys remembered in the instance would be
+   * empty on the shape this helper is documented in — built per request, from
+   * a per-request `rag(ctx)` — so a prune that defaulted on would in practice
+   * never run and never say so. Persist the set (a table, a KV key, the
+   * bucket listing itself) and hand it in.
+   *
+   * Omitted ⇒ nothing is pruned, and `pruned` comes back empty.
+   */
+  knownKeys?: Iterable<string>;
+}
+/** What one {@link RagSourceSync.sync} pass did. */
+interface RagSyncReport {
+  /** Keys indexed for the first time or re-indexed after a change. */
+  indexed: string[];
+  /** Keys deleted from the index because they no longer exist at the source. Empty unless `knownKeys` was supplied. */
+  pruned: string[];
+  /** Keys skipped — no extractor, or the source returned nothing. */
+  skipped: string[];
+  /** Keys whose content hash matched, so nothing was embedded. */
+  unchanged: string[];
+}
+/** The bound ingestion surface. */
+interface RagSourceSync {
+  /** Run one full pass over the source. */
+  sync: (source: RagObjectSource, passOptions?: RagSyncPassOptions) => Promise<RagSyncReport>;
+}
+/**
+ * Declare a bulk-ingestion pass over a {@link Rag}.
+ *
+ * ```ts
+ * const ingest = defineRagSource(docs(ctx), { namespace: ctx.shardKey });
+ *
+ * const report = await ingest.sync(
+ *     {
+ *         list: async function* () {
+ *             for await (const object of bucket.list()) {
+ *                 yield { key: object.key, metadata: { url: object.key } };
+ *             }
+ *         },
+ *         get: async (object) => (await bucket.get(object.key))?.text(),
+ *     },
+ *     // Pass what you already indexed to have deletions mirrored; omit it and
+ *     // nothing is pruned.
+ *     { knownKeys: await ctx.db.query("indexedDocs").collect().then((rows) => rows.map((row) => row.key)) },
+ * );
+ * ```
+ * @experimental
+ */
+declare const defineRagSource: (rag: Rag, options?: RagSourceOptions) => RagSourceSync;
+/**
+ * The injected SQL seam the durable RAG stores share.
+ *
+ * `@lunora/ai` depends on no database. A store is handed an executor instead,
+ * so the same adapter runs on a Durable Object's SQLite, D1, `node:sqlite`, or
+ * Postgres over Hyperdrive — and this package stays free of every one of them.
+ *
+ * The statements the stores emit are SQLite-shaped (`ON CONFLICT … DO UPDATE`,
+ * `CREATE INDEX IF NOT EXISTS`, `?` placeholders), so the executor must front a
+ * SQLite engine — a Durable Object, D1, or `node:sqlite`.
+ * @experimental
+ */
+/**
+ * Run one statement and return its rows.
+ *
+ * Rows come back as plain objects keyed by column name — the shape
+ * `SqlStorage#exec().toArray()`, D1's `.all().results`, and `postgres.js` all
+ * already produce. A statement returning nothing yields an empty array.
+ *
+ * May be synchronous: a Durable Object's SQLite is, and forcing it through a
+ * promise would add a microtask per row batch for nothing.
+ */
+type RagSqlExec = (sql: string, parameters: ReadonlyArray<unknown>) => Promise<ReadonlyArray<Record<string, unknown>>> | ReadonlyArray<Record<string, unknown>>;
+/** Options for {@link sqlLexicalStore}. */
+interface SqlLexicalStoreOptions {
+  /** Execute one statement. See {@link RagSqlExec}. */
+  exec: RagSqlExec;
+  /**
+   * Table-name prefix. Default `lunora_rag_lexical`; the postings table is
+   * this plus `_terms`. Must be a bare SQL identifier.
+   */
+  table?: string;
+}
+declare const sqlLexicalStore: (options: SqlLexicalStoreOptions) => RagLexicalStore;
+/** Options for {@link sqliteVectorStore}. */
+interface SqliteVectorStoreOptions {
+  /** Execute one statement. See {@link RagSqlExec}. */
+  exec: RagSqlExec;
+  /**
+   * Ceiling on embedding dimensionality. Defaults to `false` (no limit) —
+   * vectors are stored as JSON, so nothing here cares how wide they are, and
+   * inheriting Vectorize's 1536 would be inventing a constraint.
+   */
+  maxDimensions?: number | false;
+  /**
+   * Upper bound on how many rows a single namespace may be scanned for.
+   * Default 50,000.
+   *
+   * Search is linear, so this is the difference between a slow query and a
+   * Worker that exceeds its CPU budget and is killed with nothing explaining
+   * why. Exceeding it throws, naming the namespace and the count.
+   */
+  maxScan?: number;
+  /** Table name. Default `lunora_rag_vectors`. Must be a bare SQL identifier. */
+  table?: string;
+}
+declare const sqliteVectorStore: (options: SqliteVectorStoreOptions) => RagVectorStore;
 /**
  * Keep a RAG index in step with a table, so the table IS the index.
  *
@@ -658,4 +1186,4 @@ declare const ragSyncTriggers: <Document extends Record<string, unknown> = Recor
   afterInsert: RagSyncHandler;
   afterUpdate: RagSyncHandler;
 };
-export { type IndexInput, type IndexResult, type LexicalMatch, type Rag, type RagConfig, type RagContext, type RagEmbedder, type RagLexicalStore, type RagNamedFilter, type RagSource, type RagSyncActionReference, type RagSyncArgs, type RagSyncOptions, type RagTextStore, type RagToolOptions, type RagVectorMatch, type RagVectorMatches, type RagVectorQueryInput, type RagVectorRecord, type RagVectorUpsertInput, type RagVectors, type RemoveInput, type RetrieveOptions, type RetrieveResult, type RetrievedChunk, type StoredRagChunk, bm25LexicalStore, contentHash, defineRag, fixedWindowChunks, guessMimeTypeFromExtension, hybridRank, ragSyncTriggers };
+export { type BatchRerankerOptions, type ChunkerOptions, type IndexInput, type IndexResult, type LexicalMatch, type Rag, type RagConfig, type RagContext, type RagEmbedder, type RagExtractor, type RagLexicalStore, type RagNamedFilter, type RagObjectSource, type RagQueryTransform, type RagReranker, type RagSource, type RagSourceObject, type RagSourceOptions, type RagSourceSync, type RagSqlExec, type RagSyncActionReference, type RagSyncArgs, type RagSyncOptions, type RagSyncPassOptions, type RagSyncReport, type RagTextStore, type RagToolOptions, type RagVectorMatch, type RagVectorMatches, type RagVectorQueryInput, type RagVectorRecord, type RagVectorStore, type RagVectorStoreCapabilities, type RagVectorUpsertInput, type RagVectors, type RemoveInput, type RetrieveOptions, type RetrieveResult, type RetrievedChunk, type ScoreRerankerOptions, type SqlLexicalStoreOptions, type SqliteVectorStoreOptions, type StoredRagChunk, type TokenChunkerOptions, VECTORIZE_CAPABILITIES, batchReranker, bm25LexicalStore, contentHash, defineRag, defineRagSource, fixedWindowChunks, guessMimeTypeFromExtension, hybridRank, markdownChunker, matchesMetadataFilter, ragSyncTriggers, scoreReranker, sentenceChunker, sqlLexicalStore, sqliteVectorStore, tokenChunker, vectorizeStore };