@dudousxd/nestjs-agent-rag 0.13.0 → 0.14.0

package/README.md

@@ -197,6 +197,9 @@ await store.ensureSchema(); // CREATE EXTENSION vector + table + HNSW cosine ind
 `store.schemaStatements()` returns the same DDL as strings, for a migration instead of a boot-time
 side effect. Other options (all off by default, so existing tables behave exactly as before):
 
+- **`vectorType`** (default `vector`) — `halfvec` stores 2 bytes per dimension and its HNSW index goes
+  up to 4000 dimensions, where `vector` stops at 2000. Needs a single width. Converting an existing
+  table is a migration of yours.
 - **`upsertBatchSize`** (default 100) — `upsert` writes multi-row `INSERT … ON CONFLICT` statements;
   duplicate ids in one call collapse to the last occurrence. NUL bytes are stripped from every
   text/jsonb binding (Postgres rejects them).
@@ -439,6 +442,69 @@ Had `PgVectorStore` simply grown `searchText`, every `isLexicalVectorStore(store
 started using it against an unindexed table on upgrade. Constructing `PgLexicalVectorStore` is the
 opt-in; the index is a migration you run and watch (`schemaStatements()` prints it).
 
+## RAG for a model: `RagRepository`
+
+One class per model, as `nestjs-filter` does with filters. The entity or Drizzle table doesn't change.
+
+```ts
+import { RagModule, RagRepository } from '@dudousxd/nestjs-agent-rag/nestjs';
+
+// Once in the app
+RagModule.forRoot({ store, embedder, onUsage: (usage) => meter.add(usage) });
+
+@Injectable()
+export class DocumentsRag extends RagRepository<ProjectDocument> {
+  protected readonly name = 'project_documents';
+  readonly scope = { projectId: (d: ProjectDocument) => d.project.id };
+
+  id(d: ProjectDocument) { return d.id; }
+  protected text(d: ProjectDocument) { return d.textContent; }
+}
+
+// In the feature module
+RagModule.forFeature([DocumentsRag]);
+
+// Injected by type
+await documentsRag.index(doc);
+await documentsRag.search('question', { scope: { projectId } });
+await documentsRag.remove(doc);
+```
+
+With Drizzle, only the row type changes: `type Row = typeof documents.$inferSelect`, then
+`RagRepository<Row>` and `scope = { projectId: (d: Row) => d.projectId }`.
+
+**The scope can't be skipped.** Leaving out `scope`, returning a nullable column, or calling `search`
+without it doesn't compile. At runtime a missing, empty or malformed value throws
+`RagScopeRequiredError` before any embedding is paid for, and `filter` can't override it. Values are
+compared as strings. To search several, use `anyOf(ids)`, where an empty list matches nothing; a plain
+array is refused so a request can't widen a search. A corpus shared by everyone says `scope = {}`.
+
+**Saving doesn't index.** Embedding is slow and paid for, so call `index` from a job or wherever the
+text changed. It embeds first, so a provider failure keeps the old chunks. On `PgVectorStore` the swap is
+one transaction; on stores without `replaceDocument`, it is two statements.
+
+**Deleting.** With Drizzle, use `.returning()` and call `remove`. With MikroORM, add
+`provideRagDeleteSync(ProjectDocument, DocumentsRag)` from `@dudousxd/nestjs-agent-store-mikro-orm` to
+your providers; it removes after the commit and sees only `em.remove` (on MikroORM 6, not inside a
+nested transaction). For everything no hook sees
+(`ON DELETE CASCADE`, `nativeDelete`, raw SQL), run `reconcile`:
+
+```ts
+await documentsRag.reconcile(allLiveIds, { dryRun: true }); // { orphans, missing }
+await documentsRag.reconcile(liveIdsOfProject, { scope: { projectId } });
+```
+
+`liveIds` must be complete for the part compared: whatever it leaves out is removed.
+
+**Agents.** `documentsRag.tool({ name, scope: (ctx) => ({ projectId: ctx.actor.tenantRef as string }) })`
+gives a tool whose scope comes from the context, for `provideAgentTool`. `scoped(scope)` gives a
+`Retriever` for composing. `retrieval: { mode: 'inject' }` passes no actor, so it can't scope per
+request: use the tool for multi-tenant agents.
+
+On Redis, declare the keys up front: `filterableFields: ragFilterableFields('projectId')`.
+
+Without Nest, `RagIndex` from the main entry does the same with a plain definition.
+
 ## Tree navigation retrieval (PageIndex-style)
 
 Chunk search ranks passages; on a **long structured document** (a regulation, an RFP, a contract, a
@@ -592,6 +658,7 @@ await store.removeMany(['doc-a', 'doc-b']);              // N documents, one pas
 |---|---|
 | `{ collection: 'handbook' }` | deletes that scope, returns the chunk count |
 | `{ audience: [] }` (or any empty array) | **deletes nothing**, returns `0` — the empty array is this package's deny primitive, and a deny must never be read as "no filter" |
+| `{ tenantId: undefined }` | throws `UnsafeRemovalError` with reason `undefined-value` — a missing value is a bug, and on every read method it matches nothing |
 | `{}` | throws `UnsafeRemovalError` — an empty filter is far more often a filter built wrong than a deliberate request to wipe the store |
 | `{ notFilterable: 'x' }` (Redis) | throws `UnsafeRemovalError` — the key has no `meta_*` TAG to match on |
 

package/dist/index.cjs

@@ -19,8 +19,8 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
 // src/index.ts
-var index_exports = {};
-__export(index_exports, {
+var src_exports = {};
+__export(src_exports, {
   DEFAULT_STOP_WORDS: () => DEFAULT_STOP_WORDS,
   EmbeddingRetriever: () => EmbeddingRetriever,
   FilteredRetriever: () => FilteredRetriever,
@@ -35,8 +35,11 @@ __export(index_exports, {
   PgLexicalVectorStore: () => PgLexicalVectorStore,
   PgVectorStore: () => PgVectorStore,
   RAG_DIAGNOSTIC_LIB: () => RAG_DIAGNOSTIC_LIB,
+  RAG_ENTITY_KEY: () => RAG_ENTITY_KEY,
   RAG_RETRIEVAL_CHANNEL: () => RAG_RETRIEVAL_CHANNEL,
   RAG_RETRIEVAL_EVENT: () => RAG_RETRIEVAL_EVENT,
+  RagIndex: () => RagIndex,
+  RagScopeRequiredError: () => RagScopeRequiredError,
   RedisVectorSchemaMismatchError: () => RedisVectorSchemaMismatchError,
   RedisVectorStore: () => RedisVectorStore,
   RerankingRetriever: () => RerankingRetriever,
@@ -44,6 +47,7 @@ __export(index_exports, {
   TreeNavigationRetriever: () => TreeNavigationRetriever,
   TwoStageRetriever: () => TwoStageRetriever,
   UnsafeRemovalError: () => UnsafeRemovalError,
+  anyOf: () => anyOf,
   anyTermTsquery: () => anyTermTsquery,
   applyMetadataPatch: () => applyMetadataPatch,
   buildDocumentTree: () => buildDocumentTree,
@@ -59,6 +63,7 @@ __export(index_exports, {
   describesRetrieval: () => describesRetrieval,
   detectHeadings: () => detectHeadings,
   documentIdOf: () => documentIdOf,
+  embedCountingUsage: () => embedCountingUsage,
   emitRagRetrieval: () => emitRagRetrieval,
   estimateTokens: () => estimateTokens,
   hasSearchSyntax: () => hasSearchSyntax,
@@ -74,13 +79,15 @@ __export(index_exports, {
   openAiChatTreeLlm: () => openAiChatTreeLlm,
   openAiEmbeddings: () => openAiEmbeddings,
   parseJsonReply: () => parseJsonReply,
+  ragFilterableFields: () => ragFilterableFields,
+  replaceDocumentChunks: () => replaceDocumentChunks,
   spanText: () => spanText,
   stableHash: () => stableHash,
   toPgClient: () => toPgClient,
   treeLlmFromModelProvider: () => treeLlmFromModelProvider,
   walkTree: () => walkTree
 });
-module.exports = __toCommonJS(index_exports);
+module.exports = __toCommonJS(src_exports);
 
 // src/chunk.ts
 function chunkText(text, options = {}) {
@@ -158,6 +165,16 @@ function chunkByRecord(text, chunkSize, separator) {
 __name(chunkByRecord, "chunkByRecord");
 
 // src/vector-store.ts
+async function replaceDocumentChunks(store, documentId, records) {
+  const atomic = store;
+  if (typeof atomic.replaceDocument === "function") {
+    await atomic.replaceDocument(documentId, records);
+    return;
+  }
+  await store.remove(documentId);
+  await store.upsert(records);
+}
+__name(replaceDocumentChunks, "replaceDocumentChunks");
 function isLexicalVectorStore(store) {
   return typeof store.searchText === "function";
 }
@@ -218,12 +235,19 @@ function applyMetadataPatch(metadata, patch) {
 __name(applyMetadataPatch, "applyMetadataPatch");
 
 // src/filter.ts
+function isMissingValue(value) {
+  return value === void 0 || Array.isArray(value) && value.includes(void 0);
+}
+__name(isMissingValue, "isMissingValue");
 function matchesFilter(metadata, filter) {
   const entries = Object.entries(filter);
   if (metadata === void 0) {
     return entries.length === 0;
   }
   return entries.every(([key, expected]) => {
+    if (isMissingValue(expected)) {
+      return false;
+    }
     if (Array.isArray(expected)) {
       const actual = metadata[key];
       return Array.isArray(actual) ? actual.some((value) => expected.includes(value)) : expected.includes(actual);
@@ -236,9 +260,18 @@ function filterMatchesNothing(filter) {
   if (filter === void 0) {
     return false;
   }
-  return Object.values(filter).some((value) => Array.isArray(value) && value.length === 0);
+  return Object.values(filter).some((value) => isMissingValue(value) || Array.isArray(value) && value.length === 0);
 }
 __name(filterMatchesNothing, "filterMatchesNothing");
+function undefinedFilterKeys(filter) {
+  return Object.entries(filter).filter(([, value]) => isMissingValue(value)).map(([key]) => key);
+}
+__name(undefinedFilterKeys, "undefinedFilterKeys");
+function undefinedFilterMessage(keys) {
+  const named = keys.map((key) => `"${key}"`).join(", ");
+  return `removeWhere() refuses a filter with no value for ${named}: the scope was probably never set.`;
+}
+__name(undefinedFilterMessage, "undefinedFilterMessage");
 
 // src/memory-vector-store.ts
 var MemoryVectorStore = class {
@@ -375,6 +408,10 @@ var MemoryVectorStore = class {
     if (Object.keys(filter).length === 0) {
       throw new UnsafeRemovalError("empty-filter", "removeWhere() refuses an empty filter: it would delete every chunk in the store. Pass a filter that scopes the removal, or delete deliberately with removeMany(await store.listDocumentIds()).");
     }
+    const missing = undefinedFilterKeys(filter);
+    if (missing.length > 0) {
+      throw new UnsafeRemovalError("undefined-value", undefinedFilterMessage(missing));
+    }
     if (filterMatchesNothing(filter)) {
       return 0;
     }
@@ -644,6 +681,7 @@ var PgVectorStore = class {
   dimensions;
   /** The widths with a partial HNSW index, for a mixed-dimension table. */
   indexedDimensions;
+  vectorType;
   nullableEmbeddings;
   iterativeScan;
   efSearch;
@@ -667,6 +705,10 @@ var PgVectorStore = class {
         ...new Set(dimensions)
       ];
     }
+    this.vectorType = options.vectorType ?? "vector";
+    if (this.vectorType === "halfvec" && this.dimensions === void 0) {
+      throw new Error('PgVectorStore: vectorType "halfvec" needs a single width, not a list');
+    }
     this.nullableEmbeddings = options.nullableEmbeddings ?? false;
     this.iterativeScan = options.iterativeScan;
     this.efSearch = options.efSearch;
@@ -685,7 +727,7 @@ var PgVectorStore = class {
   */
   schemaStatements() {
     const mixed = this.dimensions === void 0;
-    const column = mixed ? "vector" : `vector(${this.dimensions})`;
+    const column = mixed ? "vector" : `${this.vectorType}(${this.dimensions})`;
     const statements = [
       "CREATE EXTENSION IF NOT EXISTS vector",
       `CREATE TABLE IF NOT EXISTS ${this.table} (
@@ -704,7 +746,7 @@ var PgVectorStore = class {
       }
     } else {
       statements.push(`CREATE INDEX IF NOT EXISTS ${this.table}_embedding_idx
-        ON ${this.table} USING hnsw (embedding vector_cosine_ops)`);
+        ON ${this.table} USING hnsw (embedding ${this.vectorType}_cosine_ops)`);
     }
     return statements;
   }
@@ -720,6 +762,23 @@ var PgVectorStore = class {
   * {@link PgVectorStoreOptions.nullableEmbeddings}).
   */
   async upsert(records) {
+    await this.upsertWith(this.client, records);
+  }
+  /**
+  * Swaps a document's chunks in one transaction, so a crash can't lose it and a search sees the old
+  * version or the new one. A client without `transaction` falls back to two statements.
+  */
+  async replaceDocument(documentId, records) {
+    const swap = /* @__PURE__ */ __name(async (client) => {
+      await client.query(`DELETE FROM ${this.table} WHERE ${DOCUMENT_ID_FROM_CHUNK} = $1`, [
+        stripNulBytes(documentId)
+      ]);
+      await this.upsertWith(client, records);
+    }, "swap");
+    const transaction = this.client.transaction?.bind(this.client);
+    await (transaction === void 0 ? swap(this.client) : transaction(swap));
+  }
+  async upsertWith(client, records) {
     const byId = /* @__PURE__ */ new Map();
     for (const record of records) {
       const id = stripNulBytes(record.id);
@@ -741,9 +800,9 @@ var PgVectorStore = class {
         const metadata = record.metadata !== void 0 ? stripNulBytes(record.metadata) : void 0;
         const base = params.length;
         params.push(record.id, text, source ?? null, metadata !== void 0 ? JSON.stringify(metadata) : null, record.embedding.length > 0 ? toVectorLiteral(record.embedding) : null);
-        return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}::jsonb, $${base + 5}::vector)`;
+        return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}::jsonb, $${base + 5}::${this.vectorType})`;
       });
-      await this.client.query(`INSERT INTO ${this.table} (id, text, source, metadata, embedding)
+      await client.query(`INSERT INTO ${this.table} (id, text, source, metadata, embedding)
          VALUES ${rows.join(", ")}
          ON CONFLICT (id) DO UPDATE SET
            text = EXCLUDED.text,
@@ -867,6 +926,10 @@ var PgVectorStore = class {
     if (Object.keys(filter).length === 0) {
       throw new UnsafeRemovalError("empty-filter", "removeWhere() refuses an empty filter: it would delete every chunk in the table. Pass a filter that scopes the removal, or delete deliberately with removeMany(await store.listDocumentIds()).");
     }
+    const missing = undefinedFilterKeys(filter);
+    if (missing.length > 0) {
+      throw new UnsafeRemovalError("undefined-value", undefinedFilterMessage(missing));
+    }
     if (filterMatchesNothing(filter)) {
       return 0;
     }
@@ -898,7 +961,7 @@ var PgVectorStore = class {
       options.topK
     ];
     const conditions = this.whereConditions(options.filter, params);
-    let vectorType = "vector";
+    let vectorType = this.vectorType;
     if (this.dimensions === void 0) {
       const width = embedding.length;
       if (this.indexedDimensions.includes(width)) {
@@ -908,7 +971,7 @@ var PgVectorStore = class {
     } else if (this.nullableEmbeddings) {
       conditions.push("embedding IS NOT NULL");
     }
-    const column = vectorType === "vector" ? "embedding" : `(embedding::${vectorType})`;
+    const column = vectorType === this.vectorType ? "embedding" : `(embedding::${vectorType})`;
     const sql = `SELECT id, text, source, metadata, 1 - (${column} <=> $1::${vectorType}) AS score
        FROM ${this.table}
        ${conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""}
@@ -1095,6 +1158,10 @@ function buildWhere(filter, params) {
   const clauses = [];
   const scalar = {};
   for (const [key, value] of Object.entries(stripped)) {
+    if (value === void 0 || Array.isArray(value) && value.includes(void 0)) {
+      clauses.push("false");
+      continue;
+    }
     if (Array.isArray(value)) {
       if (value.length === 0) {
         clauses.push("false");
@@ -1636,6 +1703,10 @@ var RedisVectorStore = class {
     if (keys.length === 0) {
       throw new UnsafeRemovalError("empty-filter", "removeWhere() refuses an empty filter: it would delete every chunk in the index. Pass a filter that scopes the removal, or delete deliberately with removeMany(await store.listDocumentIds()).");
     }
+    const missing = undefinedFilterKeys(filter);
+    if (missing.length > 0) {
+      throw new UnsafeRemovalError("undefined-value", undefinedFilterMessage(missing));
+    }
     const unindexed = keys.filter((key) => !this.filterableFields.includes(key));
     if (unindexed.length > 0) {
       const named = unindexed.map((key) => `"${key}"`).join(", ");
@@ -2186,7 +2257,7 @@ var KeywordRetriever = class {
   }
 };
 function tokenize(text) {
-  return text.toLowerCase().match(/[a-z0-9]+/g) ?? [];
+  return text.normalize("NFKD").replace(/\p{M}/gu, "").toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [];
 }
 __name(tokenize, "tokenize");
 
@@ -2507,13 +2578,19 @@ function createRetrievalTool(retriever, options = {}) {
       inputSchema: querySchema
     },
     handler: {
-      async execute(input) {
+      async execute(input, ctx) {
         if (!isQueryInput(input)) {
           throw new Error("retrieval tool expects { query: string }");
         }
-        const passages = await instrumented.retrieve(input.query, topK !== void 0 ? {
-          topK
-        } : {});
+        const filter = options.filter?.(ctx);
+        const passages = await instrumented.retrieve(input.query, {
+          ...topK !== void 0 ? {
+            topK
+          } : {},
+          ...filter !== void 0 ? {
+            filter
+          } : {}
+        });
         return {
           passages
         };
@@ -2553,7 +2630,7 @@ function openAiEmbeddings(options) {
   const maxInputChars = options.maxInputChars ?? 24e3;
   const timeoutMs = options.timeoutMs ?? 12e4;
   const doFetch = options.fetch ?? fetch;
-  const request = /* @__PURE__ */ __name(async (input) => {
+  const request = /* @__PURE__ */ __name(async (input, count) => {
     const response = await doFetch(url, {
       method: "POST",
       headers: {
@@ -2586,19 +2663,20 @@ function openAiEmbeddings(options) {
     const tokens = json2?.usage?.total_tokens ?? json2?.usage?.prompt_tokens;
     if (typeof tokens === "number" && Number.isFinite(tokens)) {
       options.onUsage?.(tokens);
+      count(tokens);
     }
     return items.map((item) => item.embedding);
   }, "request");
-  const requestSplitting = /* @__PURE__ */ __name(async (input) => {
+  const requestSplitting = /* @__PURE__ */ __name(async (input, count) => {
     if (input.length > batchSize) {
       const vectors = [];
       for (let start = 0; start < input.length; start += batchSize) {
-        vectors.push(...await requestSplitting(input.slice(start, start + batchSize)));
+        vectors.push(...await requestSplitting(input.slice(start, start + batchSize), count));
       }
       return vectors;
     }
     try {
-      return await request(input);
+      return await request(input, count);
     } catch (error) {
       if (!isBatchTooLarge(error) || input.length <= 1) {
         throw error;
@@ -2610,20 +2688,37 @@ function openAiEmbeddings(options) {
         options.onWarn?.(`embeddings server ${url} refused a batch of ${input.length} inputs (${error.message}); retrying in batches of ${half}. Raise the server's batch limit (TEI: --max-client-batch-size) or lower batchSize.`);
       }
       return [
-        ...await requestSplitting(input.slice(0, half)),
-        ...await requestSplitting(input.slice(half))
+        ...await requestSplitting(input.slice(0, half), count),
+        ...await requestSplitting(input.slice(half), count)
       ];
     }
   }, "requestSplitting");
+  const embedAll = /* @__PURE__ */ __name(async (texts, count) => {
+    const vectors = [];
+    for (let start = 0; start < texts.length; ) {
+      const input = texts.slice(start, start + batchSize).map((text) => (text.length > maxInputChars ? text.slice(0, maxInputChars) : text) || " ");
+      vectors.push(...await requestSplitting(input, count));
+      start += input.length;
+    }
+    return vectors;
+  }, "embedAll");
   return {
-    async embed(texts) {
-      const vectors = [];
-      for (let start = 0; start < texts.length; ) {
-        const input = texts.slice(start, start + batchSize).map((text) => (text.length > maxInputChars ? text.slice(0, maxInputChars) : text) || " ");
-        vectors.push(...await requestSplitting(input));
-        start += input.length;
-      }
-      return vectors;
+    embed: /* @__PURE__ */ __name((texts) => embedAll(texts, () => {
+    }), "embed"),
+    async embedWithUsage(texts) {
+      let inputTokens;
+      const vectors = await embedAll(texts, (tokens) => {
+        inputTokens = (inputTokens ?? 0) + tokens;
+      });
+      return {
+        vectors,
+        ...inputTokens !== void 0 ? {
+          usage: {
+            inputTokens,
+            modelId: options.model
+          }
+        } : {}
+      };
     }
   };
 }
@@ -2728,6 +2823,206 @@ function itemsOf(body) {
 }
 __name(itemsOf, "itemsOf");
 
+// src/embedding-usage.ts
+async function embedCountingUsage(provider, texts) {
+  const reporting = provider;
+  return typeof reporting.embedWithUsage === "function" ? reporting.embedWithUsage(texts) : {
+    vectors: await provider.embed(texts)
+  };
+}
+__name(embedCountingUsage, "embedCountingUsage");
+
+// src/rag-index.ts
+var RAG_ENTITY_KEY = "ragEntity";
+var RagScopeRequiredError = class extends Error {
+  static {
+    __name(this, "RagScopeRequiredError");
+  }
+  model;
+  key;
+  constructor(model, key) {
+    super(`${model} needs scope.${key}: its chunks are isolated by it`), this.model = model, this.key = key;
+    this.name = "RagScopeRequiredError";
+  }
+};
+var ANY_OF = /* @__PURE__ */ Symbol.for("@dudousxd/nestjs-agent-rag/anyOf");
+function anyOf(values) {
+  return {
+    [ANY_OF]: values
+  };
+}
+__name(anyOf, "anyOf");
+function isScopeValue(value) {
+  return typeof value === "number" ? Number.isFinite(value) : typeof value === "string" && value !== "";
+}
+__name(isScopeValue, "isScopeValue");
+function isAnyOf(value) {
+  return typeof value === "object" && value !== null && ANY_OF in value;
+}
+__name(isAnyOf, "isAnyOf");
+var RagIndex = class {
+  static {
+    __name(this, "RagIndex");
+  }
+  definition;
+  deps;
+  embedder;
+  constructor(definition, deps) {
+    this.definition = definition;
+    this.deps = deps;
+    const { onUsage } = deps;
+    this.embedder = onUsage === void 0 ? deps.embedder : {
+      embed: /* @__PURE__ */ __name(async (texts) => {
+        const { vectors, usage } = await embedCountingUsage(deps.embedder, texts);
+        if (usage !== void 0) {
+          await onUsage(usage);
+        }
+        return vectors;
+      }, "embed")
+    };
+  }
+  /** The id this row's chunks are stored under: `<name>:<id>`. */
+  documentId(id) {
+    if (!isScopeValue(id)) {
+      throw new Error(`${this.definition.name}: a row needs an id to be indexed or removed`);
+    }
+    return `${this.definition.name}:${String(id)}`;
+  }
+  /** A `Retriever` locked to this scope, for `HybridRetriever`, `createRetrievalTool` and so on. */
+  scoped(scope) {
+    return new FilteredRetriever(this.baseRetriever(), this.scopeFilter(scope));
+  }
+  async search(query, options) {
+    const { scope, filter, topK } = options;
+    return this.scoped(scope).retrieve(query, {
+      ...topK !== void 0 ? {
+        topK
+      } : {},
+      ...filter !== void 0 ? {
+        filter
+      } : {}
+    });
+  }
+  /**
+  * Replaces the row's chunks and returns how many were written. Embeddings come first: if the
+  * provider fails, the previous chunks are still there. The swap is one transaction on Postgres; on a
+  * store without `replaceDocument` a crash mid-swap loses the document until the next `index`.
+  */
+  async index(row) {
+    const { definition, deps } = this;
+    const documentId = this.documentId(definition.id(row));
+    const scope = this.scopeFilter(Object.fromEntries(Object.entries(definition.scope).map(([key, read]) => [
+      key,
+      read(row)
+    ])));
+    const text = definition.text(row)?.trim();
+    if (!text) {
+      await deps.store.remove(documentId);
+      return 0;
+    }
+    const pieces = chunkText(text, definition.chunk).slice(0, definition.maxChunks);
+    const embeddings = await this.embedder.embed(pieces);
+    const source = definition.source?.(row) ?? void 0;
+    const metadata = {
+      ...definition.metadata?.(row),
+      ...scope
+    };
+    await replaceDocumentChunks(deps.store, documentId, pieces.map((piece, index) => ({
+      id: `${documentId}#${index}`,
+      text: piece,
+      embedding: embeddings[index] ?? [],
+      ...source !== void 0 ? {
+        source
+      } : {},
+      metadata
+    })));
+    return pieces.length;
+  }
+  /** Takes the id itself once the row is gone. */
+  remove(rowOrId) {
+    const id = typeof rowOrId === "string" || typeof rowOrId === "number" ? rowOrId : this.definition.id(rowOrId);
+    return this.deps.store.remove(this.documentId(id));
+  }
+  /**
+  * Diffs the index against the ids that exist, and removes what no hook could: `ON DELETE CASCADE`,
+  * raw SQL, a crash after a commit. `liveIds` must be complete for the part being compared, because
+  * anything left out is removed. Use `dryRun` to look first.
+  */
+  async reconcile(liveIds, options = {}) {
+    const filter = {
+      [RAG_ENTITY_KEY]: this.definition.name
+    };
+    for (const [key, value] of Object.entries(options.scope ?? {})) {
+      if (!isScopeValue(value)) {
+        throw new RagScopeRequiredError(this.definition.name, key);
+      }
+      filter[key] = String(value);
+    }
+    const live = /* @__PURE__ */ new Set();
+    for await (const id of liveIds) {
+      live.add(this.documentId(id));
+    }
+    const indexed = await this.deps.store.listDocumentIds(filter);
+    const orphans = indexed.filter((id) => !live.has(id));
+    if (options.dryRun !== true) {
+      await this.deps.store.removeMany(orphans);
+    }
+    const known = new Set(indexed);
+    return {
+      orphans,
+      missing: [
+        ...live
+      ].filter((id) => !known.has(id))
+    };
+  }
+  /** A `search_*` tool whose scope comes from the context. Pass it to `provideAgentTool`. */
+  tool(options) {
+    return createRetrievalTool(this.baseRetriever(), {
+      name: options.name,
+      ...options.description !== void 0 ? {
+        description: options.description
+      } : {},
+      ...options.topK !== void 0 ? {
+        topK: options.topK
+      } : {},
+      filter: /* @__PURE__ */ __name((ctx) => this.scopeFilter(options.scope(ctx)), "filter")
+    });
+  }
+  baseRetriever() {
+    const { minScore } = this.definition;
+    return this.deps.retriever ?? new EmbeddingRetriever(this.embedder, this.deps.store, minScore !== void 0 ? {
+      minScore
+    } : {});
+  }
+  /** The filter a scope compiles to. Throws on anything but a usable value, before any paid call. */
+  scopeFilter(scope) {
+    const given = scope ?? {};
+    const filter = {
+      [RAG_ENTITY_KEY]: this.definition.name
+    };
+    for (const key of Object.keys(this.definition.scope)) {
+      const value = given[key];
+      if (isAnyOf(value) && value[ANY_OF].every(isScopeValue)) {
+        filter[key] = value[ANY_OF].map(String);
+      } else if (isScopeValue(value)) {
+        filter[key] = String(value);
+      } else {
+        throw new RagScopeRequiredError(this.definition.name, key);
+      }
+    }
+    return filter;
+  }
+};
+function ragFilterableFields(...scopeKeys) {
+  return [
+    .../* @__PURE__ */ new Set([
+      RAG_ENTITY_KEY,
+      ...scopeKeys
+    ])
+  ];
+}
+__name(ragFilterableFields, "ragFilterableFields");
+
 // src/tree/types.ts
 function* walkTree(nodes, path = []) {
   for (const node of nodes) {
@@ -5007,8 +5302,11 @@ __name(createNavigateDocumentTool, "createNavigateDocumentTool");
   PgLexicalVectorStore,
   PgVectorStore,
   RAG_DIAGNOSTIC_LIB,
+  RAG_ENTITY_KEY,
   RAG_RETRIEVAL_CHANNEL,
   RAG_RETRIEVAL_EVENT,
+  RagIndex,
+  RagScopeRequiredError,
   RedisVectorSchemaMismatchError,
   RedisVectorStore,
   RerankingRetriever,
@@ -5016,6 +5314,7 @@ __name(createNavigateDocumentTool, "createNavigateDocumentTool");
   TreeNavigationRetriever,
   TwoStageRetriever,
   UnsafeRemovalError,
+  anyOf,
   anyTermTsquery,
   applyMetadataPatch,
   buildDocumentTree,
@@ -5031,6 +5330,7 @@ __name(createNavigateDocumentTool, "createNavigateDocumentTool");
   describesRetrieval,
   detectHeadings,
   documentIdOf,
+  embedCountingUsage,
   emitRagRetrieval,
   estimateTokens,
   hasSearchSyntax,
@@ -5046,6 +5346,8 @@ __name(createNavigateDocumentTool, "createNavigateDocumentTool");
   openAiChatTreeLlm,
   openAiEmbeddings,
   parseJsonReply,
+  ragFilterableFields,
+  replaceDocumentChunks,
   spanText,
   stableHash,
   toPgClient,