@warlock.js/ai 4.4.0 → 4.6.0

package/esm/rag/loaders/loader.type.d.mts

@@ -0,0 +1,89 @@
+import { OutboundPolicy } from "../../security/outbound-policy.type.mjs";
+import { RagDocument } from "../contracts/rag-document.type.mjs";
+
+//#region ../@warlock.js/ai/src/rag/loaders/loader.type.d.ts
+/**
+ * The shape every loader emits — the **exact** {@link RagDocument} (or array
+ * of them) that `ai.rag(config).index()` consumes, so a load result is fed
+ * straight in with no adapter:
+ *
+ * @example
+ * const kb = ai.rag({ embedder, store });
+ * await kb.index(await ai.rag.loadWeb("https://example.com/guide"));
+ *
+ * A loader may emit one document (the common case for a single file / page)
+ * or several (e.g. one document per PDF page), so the result is uniformly an
+ * **array** — `index()` accepts an array, so callers never branch on arity.
+ */
+type RagLoaderResult = RagDocument[];
+/**
+ * Metadata keys loaders attach to every {@link RagDocument} they emit, on
+ * top of any caller-supplied `metadata`. Each is optional and only present
+ * when the loader could determine it. These keys round-trip through
+ * chunking onto the final citation, so a retrieved chunk can be traced back
+ * to its `source` URL / `title` / `page`.
+ */
+type RagLoaderMetadata = {
+  /**
+   * Where the document came from — a URL (web loader), a logical name, or
+   * the caller-supplied `id`. Always a string when present.
+   */
+  source?: string; /** Human-readable title, e.g. an HTML `<title>` or PDF document title. */
+  title?: string; /** Discriminator for the loader that produced the document. */
+  loader?: RagLoaderType; /** 1-based page number — set by {@link loadPdf} on per-page documents. */
+  page?: number; /** Total page count of the source — set by {@link loadPdf}. */
+  pageCount?: number; /** MIME content-type reported by the server — set by the web loader. */
+  contentType?: string;
+};
+/** Discriminator identifying which loader produced a {@link RagDocument}. */
+type RagLoaderType = "text" | "html" | "web" | "pdf";
+/**
+ * Shared options every loader accepts. The `id` and `metadata` flow onto
+ * the emitted {@link RagDocument} verbatim (loader-derived metadata is
+ * merged UNDER the caller's, so an explicit `metadata.title` always wins),
+ * and `tags` propagate to every chunk for `retrieve({ tags })` filtering.
+ */
+type RagLoaderOptions = {
+  /**
+   * Stable source id for the emitted document(s). Falls back to a
+   * loader-specific default (the URL for the web loader, `"document"`
+   * otherwise). Multi-document loaders suffix this (e.g. `"<id>#p3"`).
+   */
+  id?: string;
+  /**
+   * Extra metadata merged onto every emitted document. Caller keys take
+   * precedence over the loader's derived keys (`source`, `title`, …).
+   */
+  metadata?: Record<string, unknown>; /** Tags applied to every chunk written from the emitted document(s). */
+  tags?: string[];
+};
+/** Options for the HTML loader — {@link RagLoaderOptions} only. */
+type LoadHtmlOptions = RagLoaderOptions;
+/** Options for the plain-text loader — {@link RagLoaderOptions} only. */
+type LoadTextOptions = RagLoaderOptions;
+/** Options for the PDF loader. */
+type LoadPdfOptions = RagLoaderOptions & {
+  /**
+   * Emit one {@link RagDocument} per page (each tagged with `page` /
+   * `pageCount` metadata) instead of a single concatenated document.
+   * Per-page documents keep citations page-precise. Default `false`.
+   */
+  perPage?: boolean;
+};
+/**
+ * Options for the web loader. Extends {@link RagLoaderOptions} with an
+ * {@link OutboundPolicy} — the SSRF-safe fetch policy the loader hands to
+ * `guardedFetch`. Omit it and the strict defaults apply (https-only,
+ * private-IP deny on, 10s timeout, 5 MiB cap).
+ */
+type LoadWebOptions = RagLoaderOptions & {
+  /**
+   * The {@link OutboundPolicy} governing the fetch. The web loader NEVER
+   * issues a raw `fetch` — every request goes through the policy's
+   * `guardedFetch`, so an SSRF / oversized-body guard always applies.
+   */
+  policy?: OutboundPolicy;
+};
+//#endregion
+export { LoadHtmlOptions, LoadPdfOptions, LoadTextOptions, LoadWebOptions, RagLoaderMetadata, RagLoaderOptions, RagLoaderResult, RagLoaderType };
+//# sourceMappingURL=loader.type.d.mts.map

package/esm/rag/loaders/loader.type.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.type.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/loaders/loader.type.ts"],"mappings":";;;;;;AAgBA;;;;AAAyC;AASzC;;;;;;KATY,eAAA,GAAkB,WAAW;;;;;;AAwB5B;AAIb;KAnBY,iBAAA;;;AAmBa;AAQzB;EAtBE,MAAA;EAEA,KAAA,WA0BA;EAxBA,MAAA,GAAS,aAAa,EA6BX;EA3BX,IAAA,WA6BI;EA3BJ,SAAA,WA+BU;EA7BV,WAAA;AAAA;;KAIU,aAAA;AA4BZ;;;;AAA8C;AAG9C;AAHA,KApBY,gBAAA;;;AA6BH;AAST;;EAhCE,EAAA;EAsCuB;;;;EAjCvB,QAAA,GAAW,MAAM,mBAiCM;EA/BvB,IAAA;AAAA;;KAIU,eAAA,GAAkB,gBAAgB;;KAGlC,eAAA,GAAkB,gBAAgB;;KAGlC,cAAA,GAAiB,gBAAgB;;;;;;EAM3C,OAAO;AAAA;;;;;;;KASG,cAAA,GAAiB,gBAAA;;;;;;EAM3B,MAAA,GAAS,cAAc;AAAA"}

package/esm/rag/rag.d.mts

@@ -0,0 +1,38 @@
+import { Rag, RagConfig } from "./contracts/rag-config.type.mjs";
+
+//#region ../@warlock.js/ai/src/rag/rag.d.ts
+/**
+ * Create a RAG pipeline: **chunk → embed → vector store → retrieve →
+ * rerank → cite**, reusing the app's `ai.embedder` for embedding, a
+ * `@warlock.js/cache` `CacheDriver` as the vector store, and the
+ * composite-as-tool engine to expose retrieval as a tool.
+ *
+ * Resolution is loud at construction (mirroring `memory()`):
+ * - `embedder` is required — a provider with no embedder must be caught
+ *   here, not at first index.
+ * - `store` falls back to `ai.config({ defaultStore })`; if neither
+ *   resolves, construction throws.
+ *
+ * `retrieve()` is return-only — it never auto-injects into a prompt; the
+ * caller formats the cited chunks (or uses `asTool()` for the agent loop).
+ * The reranker is OFF by default (cosine-only) unless `config.reranker`
+ * is set.
+ *
+ * @example
+ * import { ai } from "@warlock.js/ai";
+ * import { MemoryCacheDriver } from "@warlock.js/cache";
+ *
+ * const kb = ai.rag({
+ *   name: "docs",
+ *   embedder: openai.embedder({ name: "text-embedding-3-small" }),
+ *   store: new MemoryCacheDriver(),
+ *   chunk: { type: "markdown", size: 800, overlap: 120 },
+ * });
+ *
+ * await kb.index([{ id: "guide", text: longMarkdown, metadata: { url: "/guide" } }]);
+ * const { chunks } = await kb.retrieve("how do I configure caching?", { topK: 4 });
+ */
+declare function rag(config: RagConfig): Rag;
+//#endregion
+export { rag };
+//# sourceMappingURL=rag.d.mts.map

package/esm/rag/rag.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rag.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/rag/rag.ts"],"mappings":";;;;;AAyDA;;;;;;;;AAA2C;;;;;;;;;;;;;;;;;;;;;iBAA3B,GAAA,CAAI,MAAA,EAAQ,SAAA,GAAY,GAAG"}

package/esm/rag/rag.mjs

@@ -0,0 +1,126 @@
+import { resolveDefaultStore } from "../config.mjs";
+import { chunk } from "./chunk/chunk.mjs";
+import { ragAsTool } from "./as-tool.mjs";
+import { retrieve } from "./retrieve.mjs";
+import { cacheVectorStore } from "./store/cache-vector-store.mjs";
+
+//#region ../@warlock.js/ai/src/rag/rag.ts
+const DEFAULT_NAME = "rag";
+const DEFAULT_NAMESPACE_PREFIX = "ai.rag";
+/**
+* Max chunk texts embedded per `embedder.embedMany()` call. One call is
+* one provider request, so a giant document is sub-batched to stay under
+* the provider's per-request token cap (the design's "chunk larger than
+* provider per-request cap" guard).
+*/
+const DEFAULT_MAX_BATCH = 96;
+/**
+* Create a RAG pipeline: **chunk → embed → vector store → retrieve →
+* rerank → cite**, reusing the app's `ai.embedder` for embedding, a
+* `@warlock.js/cache` `CacheDriver` as the vector store, and the
+* composite-as-tool engine to expose retrieval as a tool.
+*
+* Resolution is loud at construction (mirroring `memory()`):
+* - `embedder` is required — a provider with no embedder must be caught
+*   here, not at first index.
+* - `store` falls back to `ai.config({ defaultStore })`; if neither
+*   resolves, construction throws.
+*
+* `retrieve()` is return-only — it never auto-injects into a prompt; the
+* caller formats the cited chunks (or uses `asTool()` for the agent loop).
+* The reranker is OFF by default (cosine-only) unless `config.reranker`
+* is set.
+*
+* @example
+* import { ai } from "@warlock.js/ai";
+* import { MemoryCacheDriver } from "@warlock.js/cache";
+*
+* const kb = ai.rag({
+*   name: "docs",
+*   embedder: openai.embedder({ name: "text-embedding-3-small" }),
+*   store: new MemoryCacheDriver(),
+*   chunk: { type: "markdown", size: 800, overlap: 120 },
+* });
+*
+* await kb.index([{ id: "guide", text: longMarkdown, metadata: { url: "/guide" } }]);
+* const { chunks } = await kb.retrieve("how do I configure caching?", { topK: 4 });
+*/
+function rag(config) {
+	const name = config.name ?? DEFAULT_NAME;
+	if (!config.embedder) throw new Error(`rag("${name}"): an \`embedder\` is required — pass one from a provider that supports embeddings (e.g. openai.embedder({ name: "text-embedding-3-small" }))`);
+	const driver = config.store ?? resolveDefaultStore();
+	if (!driver) throw new Error(`rag("${name}"): no store — pass \`store\` (a vector-capable @warlock.js/cache CacheDriver) or call \`ai.config({ defaultStore })\` at app boot before constructing the rag`);
+	const store = cacheVectorStore(driver);
+	const namespace = config.namespace ?? `${DEFAULT_NAMESPACE_PREFIX}.${name}`;
+	const embedder = config.embedder;
+	let indexedDimensions;
+	const instance = {
+		name,
+		async index(docs, chunkOverride) {
+			const chunkOptions = chunkOverride ?? config.chunk;
+			const limits = config.limits;
+			if (limits?.maxDocuments !== void 0 && docs.length > limits.maxDocuments) throw new Error(`rag("${name}"): index() received ${docs.length} documents, exceeding the configured maxDocuments of ${limits.maxDocuments}`);
+			if (limits?.maxBytes !== void 0) {
+				const totalBytes = docs.reduce((sum, doc) => sum + Buffer.byteLength(doc.text ?? ""), 0);
+				if (totalBytes > limits.maxBytes) throw new Error(`rag("${name}"): index() received ${totalBytes} bytes of document text, exceeding the configured maxBytes of ${limits.maxBytes}`);
+			}
+			const records = [];
+			for (const doc of docs) {
+				const pieces = chunk(doc.text, chunkOptions);
+				for (const piece of pieces) {
+					const value = {
+						sourceId: doc.id,
+						chunkIndex: piece.index,
+						span: piece.span,
+						text: piece.text,
+						metadata: doc.metadata
+					};
+					records.push({
+						key: keyFor(namespace, doc.id, piece.index),
+						value,
+						text: piece.text,
+						tags: doc.tags
+					});
+				}
+			}
+			if (records.length === 0) return { chunks: 0 };
+			if (limits?.maxChunks !== void 0 && records.length > limits.maxChunks) throw new Error(`rag("${name}"): index() produced ${records.length} chunks, exceeding the configured maxChunks of ${limits.maxChunks}`);
+			for (let offset = 0; offset < records.length; offset += DEFAULT_MAX_BATCH) {
+				const batch = records.slice(offset, offset + DEFAULT_MAX_BATCH);
+				const { vectors, dimensions } = await embedder.embedMany(batch.map((record) => record.text));
+				if (indexedDimensions === void 0 && dimensions !== 0) indexedDimensions = dimensions;
+				await Promise.all(batch.map((record, position) => store.upsert(record.key, record.value, vectors[position], record.tags)));
+			}
+			return { chunks: records.length };
+		},
+		async retrieve(query, options) {
+			return retrieve(query, {
+				embedder,
+				store,
+				namespace,
+				reranker: config.reranker,
+				defaults: config.retrieve,
+				indexedDimensions
+			}, options);
+		},
+		async clear() {
+			await store.removeNamespace(namespace);
+		},
+		asTool(options) {
+			return ragAsTool(name, (query, retrieveOptions) => instance.retrieve(query, retrieveOptions), options);
+		}
+	};
+	return instance;
+}
+/**
+* Namespaced key for a stored chunk. Uses the `.` separator (matching
+* `SemanticMemory.keyFor`) so namespace-prefix filtering on the returned
+* `hit.key` stays aligned with the cache's `parseKey` normalization.
+*/
+function keyFor(namespace, sourceId, chunkIndex) {
+	return `${namespace}.${sourceId}.${chunkIndex}`;
+}
+
+//#endregion
+export { rag };
+//# sourceMappingURL=rag.mjs.map

package/esm/rag/rag.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"rag.mjs","names":["chunkText","runRetrieve"],"sources":["../../../../../../../@warlock.js/ai/src/rag/rag.ts"],"sourcesContent":["import { resolveDefaultStore } from \"../config\";\nimport { chunk as chunkText } from \"./chunk/chunk\";\nimport type { ChunkOptions } from \"./contracts/chunk-options.type\";\nimport type { RetrieveOptions, RetrieveResult } from \"./contracts/citation.type\";\nimport type {\n  Rag,\n  RagAsToolOptions,\n  RagConfig,\n} from \"./contracts/rag-config.type\";\nimport type { RagDocument } from \"./contracts/rag-document.type\";\nimport { ragAsTool } from \"./as-tool\";\nimport { retrieve as runRetrieve, type StoredChunk } from \"./retrieve\";\nimport { cacheVectorStore } from \"./store/cache-vector-store\";\nimport type { VectorStore } from \"./store/vector-store.contract\";\n\nconst DEFAULT_NAME = \"rag\";\nconst DEFAULT_NAMESPACE_PREFIX = \"ai.rag\";\n\n/**\n * Max chunk texts embedded per `embedder.embedMany()` call. One call is\n * one provider request, so a giant document is sub-batched to stay under\n * the provider's per-request token cap (the design's \"chunk larger than\n * provider per-request cap\" guard).\n */\nconst DEFAULT_MAX_BATCH = 96;\n\n/**\n * Create a RAG pipeline: **chunk → embed → vector store → retrieve →\n * rerank → cite**, reusing the app's `ai.embedder` for embedding, a\n * `@warlock.js/cache` `CacheDriver` as the vector store, and the\n * composite-as-tool engine to expose retrieval as a tool.\n *\n * Resolution is loud at construction (mirroring `memory()`):\n * - `embedder` is required — a provider with no embedder must be caught\n *   here, not at first index.\n * - `store` falls back to `ai.config({ defaultStore })`; if neither\n *   resolves, construction throws.\n *\n * `retrieve()` is return-only — it never auto-injects into a prompt; the\n * caller formats the cited chunks (or uses `asTool()` for the agent loop).\n * The reranker is OFF by default (cosine-only) unless `config.reranker`\n * is set.\n *\n * @example\n * import { ai } from \"@warlock.js/ai\";\n * import { MemoryCacheDriver } from \"@warlock.js/cache\";\n *\n * const kb = ai.rag({\n *   name: \"docs\",\n *   embedder: openai.embedder({ name: \"text-embedding-3-small\" }),\n *   store: new MemoryCacheDriver(),\n *   chunk: { type: \"markdown\", size: 800, overlap: 120 },\n * });\n *\n * await kb.index([{ id: \"guide\", text: longMarkdown, metadata: { url: \"/guide\" } }]);\n * const { chunks } = await kb.retrieve(\"how do I configure caching?\", { topK: 4 });\n */\nexport function rag(config: RagConfig): Rag {\n  const name = config.name ?? DEFAULT_NAME;\n\n  if (!config.embedder) {\n    throw new Error(\n      `rag(\"${name}\"): an \\`embedder\\` is required — pass one from a provider that supports embeddings (e.g. openai.embedder({ name: \"text-embedding-3-small\" }))`,\n    );\n  }\n\n  const driver = config.store ?? resolveDefaultStore();\n\n  if (!driver) {\n    throw new Error(\n      `rag(\"${name}\"): no store — pass \\`store\\` (a vector-capable @warlock.js/cache CacheDriver) or call \\`ai.config({ defaultStore })\\` at app boot before constructing the rag`,\n    );\n  }\n\n  const store: VectorStore = cacheVectorStore(driver);\n  const namespace = config.namespace ?? `${DEFAULT_NAMESPACE_PREFIX}.${name}`;\n  const embedder = config.embedder;\n\n  // Captured at first index for the dimension-mismatch guard in retrieve().\n  let indexedDimensions: number | undefined;\n\n  const instance: Rag = {\n    name,\n\n    async index(\n      docs: RagDocument[],\n      chunkOverride?: ChunkOptions,\n    ): Promise<{ chunks: number }> {\n      const chunkOptions = chunkOverride ?? config.chunk;\n\n      // Ingestion guardrails (D5) — fail BEFORE any embedding spend.\n      const limits = config.limits;\n      if (limits?.maxDocuments !== undefined && docs.length > limits.maxDocuments) {\n        throw new Error(\n          `rag(\"${name}\"): index() received ${docs.length} documents, exceeding the configured maxDocuments of ${limits.maxDocuments}`,\n        );\n      }\n      if (limits?.maxBytes !== undefined) {\n        const totalBytes = docs.reduce(\n          (sum, doc) => sum + Buffer.byteLength(doc.text ?? \"\"),\n          0,\n        );\n        if (totalBytes > limits.maxBytes) {\n          throw new Error(\n            `rag(\"${name}\"): index() received ${totalBytes} bytes of document text, exceeding the configured maxBytes of ${limits.maxBytes}`,\n          );\n        }\n      }\n\n      // Flatten every document into stored-chunk records + their texts,\n      // preserving document order so a single batched embed maps back 1:1.\n      const records: { key: string; value: StoredChunk; text: string; tags?: string[] }[] = [];\n\n      for (const doc of docs) {\n        const pieces = chunkText(doc.text, chunkOptions);\n\n        for (const piece of pieces) {\n          const value: StoredChunk = {\n            sourceId: doc.id,\n            chunkIndex: piece.index,\n            span: piece.span,\n            text: piece.text,\n            metadata: doc.metadata,\n          };\n\n          records.push({\n            key: keyFor(namespace, doc.id, piece.index),\n            value,\n            text: piece.text,\n            tags: doc.tags,\n          });\n        }\n      }\n\n      // Empty / whitespace-only documents yield zero chunks — write\n      // nothing and never embed an empty batch.\n      if (records.length === 0) {\n        return { chunks: 0 };\n      }\n\n      // Chunk cap (D5) — checked after chunking, still before embedding.\n      if (limits?.maxChunks !== undefined && records.length > limits.maxChunks) {\n        throw new Error(\n          `rag(\"${name}\"): index() produced ${records.length} chunks, exceeding the configured maxChunks of ${limits.maxChunks}`,\n        );\n      }\n\n      // Sub-batch the embed calls so one giant document does not blow the\n      // provider's per-request token cap.\n      for (let offset = 0; offset < records.length; offset += DEFAULT_MAX_BATCH) {\n        const batch = records.slice(offset, offset + DEFAULT_MAX_BATCH);\n        const { vectors, dimensions } = await embedder.embedMany(\n          batch.map((record) => record.text),\n        );\n\n        if (indexedDimensions === undefined && dimensions !== 0) {\n          indexedDimensions = dimensions;\n        }\n\n        await Promise.all(\n          batch.map((record, position) =>\n            store.upsert(record.key, record.value, vectors[position], record.tags),\n          ),\n        );\n      }\n\n      return { chunks: records.length };\n    },\n\n    async retrieve(query: string, options?: RetrieveOptions): Promise<RetrieveResult> {\n      return runRetrieve(\n        query,\n        {\n          embedder,\n          store,\n          namespace,\n          reranker: config.reranker,\n          defaults: config.retrieve,\n          indexedDimensions,\n        },\n        options,\n      );\n    },\n\n    async clear(): Promise<void> {\n      await store.removeNamespace(namespace);\n    },\n\n    asTool(options?: RagAsToolOptions) {\n      return ragAsTool(name, (query, retrieveOptions) => instance.retrieve(query, retrieveOptions), options);\n    },\n  };\n\n  return instance;\n}\n\n/**\n * Namespaced key for a stored chunk. Uses the `.` separator (matching\n * `SemanticMemory.keyFor`) so namespace-prefix filtering on the returned\n * `hit.key` stays aligned with the cache's `parseKey` normalization.\n */\nfunction keyFor(namespace: string, sourceId: string, chunkIndex: number): string {\n  return `${namespace}.${sourceId}.${chunkIndex}`;\n}\n"],"mappings":";;;;;;;AAeA,MAAM,eAAe;AACrB,MAAM,2BAA2B;;;;;;;AAQjC,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiC1B,SAAgB,IAAI,QAAwB;CAC1C,MAAM,OAAO,OAAO,QAAQ;CAE5B,IAAI,CAAC,OAAO,UACV,MAAM,IAAI,MACR,QAAQ,KAAK,+IACf;CAGF,MAAM,SAAS,OAAO,SAAS,oBAAoB;CAEnD,IAAI,CAAC,QACH,MAAM,IAAI,MACR,QAAQ,KAAK,+JACf;CAGF,MAAM,QAAqB,iBAAiB,MAAM;CAClD,MAAM,YAAY,OAAO,aAAa,GAAG,yBAAyB,GAAG;CACrE,MAAM,WAAW,OAAO;CAGxB,IAAI;CAEJ,MAAM,WAAgB;EACpB;EAEA,MAAM,MACJ,MACA,eAC6B;GAC7B,MAAM,eAAe,iBAAiB,OAAO;GAG7C,MAAM,SAAS,OAAO;GACtB,IAAI,QAAQ,iBAAiB,UAAa,KAAK,SAAS,OAAO,cAC7D,MAAM,IAAI,MACR,QAAQ,KAAK,uBAAuB,KAAK,OAAO,uDAAuD,OAAO,cAChH;GAEF,IAAI,QAAQ,aAAa,QAAW;IAClC,MAAM,aAAa,KAAK,QACrB,KAAK,QAAQ,MAAM,OAAO,WAAW,IAAI,QAAQ,EAAE,GACpD,CACF;IACA,IAAI,aAAa,OAAO,UACtB,MAAM,IAAI,MACR,QAAQ,KAAK,uBAAuB,WAAW,gEAAgE,OAAO,UACxH;GAEJ;GAIA,MAAM,UAAgF,CAAC;GAEvF,KAAK,MAAM,OAAO,MAAM;IACtB,MAAM,SAASA,MAAU,IAAI,MAAM,YAAY;IAE/C,KAAK,MAAM,SAAS,QAAQ;KAC1B,MAAM,QAAqB;MACzB,UAAU,IAAI;MACd,YAAY,MAAM;MAClB,MAAM,MAAM;MACZ,MAAM,MAAM;MACZ,UAAU,IAAI;KAChB;KAEA,QAAQ,KAAK;MACX,KAAK,OAAO,WAAW,IAAI,IAAI,MAAM,KAAK;MAC1C;MACA,MAAM,MAAM;MACZ,MAAM,IAAI;KACZ,CAAC;IACH;GACF;GAIA,IAAI,QAAQ,WAAW,GACrB,OAAO,EAAE,QAAQ,EAAE;GAIrB,IAAI,QAAQ,cAAc,UAAa,QAAQ,SAAS,OAAO,WAC7D,MAAM,IAAI,MACR,QAAQ,KAAK,uBAAuB,QAAQ,OAAO,iDAAiD,OAAO,WAC7G;GAKF,KAAK,IAAI,SAAS,GAAG,SAAS,QAAQ,QAAQ,UAAU,mBAAmB;IACzE,MAAM,QAAQ,QAAQ,MAAM,QAAQ,SAAS,iBAAiB;IAC9D,MAAM,EAAE,SAAS,eAAe,MAAM,SAAS,UAC7C,MAAM,KAAK,WAAW,OAAO,IAAI,CACnC;IAEA,IAAI,sBAAsB,UAAa,eAAe,GACpD,oBAAoB;IAGtB,MAAM,QAAQ,IACZ,MAAM,KAAK,QAAQ,aACjB,MAAM,OAAO,OAAO,KAAK,OAAO,OAAO,QAAQ,WAAW,OAAO,IAAI,CACvE,CACF;GACF;GAEA,OAAO,EAAE,QAAQ,QAAQ,OAAO;EAClC;EAEA,MAAM,SAAS,OAAe,SAAoD;GAChF,OAAOC,SACL,OACA;IACE;IACA;IACA;IACA,UAAU,OAAO;IACjB,UAAU,OAAO;IACjB;GACF,GACA,OACF;EACF;EAEA,MAAM,QAAuB;GAC3B,MAAM,MAAM,gBAAgB,SAAS;EACvC;EAEA,OAAO,SAA4B;GACjC,OAAO,UAAU,OAAO,OAAO,oBAAoB,SAAS,SAAS,OAAO,eAAe,GAAG,OAAO;EACvG;CACF;CAEA,OAAO;AACT;;;;;;AAOA,SAAS,OAAO,WAAmB,UAAkB,YAA4B;CAC/E,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG;AACrC"}

package/esm/rag/rerank/keyword-reranker.d.mts

@@ -0,0 +1,32 @@
+import { RagReranker } from "./reranker.contract.mjs";
+
+//#region ../@warlock.js/ai/src/rag/rerank/keyword-reranker.d.ts
+/** Options for the {@link keywordReranker}. */
+type KeywordRerankerOptions = {
+  /**
+   * Weight of the lexical-overlap signal blended with the original cosine
+   * score, in `[0, 1]`. `1` ranks purely by keyword overlap; `0` keeps the
+   * cosine order. Default `0.5`.
+   */
+  weight?: number;
+};
+/**
+ * Zero-dependency lexical reranker (a BM25-lite, IDF-free keyword overlap).
+ *
+ * For each candidate it computes the fraction of distinct query terms that
+ * appear in the chunk, blends that with the candidate's original cosine
+ * score by `weight`, and sorts descending. A pure-lexical pass costs
+ * nothing beyond string splits — no peer, no model — so it is the
+ * recommended opt-in reranker when an embedding-only ranking surfaces a
+ * keyword-rich chunk too low.
+ *
+ * Ties (equal blended score) preserve the incoming order, so the cosine
+ * ranking breaks ties deterministically.
+ *
+ * @example
+ * const kb = ai.rag({ embedder, store, reranker: ai.rag.keywordReranker() });
+ */
+declare function keywordReranker(options?: KeywordRerankerOptions): RagReranker;
+//#endregion
+export { KeywordRerankerOptions, keywordReranker };
+//# sourceMappingURL=keyword-reranker.d.mts.map

package/esm/rag/rerank/keyword-reranker.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"keyword-reranker.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/rerank/keyword-reranker.ts"],"mappings":";;;;KAIY,sBAAA;EAAA;;;;AAMJ;EAAN,MAAM;AAAA;;;;;;;AA2B0E;;;;;;;;;;iBAAlE,eAAA,CAAgB,OAAA,GAAS,sBAAA,GAA8B,WAAW"}

package/esm/rag/rerank/keyword-reranker.mjs

@@ -0,0 +1,58 @@
+//#region ../@warlock.js/ai/src/rag/rerank/keyword-reranker.ts
+/** Splits text into lowercase alphanumeric terms. */
+function tokenize(text) {
+	return text.toLowerCase().split(/[^a-z0-9]+/).filter((term) => term.length > 0);
+}
+/**
+* Zero-dependency lexical reranker (a BM25-lite, IDF-free keyword overlap).
+*
+* For each candidate it computes the fraction of distinct query terms that
+* appear in the chunk, blends that with the candidate's original cosine
+* score by `weight`, and sorts descending. A pure-lexical pass costs
+* nothing beyond string splits — no peer, no model — so it is the
+* recommended opt-in reranker when an embedding-only ranking surfaces a
+* keyword-rich chunk too low.
+*
+* Ties (equal blended score) preserve the incoming order, so the cosine
+* ranking breaks ties deterministically.
+*
+* @example
+* const kb = ai.rag({ embedder, store, reranker: ai.rag.keywordReranker() });
+*/
+function keywordReranker(options = {}) {
+	const weight = options.weight ?? .5;
+	return {
+		name: "keyword",
+		async rerank(query, candidates) {
+			if (candidates.length === 0) return [];
+			const queryTerms = new Set(tokenize(query));
+			if (queryTerms.size === 0) return [...candidates];
+			const scored = candidates.map((candidate, position) => {
+				const chunkTerms = new Set(tokenize(candidate.text));
+				let overlap = 0;
+				for (const term of queryTerms) if (chunkTerms.has(term)) overlap += 1;
+				return {
+					candidate,
+					blended: weight * (overlap / queryTerms.size) + (1 - weight) * candidate.score,
+					position
+				};
+			});
+			scored.sort((first, second) => {
+				if (second.blended !== first.blended) return second.blended - first.blended;
+				return first.position - second.position;
+			});
+			return scored.map((entry) => ({
+				...entry.candidate,
+				score: entry.blended,
+				citation: {
+					...entry.candidate.citation,
+					score: entry.blended
+				}
+			}));
+		}
+	};
+}
+
+//#endregion
+export { keywordReranker };
+//# sourceMappingURL=keyword-reranker.mjs.map

package/esm/rag/rerank/keyword-reranker.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"keyword-reranker.mjs","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/rerank/keyword-reranker.ts"],"sourcesContent":["import type { RetrievedChunk } from \"../contracts/citation.type\";\nimport type { RagReranker } from \"./reranker.contract\";\n\n/** Options for the {@link keywordReranker}. */\nexport type KeywordRerankerOptions = {\n  /**\n   * Weight of the lexical-overlap signal blended with the original cosine\n   * score, in `[0, 1]`. `1` ranks purely by keyword overlap; `0` keeps the\n   * cosine order. Default `0.5`.\n   */\n  weight?: number;\n};\n\n/** Splits text into lowercase alphanumeric terms. */\nfunction tokenize(text: string): string[] {\n  return text\n    .toLowerCase()\n    .split(/[^a-z0-9]+/)\n    .filter((term) => term.length > 0);\n}\n\n/**\n * Zero-dependency lexical reranker (a BM25-lite, IDF-free keyword overlap).\n *\n * For each candidate it computes the fraction of distinct query terms that\n * appear in the chunk, blends that with the candidate's original cosine\n * score by `weight`, and sorts descending. A pure-lexical pass costs\n * nothing beyond string splits — no peer, no model — so it is the\n * recommended opt-in reranker when an embedding-only ranking surfaces a\n * keyword-rich chunk too low.\n *\n * Ties (equal blended score) preserve the incoming order, so the cosine\n * ranking breaks ties deterministically.\n *\n * @example\n * const kb = ai.rag({ embedder, store, reranker: ai.rag.keywordReranker() });\n */\nexport function keywordReranker(options: KeywordRerankerOptions = {}): RagReranker {\n  const weight = options.weight ?? 0.5;\n\n  return {\n    name: \"keyword\",\n    async rerank(query: string, candidates: RetrievedChunk[]): Promise<RetrievedChunk[]> {\n      if (candidates.length === 0) {\n        return [];\n      }\n\n      const queryTerms = new Set(tokenize(query));\n\n      if (queryTerms.size === 0) {\n        return [...candidates];\n      }\n\n      const scored = candidates.map((candidate, position) => {\n        const chunkTerms = new Set(tokenize(candidate.text));\n\n        let overlap = 0;\n        for (const term of queryTerms) {\n          if (chunkTerms.has(term)) {\n            overlap += 1;\n          }\n        }\n\n        const lexical = overlap / queryTerms.size;\n        const blended = weight * lexical + (1 - weight) * candidate.score;\n\n        return { candidate, blended, position };\n      });\n\n      scored.sort((first, second) => {\n        if (second.blended !== first.blended) {\n          return second.blended - first.blended;\n        }\n\n        // Stable on ties: keep the incoming (cosine) order.\n        return first.position - second.position;\n      });\n\n      return scored.map((entry) => ({\n        ...entry.candidate,\n        score: entry.blended,\n        citation: { ...entry.candidate.citation, score: entry.blended },\n      }));\n    },\n  };\n}\n"],"mappings":";;AAcA,SAAS,SAAS,MAAwB;CACxC,OAAO,KACJ,YAAY,CAAC,CACb,MAAM,YAAY,CAAC,CACnB,QAAQ,SAAS,KAAK,SAAS,CAAC;AACrC;;;;;;;;;;;;;;;;;AAkBA,SAAgB,gBAAgB,UAAkC,CAAC,GAAgB;CACjF,MAAM,SAAS,QAAQ,UAAU;CAEjC,OAAO;EACL,MAAM;EACN,MAAM,OAAO,OAAe,YAAyD;GACnF,IAAI,WAAW,WAAW,GACxB,OAAO,CAAC;GAGV,MAAM,aAAa,IAAI,IAAI,SAAS,KAAK,CAAC;GAE1C,IAAI,WAAW,SAAS,GACtB,OAAO,CAAC,GAAG,UAAU;GAGvB,MAAM,SAAS,WAAW,KAAK,WAAW,aAAa;IACrD,MAAM,aAAa,IAAI,IAAI,SAAS,UAAU,IAAI,CAAC;IAEnD,IAAI,UAAU;IACd,KAAK,MAAM,QAAQ,YACjB,IAAI,WAAW,IAAI,IAAI,GACrB,WAAW;IAOf,OAAO;KAAE;KAAW,SAFJ,UADA,UAAU,WAAW,SACD,IAAI,UAAU,UAAU;KAE/B;IAAS;GACxC,CAAC;GAED,OAAO,MAAM,OAAO,WAAW;IAC7B,IAAI,OAAO,YAAY,MAAM,SAC3B,OAAO,OAAO,UAAU,MAAM;IAIhC,OAAO,MAAM,WAAW,OAAO;GACjC,CAAC;GAED,OAAO,OAAO,KAAK,WAAW;IAC5B,GAAG,MAAM;IACT,OAAO,MAAM;IACb,UAAU;KAAE,GAAG,MAAM,UAAU;KAAU,OAAO,MAAM;IAAQ;GAChE,EAAE;EACJ;CACF;AACF"}

package/esm/rag/rerank/llm-reranker.d.mts

@@ -0,0 +1,36 @@
+import { ModelContract } from "../../contracts/model.contract.mjs";
+import { RagReranker } from "./reranker.contract.mjs";
+
+//#region ../@warlock.js/ai/src/rag/rerank/llm-reranker.d.ts
+/** Options for the {@link llmReranker}. */
+type LlmRerankerOptions = {
+  /** The model used to score candidate relevance. Required. */model: ModelContract;
+  /**
+   * How many candidates to score per model call. Larger batches mean
+   * fewer round-trips but a longer prompt. Default `10`.
+   */
+  batchSize?: number;
+};
+/**
+ * Optional model-backed reranker.
+ *
+ * Asks an LLM to grade each over-fetched candidate's relevance to the
+ * query on a `0..1` scale, then sorts descending by the model's score.
+ * Candidates the model does not score keep their original cosine score, so
+ * a partial/garbled reply degrades gracefully rather than dropping hits.
+ * Scoring is batched (`batchSize`) to bound prompt length.
+ *
+ * Unlike {@link keywordReranker}, this costs one or more model calls per
+ * retrieval — opt in only when precision matters more than latency/cost.
+ *
+ * @example
+ * const kb = ai.rag({
+ *   embedder,
+ *   store,
+ *   reranker: ai.rag.llmReranker({ model: openai.model({ name: "gpt-4o-mini" }) }),
+ * });
+ */
+declare function llmReranker(options: LlmRerankerOptions): RagReranker;
+//#endregion
+export { LlmRerankerOptions, llmReranker };
+//# sourceMappingURL=llm-reranker.d.mts.map

package/esm/rag/rerank/llm-reranker.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-reranker.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/rerank/llm-reranker.ts"],"mappings":";;;;;KAMY,kBAAA;EAAA,6DAEV,KAAA,EAAO,aAAa;;;;;EAKpB,SAAA;AAAA;AAAS;AA8FX;;;;;;;;AAAqE;;;;;;;;;;AA9F1D,iBA8FK,WAAA,CAAY,OAAA,EAAS,kBAAA,GAAqB,WAAW"}

package/esm/rag/rerank/llm-reranker.mjs

@@ -0,0 +1,85 @@
+//#region ../@warlock.js/ai/src/rag/rerank/llm-reranker.ts
+/**
+* Build the scoring prompt — the model rates each candidate's relevance to
+* the query on a `0..1` scale and replies with one `index: score` line per
+* candidate. Kept terse and JSON-light so any chat model can answer.
+*/
+function buildPrompt(query, candidates) {
+	return [{
+		role: "system",
+		content: "You are a relevance grader. For each numbered passage, rate how well it answers the query on a scale from 0 (irrelevant) to 1 (fully relevant). Reply with ONLY a JSON array of objects like [{\"index\":0,\"score\":0.9}], one entry per passage, no prose."
+	}, {
+		role: "user",
+		content: `Query: ${query}\n\nPassages:\n${candidates.map((candidate, index) => `[${index}] ${candidate.text}`).join("\n\n")}`
+	}];
+}
+/**
+* Parse the model's reply into a score map. Tolerant of surrounding prose:
+* extracts the first JSON array and reads `{ index, score }` entries.
+* Returns an empty map when nothing parseable is found, so the caller can
+* fall back to the original order.
+*/
+function parseScores(reply) {
+	const scores = /* @__PURE__ */ new Map();
+	const start = reply.indexOf("[");
+	const end = reply.lastIndexOf("]");
+	if (start === -1 || end === -1 || end <= start) return scores;
+	let parsed;
+	try {
+		parsed = JSON.parse(reply.slice(start, end + 1));
+	} catch {
+		return scores;
+	}
+	if (!Array.isArray(parsed)) return scores;
+	for (const entry of parsed) if (entry && typeof entry.index === "number" && typeof entry.score === "number" && Number.isFinite(entry.score)) scores.set(entry.index, Math.max(0, Math.min(1, entry.score)));
+	return scores;
+}
+/**
+* Optional model-backed reranker.
+*
+* Asks an LLM to grade each over-fetched candidate's relevance to the
+* query on a `0..1` scale, then sorts descending by the model's score.
+* Candidates the model does not score keep their original cosine score, so
+* a partial/garbled reply degrades gracefully rather than dropping hits.
+* Scoring is batched (`batchSize`) to bound prompt length.
+*
+* Unlike {@link keywordReranker}, this costs one or more model calls per
+* retrieval — opt in only when precision matters more than latency/cost.
+*
+* @example
+* const kb = ai.rag({
+*   embedder,
+*   store,
+*   reranker: ai.rag.llmReranker({ model: openai.model({ name: "gpt-4o-mini" }) }),
+* });
+*/
+function llmReranker(options) {
+	const batchSize = options.batchSize ?? 10;
+	return {
+		name: "llm",
+		async rerank(query, candidates) {
+			if (candidates.length === 0) return [];
+			const rescored = [];
+			for (let offset = 0; offset < candidates.length; offset += batchSize) {
+				const batch = candidates.slice(offset, offset + batchSize);
+				const scores = parseScores((await options.model.complete(buildPrompt(query, batch))).content);
+				batch.forEach((candidate, index) => {
+					const score = scores.has(index) ? scores.get(index) : candidate.score;
+					rescored.push({
+						...candidate,
+						score,
+						citation: {
+							...candidate.citation,
+							score
+						}
+					});
+				});
+			}
+			return rescored.sort((first, second) => second.score - first.score);
+		}
+	};
+}
+
+//#endregion
+export { llmReranker };
+//# sourceMappingURL=llm-reranker.mjs.map

package/esm/rag/rerank/llm-reranker.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-reranker.mjs","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/rerank/llm-reranker.ts"],"sourcesContent":["import type { Message } from \"../../contracts/conversation-message.type\";\nimport type { ModelContract } from \"../../contracts/model.contract\";\nimport type { RetrievedChunk } from \"../contracts/citation.type\";\nimport type { RagReranker } from \"./reranker.contract\";\n\n/** Options for the {@link llmReranker}. */\nexport type LlmRerankerOptions = {\n  /** The model used to score candidate relevance. Required. */\n  model: ModelContract;\n  /**\n   * How many candidates to score per model call. Larger batches mean\n   * fewer round-trips but a longer prompt. Default `10`.\n   */\n  batchSize?: number;\n};\n\n/**\n * A single relevance score the model returns for a candidate, in `[0, 1]`,\n * keyed by the candidate's position in the batch.\n */\ntype ScoreLine = {\n  index: number;\n  score: number;\n};\n\n/**\n * Build the scoring prompt — the model rates each candidate's relevance to\n * the query on a `0..1` scale and replies with one `index: score` line per\n * candidate. Kept terse and JSON-light so any chat model can answer.\n */\nfunction buildPrompt(query: string, candidates: RetrievedChunk[]): Message[] {\n  const lines = candidates\n    .map((candidate, index) => `[${index}] ${candidate.text}`)\n    .join(\"\\n\\n\");\n\n  return [\n    {\n      role: \"system\",\n      content:\n        \"You are a relevance grader. For each numbered passage, rate how well it answers the query on a scale from 0 (irrelevant) to 1 (fully relevant). Reply with ONLY a JSON array of objects like [{\\\"index\\\":0,\\\"score\\\":0.9}], one entry per passage, no prose.\",\n    },\n    {\n      role: \"user\",\n      content: `Query: ${query}\\n\\nPassages:\\n${lines}`,\n    },\n  ];\n}\n\n/**\n * Parse the model's reply into a score map. Tolerant of surrounding prose:\n * extracts the first JSON array and reads `{ index, score }` entries.\n * Returns an empty map when nothing parseable is found, so the caller can\n * fall back to the original order.\n */\nfunction parseScores(reply: string): Map<number, number> {\n  const scores = new Map<number, number>();\n  const start = reply.indexOf(\"[\");\n  const end = reply.lastIndexOf(\"]\");\n\n  if (start === -1 || end === -1 || end <= start) {\n    return scores;\n  }\n\n  let parsed: unknown;\n  try {\n    parsed = JSON.parse(reply.slice(start, end + 1));\n  } catch {\n    return scores;\n  }\n\n  if (!Array.isArray(parsed)) {\n    return scores;\n  }\n\n  for (const entry of parsed as ScoreLine[]) {\n    if (\n      entry &&\n      typeof entry.index === \"number\" &&\n      typeof entry.score === \"number\" &&\n      Number.isFinite(entry.score)\n    ) {\n      scores.set(entry.index, Math.max(0, Math.min(1, entry.score)));\n    }\n  }\n\n  return scores;\n}\n\n/**\n * Optional model-backed reranker.\n *\n * Asks an LLM to grade each over-fetched candidate's relevance to the\n * query on a `0..1` scale, then sorts descending by the model's score.\n * Candidates the model does not score keep their original cosine score, so\n * a partial/garbled reply degrades gracefully rather than dropping hits.\n * Scoring is batched (`batchSize`) to bound prompt length.\n *\n * Unlike {@link keywordReranker}, this costs one or more model calls per\n * retrieval — opt in only when precision matters more than latency/cost.\n *\n * @example\n * const kb = ai.rag({\n *   embedder,\n *   store,\n *   reranker: ai.rag.llmReranker({ model: openai.model({ name: \"gpt-4o-mini\" }) }),\n * });\n */\nexport function llmReranker(options: LlmRerankerOptions): RagReranker {\n  const batchSize = options.batchSize ?? 10;\n\n  return {\n    name: \"llm\",\n    async rerank(query: string, candidates: RetrievedChunk[]): Promise<RetrievedChunk[]> {\n      if (candidates.length === 0) {\n        return [];\n      }\n\n      const rescored: RetrievedChunk[] = [];\n\n      for (let offset = 0; offset < candidates.length; offset += batchSize) {\n        const batch = candidates.slice(offset, offset + batchSize);\n        const response = await options.model.complete(buildPrompt(query, batch));\n        const scores = parseScores(response.content);\n\n        batch.forEach((candidate, index) => {\n          const score = scores.has(index) ? (scores.get(index) as number) : candidate.score;\n\n          rescored.push({\n            ...candidate,\n            score,\n            citation: { ...candidate.citation, score },\n          });\n        });\n      }\n\n      return rescored.sort((first, second) => second.score - first.score);\n    },\n  };\n}\n"],"mappings":";;;;;;AA8BA,SAAS,YAAY,OAAe,YAAyC;CAK3E,OAAO,CACL;EACE,MAAM;EACN,SACE;CACJ,GACA;EACE,MAAM;EACN,SAAS,UAAU,MAAM,iBAZf,WACX,KAAK,WAAW,UAAU,IAAI,MAAM,IAAI,UAAU,MAAM,CAAC,CACzD,KAAK,MAU0C;CAChD,CACF;AACF;;;;;;;AAQA,SAAS,YAAY,OAAoC;CACvD,MAAM,yBAAS,IAAI,IAAoB;CACvC,MAAM,QAAQ,MAAM,QAAQ,GAAG;CAC/B,MAAM,MAAM,MAAM,YAAY,GAAG;CAEjC,IAAI,UAAU,MAAM,QAAQ,MAAM,OAAO,OACvC,OAAO;CAGT,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,MAAM,MAAM,OAAO,MAAM,CAAC,CAAC;CACjD,QAAQ;EACN,OAAO;CACT;CAEA,IAAI,CAAC,MAAM,QAAQ,MAAM,GACvB,OAAO;CAGT,KAAK,MAAM,SAAS,QAClB,IACE,SACA,OAAO,MAAM,UAAU,YACvB,OAAO,MAAM,UAAU,YACvB,OAAO,SAAS,MAAM,KAAK,GAE3B,OAAO,IAAI,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,KAAK,CAAC,CAAC;CAIjE,OAAO;AACT;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,YAAY,SAA0C;CACpE,MAAM,YAAY,QAAQ,aAAa;CAEvC,OAAO;EACL,MAAM;EACN,MAAM,OAAO,OAAe,YAAyD;GACnF,IAAI,WAAW,WAAW,GACxB,OAAO,CAAC;GAGV,MAAM,WAA6B,CAAC;GAEpC,KAAK,IAAI,SAAS,GAAG,SAAS,WAAW,QAAQ,UAAU,WAAW;IACpE,MAAM,QAAQ,WAAW,MAAM,QAAQ,SAAS,SAAS;IAEzD,MAAM,SAAS,aAAY,MADJ,QAAQ,MAAM,SAAS,YAAY,OAAO,KAAK,CAAC,EACpC,CAAC,OAAO;IAE3C,MAAM,SAAS,WAAW,UAAU;KAClC,MAAM,QAAQ,OAAO,IAAI,KAAK,IAAK,OAAO,IAAI,KAAK,IAAe,UAAU;KAE5E,SAAS,KAAK;MACZ,GAAG;MACH;MACA,UAAU;OAAE,GAAG,UAAU;OAAU;MAAM;KAC3C,CAAC;IACH,CAAC;GACH;GAEA,OAAO,SAAS,MAAM,OAAO,WAAW,OAAO,QAAQ,MAAM,KAAK;EACpE;CACF;AACF"}

package/esm/rag/rerank/reranker.contract.d.mts

@@ -0,0 +1,28 @@
+import { RetrievedChunk } from "../contracts/citation.type.mjs";
+
+//#region ../@warlock.js/ai/src/rag/rerank/reranker.contract.d.ts
+/**
+ * Reranks an over-fetched candidate set against the query before the
+ * pipeline slices to `topK`. Optional — when no reranker is configured the
+ * raw cosine order from the vector store is kept.
+ *
+ * A reranker receives the candidates already scored by cosine similarity
+ * and returns them reordered (and typically re-scored). The pipeline does
+ * NOT trust the input order; it relies entirely on the returned order. A
+ * reranker that throws is caught by the pipeline, which falls back to the
+ * cosine order — so an implementation may throw to opt out of a given
+ * query rather than returning garbage.
+ */
+interface RagReranker {
+  /** Stable name for logs / diagnostics. */
+  readonly name: string;
+  /**
+   * Reorder (and optionally re-score) `candidates` by relevance to
+   * `query`. Returns the reranked list; the pipeline slices `topK` from
+   * the front.
+   */
+  rerank(query: string, candidates: RetrievedChunk[]): Promise<RetrievedChunk[]>;
+}
+//#endregion
+export { RagReranker };
+//# sourceMappingURL=reranker.contract.d.mts.map

package/esm/rag/rerank/reranker.contract.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.contract.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/rerank/reranker.contract.ts"],"mappings":";;;;;AAcA;;;;;;;;;;UAAiB,WAAA;EAQmB;EAAA,SANzB,IAAA;EAM4C;;;AAAsB;;EAA3E,MAAA,CAAO,KAAA,UAAe,UAAA,EAAY,cAAA,KAAmB,OAAA,CAAQ,cAAA;AAAA"}

package/esm/rag/retrieve.mjs

@@ -0,0 +1,68 @@
+//#region ../@warlock.js/ai/src/rag/retrieve.ts
+/** Default number of chunks returned after reranking. */
+const DEFAULT_TOP_K = 5;
+/** Default cosine floor applied at the vector-store stage. */
+const DEFAULT_THRESHOLD = .5;
+/**
+* The cite pipeline: embed the query → over-fetch candidates from the
+* store → filter to this rag's namespace → map to {@link RetrievedChunk}s
+* with a {@link Citation} → optionally rerank → slice `topK`.
+*
+* Behavior matches the design's failure modes:
+* - No hits clearing the threshold → `{ query, chunks: [] }`, never throws.
+* - Namespace-prefix filtering keeps two rags sharing one driver isolated.
+* - A reranker that throws is caught; the raw cosine order is used instead.
+* - A dimension mismatch (indexed with model A, queried with model B)
+*   throws a clear error rather than returning garbage hits.
+*/
+async function retrieve(query, deps, options = {}) {
+	const topK = options.topK ?? deps.defaults?.topK ?? 5;
+	const threshold = options.threshold ?? deps.defaults?.threshold ?? .5;
+	const tags = options.tags ?? deps.defaults?.tags;
+	const candidates = options.candidates ?? deps.defaults?.candidates ?? Math.max(topK * 4, topK);
+	const { vector, dimensions } = await deps.embedder.embed(query);
+	if (deps.indexedDimensions !== void 0 && dimensions !== 0 && deps.indexedDimensions !== 0 && dimensions !== deps.indexedDimensions) throw new Error(`rag.retrieve(): query embedder dimensions (${dimensions}) do not match the dimensions captured at index time (${deps.indexedDimensions}); index and query must use the same embedding model`);
+	const hits = await deps.store.query(vector, {
+		topK: candidates,
+		threshold,
+		tags
+	});
+	const prefix = `${deps.namespace}.`;
+	let retrieved = hits.filter((hit) => hit.key.startsWith(prefix)).map((hit) => toRetrievedChunk(hit.value, hit.score));
+	retrieved = await applyReranker(query, retrieved, deps.reranker);
+	return {
+		query,
+		chunks: retrieved.slice(0, topK)
+	};
+}
+/** Build a cited {@link RetrievedChunk} from a stored chunk + its cosine score. */
+function toRetrievedChunk(stored, score) {
+	const citation = {
+		sourceId: stored.sourceId,
+		chunkIndex: stored.chunkIndex,
+		span: stored.span,
+		score,
+		metadata: stored.metadata
+	};
+	return {
+		text: stored.text,
+		score,
+		citation
+	};
+}
+/**
+* Run the optional reranker, degrading to the raw cosine order if it
+* throws — a flaky optional reranker must never fail the whole retrieval.
+*/
+async function applyReranker(query, candidates, reranker) {
+	if (!reranker) return candidates;
+	try {
+		return await reranker.rerank(query, candidates);
+	} catch {
+		return candidates;
+	}
+}
+
+//#endregion
+export { retrieve };
+//# sourceMappingURL=retrieve.mjs.map

package/esm/rag/retrieve.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieve.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/rag/retrieve.ts"],"sourcesContent":["import type { EmbedderContract } from \"../contracts/embedder.contract\";\nimport type {\n  Citation,\n  RetrievedChunk,\n  RetrieveOptions,\n  RetrieveResult,\n} from \"./contracts/citation.type\";\nimport type { RagReranker } from \"./rerank/reranker.contract\";\nimport type { VectorStore } from \"./store/vector-store.contract\";\n\n/** Default number of chunks returned after reranking. */\nexport const DEFAULT_TOP_K = 5;\n\n/** Default cosine floor applied at the vector-store stage. */\nexport const DEFAULT_THRESHOLD = 0.5;\n\n/**\n * Shape persisted per chunk in the vector store. The vector itself is held\n * by the driver's own index (passed via `set({ vector })`), so it is not\n * duplicated here.\n */\nexport type StoredChunk = {\n  sourceId: string;\n  chunkIndex: number;\n  span: [start: number, end: number];\n  text: string;\n  metadata?: Record<string, unknown>;\n};\n\n/** Dependencies the retrieve pipeline needs, resolved once by `rag()`. */\nexport type RetrieveDeps = {\n  embedder: EmbedderContract;\n  store: VectorStore;\n  /** Namespace prefix every stored key carries (e.g. `\"ai.rag.docs\"`). */\n  namespace: string;\n  /** Optional reranker; when absent the cosine order is kept. */\n  reranker?: RagReranker;\n  /** Pipeline-level retrieval defaults. */\n  defaults?: RetrieveOptions;\n  /**\n   * Dimension count captured at first index for the mismatch guard. When\n   * set, the query embedder's `dimensions` must equal it.\n   */\n  indexedDimensions?: number;\n};\n\n/**\n * The cite pipeline: embed the query → over-fetch candidates from the\n * store → filter to this rag's namespace → map to {@link RetrievedChunk}s\n * with a {@link Citation} → optionally rerank → slice `topK`.\n *\n * Behavior matches the design's failure modes:\n * - No hits clearing the threshold → `{ query, chunks: [] }`, never throws.\n * - Namespace-prefix filtering keeps two rags sharing one driver isolated.\n * - A reranker that throws is caught; the raw cosine order is used instead.\n * - A dimension mismatch (indexed with model A, queried with model B)\n *   throws a clear error rather than returning garbage hits.\n */\nexport async function retrieve(\n  query: string,\n  deps: RetrieveDeps,\n  options: RetrieveOptions = {},\n): Promise<RetrieveResult> {\n  const topK = options.topK ?? deps.defaults?.topK ?? DEFAULT_TOP_K;\n  const threshold = options.threshold ?? deps.defaults?.threshold ?? DEFAULT_THRESHOLD;\n  const tags = options.tags ?? deps.defaults?.tags;\n  const candidates = options.candidates ?? deps.defaults?.candidates ?? Math.max(topK * 4, topK);\n\n  const { vector, dimensions } = await deps.embedder.embed(query);\n\n  if (\n    deps.indexedDimensions !== undefined &&\n    dimensions !== 0 &&\n    deps.indexedDimensions !== 0 &&\n    dimensions !== deps.indexedDimensions\n  ) {\n    throw new Error(\n      `rag.retrieve(): query embedder dimensions (${dimensions}) do not match the dimensions captured at index time (${deps.indexedDimensions}); index and query must use the same embedding model`,\n    );\n  }\n\n  const hits = await deps.store.query<StoredChunk>(vector, {\n    topK: candidates,\n    threshold,\n    tags,\n  });\n\n  const prefix = `${deps.namespace}.`;\n\n  let retrieved: RetrievedChunk[] = hits\n    .filter((hit) => hit.key.startsWith(prefix))\n    .map((hit) => toRetrievedChunk(hit.value, hit.score));\n\n  retrieved = await applyReranker(query, retrieved, deps.reranker);\n\n  return { query, chunks: retrieved.slice(0, topK) };\n}\n\n/** Build a cited {@link RetrievedChunk} from a stored chunk + its cosine score. */\nfunction toRetrievedChunk(stored: StoredChunk, score: number): RetrievedChunk {\n  const citation: Citation = {\n    sourceId: stored.sourceId,\n    chunkIndex: stored.chunkIndex,\n    span: stored.span,\n    score,\n    metadata: stored.metadata,\n  };\n\n  return { text: stored.text, score, citation };\n}\n\n/**\n * Run the optional reranker, degrading to the raw cosine order if it\n * throws — a flaky optional reranker must never fail the whole retrieval.\n */\nasync function applyReranker(\n  query: string,\n  candidates: RetrievedChunk[],\n  reranker: RagReranker | undefined,\n): Promise<RetrievedChunk[]> {\n  if (!reranker) {\n    return candidates;\n  }\n\n  try {\n    return await reranker.rerank(query, candidates);\n  } catch {\n    // Logged at the call site in a richer build; here we degrade silently\n    // to vector-only ranking rather than aborting the retrieval.\n    return candidates;\n  }\n}\n"],"mappings":";;AAWA,MAAa,gBAAgB;;AAG7B,MAAa,oBAAoB;;;;;;;;;;;;;AA4CjC,eAAsB,SACpB,OACA,MACA,UAA2B,CAAC,GACH;CACzB,MAAM,OAAO,QAAQ,QAAQ,KAAK,UAAU;CAC5C,MAAM,YAAY,QAAQ,aAAa,KAAK,UAAU;CACtD,MAAM,OAAO,QAAQ,QAAQ,KAAK,UAAU;CAC5C,MAAM,aAAa,QAAQ,cAAc,KAAK,UAAU,cAAc,KAAK,IAAI,OAAO,GAAG,IAAI;CAE7F,MAAM,EAAE,QAAQ,eAAe,MAAM,KAAK,SAAS,MAAM,KAAK;CAE9D,IACE,KAAK,sBAAsB,UAC3B,eAAe,KACf,KAAK,sBAAsB,KAC3B,eAAe,KAAK,mBAEpB,MAAM,IAAI,MACR,8CAA8C,WAAW,wDAAwD,KAAK,kBAAkB,qDAC1I;CAGF,MAAM,OAAO,MAAM,KAAK,MAAM,MAAmB,QAAQ;EACvD,MAAM;EACN;EACA;CACF,CAAC;CAED,MAAM,SAAS,GAAG,KAAK,UAAU;CAEjC,IAAI,YAA8B,KAC/B,QAAQ,QAAQ,IAAI,IAAI,WAAW,MAAM,CAAC,CAAC,CAC3C,KAAK,QAAQ,iBAAiB,IAAI,OAAO,IAAI,KAAK,CAAC;CAEtD,YAAY,MAAM,cAAc,OAAO,WAAW,KAAK,QAAQ;CAE/D,OAAO;EAAE;EAAO,QAAQ,UAAU,MAAM,GAAG,IAAI;CAAE;AACnD;;AAGA,SAAS,iBAAiB,QAAqB,OAA+B;CAC5E,MAAM,WAAqB;EACzB,UAAU,OAAO;EACjB,YAAY,OAAO;EACnB,MAAM,OAAO;EACb;EACA,UAAU,OAAO;CACnB;CAEA,OAAO;EAAE,MAAM,OAAO;EAAM;EAAO;CAAS;AAC9C;;;;;AAMA,eAAe,cACb,OACA,YACA,UAC2B;CAC3B,IAAI,CAAC,UACH,OAAO;CAGT,IAAI;EACF,OAAO,MAAM,SAAS,OAAO,OAAO,UAAU;CAChD,QAAQ;EAGN,OAAO;CACT;AACF"}