@warlock.js/ai 4.3.0 → 4.5.0

package/esm/rag/chunk/sentence.mjs

@@ -0,0 +1,73 @@
+//#region ../@warlock.js/ai/src/rag/chunk/sentence.ts
+/** Matches a sentence terminator (`.`, `!`, `?`) followed by whitespace. */
+const SENTENCE_BOUNDARY = /([.!?])\s+/g;
+/**
+* Sentence-aware character splitter.
+*
+* Splits the text on sentence terminators (`. `, `! `, `? `), keeping the
+* terminator attached, then greedily packs whole sentences into chunks up
+* to `size` characters, carrying `overlap` characters forward between
+* adjacent chunks. A single sentence longer than `size` becomes its own
+* (oversize) chunk rather than being cut mid-sentence. Spans are exact.
+*/
+function sentenceChunk(text, size, overlap) {
+	if (text.trim().length === 0) return [];
+	const sentences = splitSentences(text);
+	const chunks = [];
+	let bufferStart = -1;
+	let bufferEnd = -1;
+	let index = 0;
+	const flush = () => {
+		if (bufferStart === -1) return;
+		chunks.push({
+			text: text.slice(bufferStart, bufferEnd),
+			index,
+			span: [bufferStart, bufferEnd]
+		});
+		index += 1;
+	};
+	for (const sentence of sentences) {
+		if (bufferStart === -1) {
+			bufferStart = sentence.start;
+			bufferEnd = sentence.end;
+			continue;
+		}
+		if (sentence.end - bufferStart <= size) {
+			bufferEnd = sentence.end;
+			continue;
+		}
+		flush();
+		bufferStart = overlap > 0 ? Math.max(bufferStart, sentence.start - overlap) : sentence.start;
+		bufferEnd = sentence.end;
+	}
+	flush();
+	return chunks;
+}
+/**
+* Split `text` into sentence spans on terminator + whitespace, keeping the
+* terminator with its sentence and absorbing the trailing whitespace into
+* the boundary so reconstructing the spans loses no characters.
+*/
+function splitSentences(text) {
+	const spans = [];
+	let start = 0;
+	let match;
+	SENTENCE_BOUNDARY.lastIndex = 0;
+	while ((match = SENTENCE_BOUNDARY.exec(text)) !== null) {
+		const end = match.index + match[0].length;
+		spans.push({
+			start,
+			end
+		});
+		start = end;
+	}
+	if (start < text.length) spans.push({
+		start,
+		end: text.length
+	});
+	return spans;
+}
+
+//#endregion
+export { sentenceChunk };
+//# sourceMappingURL=sentence.mjs.map

package/esm/rag/chunk/sentence.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentence.mjs","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/chunk/sentence.ts"],"sourcesContent":["import type { Chunk } from \"../contracts/chunk-options.type\";\n\n/** Matches a sentence terminator (`.`, `!`, `?`) followed by whitespace. */\nconst SENTENCE_BOUNDARY = /([.!?])\\s+/g;\n\n/**\n * Sentence-aware character splitter.\n *\n * Splits the text on sentence terminators (`. `, `! `, `? `), keeping the\n * terminator attached, then greedily packs whole sentences into chunks up\n * to `size` characters, carrying `overlap` characters forward between\n * adjacent chunks. A single sentence longer than `size` becomes its own\n * (oversize) chunk rather than being cut mid-sentence. Spans are exact.\n */\nexport function sentenceChunk(text: string, size: number, overlap: number): Chunk[] {\n  if (text.trim().length === 0) {\n    return [];\n  }\n\n  const sentences = splitSentences(text);\n  const chunks: Chunk[] = [];\n\n  let bufferStart = -1;\n  let bufferEnd = -1;\n  let index = 0;\n\n  const flush = (): void => {\n    if (bufferStart === -1) {\n      return;\n    }\n\n    chunks.push({\n      text: text.slice(bufferStart, bufferEnd),\n      index,\n      span: [bufferStart, bufferEnd],\n    });\n    index += 1;\n  };\n\n  for (const sentence of sentences) {\n    if (bufferStart === -1) {\n      bufferStart = sentence.start;\n      bufferEnd = sentence.end;\n\n      continue;\n    }\n\n    if (sentence.end - bufferStart <= size) {\n      bufferEnd = sentence.end;\n\n      continue;\n    }\n\n    flush();\n\n    const overlapStart =\n      overlap > 0 ? Math.max(bufferStart, sentence.start - overlap) : sentence.start;\n\n    bufferStart = overlapStart;\n    bufferEnd = sentence.end;\n  }\n\n  flush();\n\n  return chunks;\n}\n\n/** A sentence with its absolute `[start, end)` span in the original text. */\ntype SentenceSpan = {\n  start: number;\n  end: number;\n};\n\n/**\n * Split `text` into sentence spans on terminator + whitespace, keeping the\n * terminator with its sentence and absorbing the trailing whitespace into\n * the boundary so reconstructing the spans loses no characters.\n */\nfunction splitSentences(text: string): SentenceSpan[] {\n  const spans: SentenceSpan[] = [];\n  let start = 0;\n  let match: RegExpExecArray | null;\n\n  SENTENCE_BOUNDARY.lastIndex = 0;\n\n  while ((match = SENTENCE_BOUNDARY.exec(text)) !== null) {\n    const end = match.index + match[0].length;\n\n    spans.push({ start, end });\n    start = end;\n  }\n\n  if (start < text.length) {\n    spans.push({ start, end: text.length });\n  }\n\n  return spans;\n}\n"],"mappings":";;AAGA,MAAM,oBAAoB;;;;;;;;;;AAW1B,SAAgB,cAAc,MAAc,MAAc,SAA0B;CAClF,IAAI,KAAK,KAAK,CAAC,CAAC,WAAW,GACzB,OAAO,CAAC;CAGV,MAAM,YAAY,eAAe,IAAI;CACrC,MAAM,SAAkB,CAAC;CAEzB,IAAI,cAAc;CAClB,IAAI,YAAY;CAChB,IAAI,QAAQ;CAEZ,MAAM,cAAoB;EACxB,IAAI,gBAAgB,IAClB;EAGF,OAAO,KAAK;GACV,MAAM,KAAK,MAAM,aAAa,SAAS;GACvC;GACA,MAAM,CAAC,aAAa,SAAS;EAC/B,CAAC;EACD,SAAS;CACX;CAEA,KAAK,MAAM,YAAY,WAAW;EAChC,IAAI,gBAAgB,IAAI;GACtB,cAAc,SAAS;GACvB,YAAY,SAAS;GAErB;EACF;EAEA,IAAI,SAAS,MAAM,eAAe,MAAM;GACtC,YAAY,SAAS;GAErB;EACF;EAEA,MAAM;EAKN,cAFE,UAAU,IAAI,KAAK,IAAI,aAAa,SAAS,QAAQ,OAAO,IAAI,SAAS;EAG3E,YAAY,SAAS;CACvB;CAEA,MAAM;CAEN,OAAO;AACT;;;;;;AAaA,SAAS,eAAe,MAA8B;CACpD,MAAM,QAAwB,CAAC;CAC/B,IAAI,QAAQ;CACZ,IAAI;CAEJ,kBAAkB,YAAY;CAE9B,QAAQ,QAAQ,kBAAkB,KAAK,IAAI,OAAO,MAAM;EACtD,MAAM,MAAM,MAAM,QAAQ,MAAM,EAAE,CAAC;EAEnC,MAAM,KAAK;GAAE;GAAO;EAAI,CAAC;EACzB,QAAQ;CACV;CAEA,IAAI,QAAQ,KAAK,QACf,MAAM,KAAK;EAAE;EAAO,KAAK,KAAK;CAAO,CAAC;CAGxC,OAAO;AACT"}

package/esm/rag/contracts/chunk-options.type.d.mts

@@ -0,0 +1,35 @@
+//#region ../@warlock.js/ai/src/rag/contracts/chunk-options.type.d.ts
+/** Splitter strategy used to break a document's text before embedding. */
+type ChunkType = "recursive" | "sentence" | "fixed" | "markdown";
+/**
+ * How a document's text is split before embedding.
+ *
+ * All sizing is in **characters**, not tokens, so the pipeline stays
+ * tokenizer-free (the embedder owns token counting). A character budget
+ * is deterministic and dep-free; it may occasionally over/under-shoot a
+ * provider's per-request token cap, which the sub-batching guard in
+ * `index()` absorbs.
+ */
+type ChunkOptions = {
+  /** Splitter strategy. Default `"recursive"`. */type?: ChunkType; /** Target chunk size in characters (not tokens, to stay tokenizer-free). Default 1000. */
+  size?: number; /** Character overlap carried between adjacent chunks. Default 200. */
+  overlap?: number;
+  /**
+   * Ordered separators for the `"recursive"` splitter, tried largest-unit
+   * first. Default `["\n\n", "\n", ". ", " ", ""]`.
+   */
+  separators?: string[];
+};
+/**
+ * A single chunk emitted by a splitter — its text plus the exact
+ * `[start, end)` character span inside the original source text, so the
+ * citation's `span` is precise.
+ */
+type Chunk = {
+  /** The chunk text. */text: string; /** 0-based index of the chunk within its source document. */
+  index: number; /** Character span `[start, end)` of the chunk inside the original source text. */
+  span: [start: number, end: number];
+};
+//#endregion
+export { Chunk, ChunkOptions, ChunkType };
+//# sourceMappingURL=chunk-options.type.d.mts.map

package/esm/rag/contracts/chunk-options.type.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-options.type.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/contracts/chunk-options.type.ts"],"mappings":";;KACY,SAAA;;;;AAAS;AAWrB;;;;;KAAY,YAAA;EAIV,gDAFA,IAAA,GAAO,SAAS,EAShB;EAPA,IAAA,WAOU;EALV,OAAA;EAae;;;;EARf,UAAA;AAAA;;;;AAcyB;;KANf,KAAA;wBAEV,IAAA;EAEA,KAAA;EAEA,IAAA,GAAO,KAAA,UAAe,GAAA;AAAA"}

package/esm/rag/contracts/citation.type.d.mts

@@ -0,0 +1,35 @@
+//#region ../@warlock.js/ai/src/rag/contracts/citation.type.d.ts
+/** Where a retrieved chunk came from — the unit an answer cites. */
+type Citation = {
+  /** `id` of the source `RagDocument` this chunk was split from. */sourceId: string; /** 0-based index of the chunk within its source document. */
+  chunkIndex: number; /** Character span `[start, end)` of the chunk inside the original source text. */
+  span: [start: number, end: number]; /** Relevance score the retrieval assigned, in `[0, 1]`. */
+  score: number; /** Metadata copied verbatim from the source document. */
+  metadata?: Record<string, unknown>;
+};
+/** A single retrieval hit, carrying its citation. */
+type RetrievedChunk = {
+  /** Chunk text injected into the model prompt. */text: string; /** Relevance in `[0, 1]` — cosine similarity, or the reranker's score when one ran. */
+  score: number; /** Provenance for grounding the answer. */
+  citation: Citation;
+};
+/** Knobs controlling a single `retrieve()` call. */
+type RetrieveOptions = {
+  /** Number of chunks to return AFTER reranking. Default 5. */topK?: number; /** Cosine floor `[0, 1]` applied at the vector-store stage. Default 0.5. */
+  threshold?: number;
+  /**
+   * Candidate pool size fetched from the store before reranking. Defaults
+   * to `topK * 4` (clamped to >= topK) — an overscan that mirrors the
+   * episodic/procedural memory tiers' `k * 5` pattern.
+   */
+  candidates?: number; /** Restrict retrieval to chunks whose source had one of these tags. */
+  tags?: string[];
+};
+/** Result of a single retrieval. */
+type RetrieveResult = {
+  /** The query that was embedded. */query: string; /** Ranked, cited chunks. */
+  chunks: RetrievedChunk[];
+};
+//#endregion
+export { Citation, RetrieveOptions, RetrieveResult, RetrievedChunk };
+//# sourceMappingURL=citation.type.d.mts.map

package/esm/rag/contracts/citation.type.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"citation.type.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/contracts/citation.type.ts"],"mappings":";;KACY,QAAA;EAAQ,kEAElB,QAAA,UAQiB;EANjB,UAAA;EAEA,IAAA,GAAO,KAAA,UAAe,GAAA,WAAf;EAEP,KAAA;EAEA,QAAA,GAAW,MAAM;AAAA;;KAIP,cAAA;EAAA,iDAEV,IAAA;EAEA,KAAA,UAFA;EAIA,QAAA,EAAU,QAAQ;AAAA;;KAIR,eAAA;EAJQ,6DAMlB,IAAA,WAFyB;EAIzB,SAAA;EAJyB;;;;;EAUzB,UAAA,WAEI;EAAJ,IAAA;AAAA;;KAIU,cAAA;EAEV,mCAAA,KAAA,UAEQ;EAAR,MAAA,EAAQ,cAAc;AAAA"}

package/esm/rag/contracts/index.d.mts

@@ -0,0 +1,4 @@
+import { Citation, RetrieveOptions, RetrieveResult, RetrievedChunk } from "./citation.type.mjs";
+import { Chunk, ChunkOptions, ChunkType } from "./chunk-options.type.mjs";
+import { RagDocument } from "./rag-document.type.mjs";
+import { Rag, RagAsToolOptions, RagConfig } from "./rag-config.type.mjs";

package/esm/rag/contracts/rag-config.type.d.mts

@@ -0,0 +1,68 @@
+import { ToolContract } from "../../tool/tool.mjs";
+import { EmbedderContract } from "../../contracts/embedder.contract.mjs";
+import { RetrieveOptions, RetrieveResult } from "./citation.type.mjs";
+import { RagReranker } from "../rerank/reranker.contract.mjs";
+import { ChunkOptions } from "./chunk-options.type.mjs";
+import { RagDocument } from "./rag-document.type.mjs";
+import { CacheDriver } from "@warlock.js/cache";
+
+//#region ../@warlock.js/ai/src/rag/contracts/rag-config.type.d.ts
+/** Options for `rag.asTool()` — how retrieval is exposed to an agent. */
+type RagAsToolOptions = {
+  /** LLM tool name. Default `"retrieve_<rag.name>"`. */name?: string; /** Tool description the model reads. Sensible default derived from the rag name. */
+  description?: string; /** Override `topK` / `threshold` / `tags` for the tool path. */
+  retrieve?: RetrieveOptions;
+};
+/** Configuration for the `rag()` factory. */
+type RagConfig = {
+  /** Stable name — used in tool names, cache namespace, logs. Default `"rag"`. */name?: string; /** Embedder for both indexing and query embedding. Required. */
+  embedder: EmbedderContract;
+  /**
+   * Vector-capable cache driver = the vector store. Falls back to
+   * `ai.config({ defaultStore })` when omitted; throws at construction if
+   * neither resolves (same rule as `SemanticMemoryConfig.store`).
+   */
+  store?: CacheDriver<any, any>; /** Namespace prefix for every key written. Default `"ai.rag." + name`. */
+  namespace?: string; /** Chunking defaults for `index()`. Overridable per `index()` call. */
+  chunk?: ChunkOptions; /** Optional reranker run between the store fetch and the topK slice. */
+  reranker?: RagReranker; /** Default retrieval knobs. */
+  retrieve?: RetrieveOptions;
+  /**
+   * Ingestion guardrails for `index()` (D5). Each is a hard cap that
+   * throws before any embedding spend when exceeded — unbounded ingestion
+   * is a real cost / memory / quota foot-gun. All optional; omit for no
+   * limit (the prior behavior).
+   */
+  limits?: {
+    /** Max documents accepted in a single `index()` call. */maxDocuments?: number; /** Max chunks produced across the call (after chunking). */
+    maxChunks?: number; /** Max total bytes of document text accepted in a single call. */
+    maxBytes?: number;
+  };
+};
+/**
+ * A configured RAG pipeline: chunk → embed → vector store → retrieve →
+ * rerank → cite. Built by the `rag()` factory.
+ */
+interface Rag {
+  /** Stable name used in tool names, namespace, and logs. */
+  readonly name: string;
+  /** Chunk → embed (batched) → store. Returns the chunk count written. */
+  index(docs: RagDocument[], chunk?: ChunkOptions): Promise<{
+    chunks: number;
+  }>;
+  /** Embed the query, fetch candidates, rerank, slice topK, attach citations. */
+  retrieve(query: string, options?: RetrieveOptions): Promise<RetrieveResult>;
+  /** Drop every entry written under this rag's namespace. */
+  clear(): Promise<void>;
+  /**
+   * Expose `retrieve()` as a tool for an agent's `tools: []` loop. Input is
+   * `{ query: string }`; output is `RetrieveResult`. Built via the same
+   * composite-as-tool engine the other primitives use.
+   */
+  asTool(options?: RagAsToolOptions): ToolContract<{
+    query: string;
+  }, RetrieveResult>;
+}
+//#endregion
+export { Rag, RagAsToolOptions, RagConfig };
+//# sourceMappingURL=rag-config.type.d.mts.map

package/esm/rag/contracts/rag-config.type.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rag-config.type.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/contracts/rag-config.type.ts"],"mappings":";;;;;;;;;;KASY,gBAAA;EAAA,sDAEV,IAAA;EAEA,WAAA,WAFA;EAIA,QAAA,GAAW,eAAe;AAAA;;KAIhB,SAAA;EAJgB,gFAM1B,IAAA,WAFmB;EAInB,QAAA,EAAU,gBAAA;EAAA;;;;;EAMV,KAAA,GAAQ,WAAA,YAQkB;EAN1B,SAAA,WARA;EAUA,KAAA,GAAQ,YAAA,EAJR;EAMA,QAAA,GAAW,WAAA,EAJX;EAMA,QAAA,GAAW,eAAA;EAJH;;;;;;EAWR,MAAA;IAIE,yDAFA,YAAA,WAIQ;IAFR,SAAA,WAUa;IARb,QAAA;EAAA;AAAA;;;;;UAQa,GAAA;EAQN;EAAA,SANA,IAAA;EAY2D;EAVpE,KAAA,CAAM,IAAA,EAAM,WAAA,IAAe,KAAA,GAAQ,YAAA,GAAe,OAAA;IAAU,MAAA;EAAA;EAFnD;EAIT,QAAA,CAAS,KAAA,UAAe,OAAA,GAAU,eAAA,GAAkB,OAAA,CAAQ,cAAA;EAFhD;EAIZ,KAAA,IAAS,OAAA;EAJ0B;;;;;EAUnC,MAAA,CAAO,OAAA,GAAU,gBAAA,GAAmB,YAAA;IAAe,KAAA;EAAA,GAAiB,cAAA;AAAA"}

package/esm/rag/contracts/rag-document.type.d.mts

@@ -0,0 +1,21 @@
+//#region ../@warlock.js/ai/src/rag/contracts/rag-document.type.d.ts
+/**
+ * A raw document handed to the pipeline before chunking.
+ *
+ * The caller is responsible for loading documents (from files, a DB, an
+ * API, …) and parsing them down to text — document loaders are out of
+ * scope for v1. `index()` takes already-loaded `{ id, text }` documents.
+ */
+type RagDocument = {
+  /** Stable source id — propagated to every chunk + citation from this doc. */id: string; /** Full text to be chunked + embedded. */
+  text: string; /** Opaque metadata round-tripped onto chunks + citations (url, title, page…). */
+  metadata?: Record<string, unknown>;
+  /**
+   * Optional tags applied to every chunk written from this document, so
+   * `retrieve({ tags })` can restrict retrieval to a subset of sources.
+   */
+  tags?: string[];
+};
+//#endregion
+export { RagDocument };
+//# sourceMappingURL=rag-document.type.d.mts.map

package/esm/rag/contracts/rag-document.type.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rag-document.type.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/contracts/rag-document.type.ts"],"mappings":";;AAOA;;;;;;KAAY,WAAA;EAMC,6EAJX,EAAA,UASI;EAPJ,IAAA;EAEA,QAAA,GAAW,MAAM;;;;;EAKjB,IAAA;AAAA"}

package/esm/rag/hybrid/bm25.d.mts

@@ -0,0 +1,23 @@
+import { RankedItem } from "./rrf.mjs";
+
+//#region ../@warlock.js/ai/src/rag/hybrid/bm25.d.ts
+/** A document to score lexically. */
+type LexicalDoc = {
+  id: string;
+  text: string;
+};
+/**
+ * Rank `docs` against `query` with BM25 (A4) — the lexical half of hybrid
+ * retrieval. Scores keyword overlap with TF saturation (`k1`) and length
+ * normalization (`b`) over the candidate set, so an exact-term match
+ * surfaces even when dense embeddings miss it. Returns docs sorted by
+ * score (highest first); zero-score docs are dropped.
+ *
+ * Operates over the supplied candidate set (typically the dense retriever's
+ * over-fetch), so it needs no global corpus index — ideal for fusing with
+ * a vector ranking via {@link reciprocalRankFusion}.
+ */
+declare function bm25Rank(query: string, docs: ReadonlyArray<LexicalDoc>): RankedItem[];
+//#endregion
+export { LexicalDoc, bm25Rank };
+//# sourceMappingURL=bm25.d.mts.map

package/esm/rag/hybrid/bm25.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/hybrid/bm25.ts"],"mappings":";;;;KAGY,UAAA;EAAe,EAAA;EAAY,IAAI;AAAA;;AAAA;AAwB3C;;;;;;;;;iBAAgB,QAAA,CAAS,KAAA,UAAe,IAAA,EAAM,aAAA,CAAc,UAAA,IAAc,UAAA"}

package/esm/rag/hybrid/bm25.mjs

@@ -0,0 +1,51 @@
+//#region ../@warlock.js/ai/src/rag/hybrid/bm25.ts
+const BM25_K1 = 1.5;
+const BM25_B = .75;
+/** Lowercase + split on non-word characters; drop empties. */
+function tokenize(text) {
+	return text.toLowerCase().split(/[^a-z0-9]+/i).filter(Boolean);
+}
+/**
+* Rank `docs` against `query` with BM25 (A4) — the lexical half of hybrid
+* retrieval. Scores keyword overlap with TF saturation (`k1`) and length
+* normalization (`b`) over the candidate set, so an exact-term match
+* surfaces even when dense embeddings miss it. Returns docs sorted by
+* score (highest first); zero-score docs are dropped.
+*
+* Operates over the supplied candidate set (typically the dense retriever's
+* over-fetch), so it needs no global corpus index — ideal for fusing with
+* a vector ranking via {@link reciprocalRankFusion}.
+*/
+function bm25Rank(query, docs) {
+	const queryTerms = [...new Set(tokenize(query))];
+	if (queryTerms.length === 0 || docs.length === 0) return [];
+	const tokenized = docs.map((doc) => ({
+		id: doc.id,
+		terms: tokenize(doc.text)
+	}));
+	const avgLen = tokenized.reduce((sum, d) => sum + d.terms.length, 0) / tokenized.length || 1;
+	const df = /* @__PURE__ */ new Map();
+	for (const term of queryTerms) df.set(term, tokenized.filter((d) => d.terms.includes(term)).length);
+	const n = tokenized.length;
+	return tokenized.map((doc) => {
+		const len = doc.terms.length || 1;
+		let score = 0;
+		for (const term of queryTerms) {
+			const tf = doc.terms.filter((t) => t === term).length;
+			if (tf === 0) continue;
+			const docFreq = df.get(term) ?? 0;
+			const idf = Math.log(1 + (n - docFreq + .5) / (docFreq + .5));
+			const numerator = tf * 2.5;
+			const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * (len / avgLen));
+			score += idf * (numerator / denominator);
+		}
+		return {
+			id: doc.id,
+			score
+		};
+	}).filter((item) => item.score > 0).sort((a, b) => b.score - a.score);
+}
+
+//#endregion
+export { bm25Rank };
+//# sourceMappingURL=bm25.mjs.map

package/esm/rag/hybrid/bm25.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.mjs","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/hybrid/bm25.ts"],"sourcesContent":["import type { RankedItem } from \"./rrf\";\n\n/** A document to score lexically. */\nexport type LexicalDoc = { id: string; text: string };\n\nconst BM25_K1 = 1.5;\nconst BM25_B = 0.75;\n\n/** Lowercase + split on non-word characters; drop empties. */\nfunction tokenize(text: string): string[] {\n  return text\n    .toLowerCase()\n    .split(/[^a-z0-9]+/i)\n    .filter(Boolean);\n}\n\n/**\n * Rank `docs` against `query` with BM25 (A4) — the lexical half of hybrid\n * retrieval. Scores keyword overlap with TF saturation (`k1`) and length\n * normalization (`b`) over the candidate set, so an exact-term match\n * surfaces even when dense embeddings miss it. Returns docs sorted by\n * score (highest first); zero-score docs are dropped.\n *\n * Operates over the supplied candidate set (typically the dense retriever's\n * over-fetch), so it needs no global corpus index — ideal for fusing with\n * a vector ranking via {@link reciprocalRankFusion}.\n */\nexport function bm25Rank(query: string, docs: ReadonlyArray<LexicalDoc>): RankedItem[] {\n  const queryTerms = [...new Set(tokenize(query))];\n  if (queryTerms.length === 0 || docs.length === 0) return [];\n\n  const tokenized = docs.map(doc => ({ id: doc.id, terms: tokenize(doc.text) }));\n  const avgLen =\n    tokenized.reduce((sum, d) => sum + d.terms.length, 0) / tokenized.length || 1;\n\n  // Document frequency per query term, across the candidate set.\n  const df = new Map<string, number>();\n  for (const term of queryTerms) {\n    df.set(\n      term,\n      tokenized.filter(d => d.terms.includes(term)).length,\n    );\n  }\n\n  const n = tokenized.length;\n\n  const scored = tokenized.map(doc => {\n    const len = doc.terms.length || 1;\n    let score = 0;\n\n    for (const term of queryTerms) {\n      const tf = doc.terms.filter(t => t === term).length;\n      if (tf === 0) continue;\n\n      const docFreq = df.get(term) ?? 0;\n      // BM25 idf (with the +1 to keep it non-negative).\n      const idf = Math.log(1 + (n - docFreq + 0.5) / (docFreq + 0.5));\n      const numerator = tf * (BM25_K1 + 1);\n      const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * (len / avgLen));\n      score += idf * (numerator / denominator);\n    }\n\n    return { id: doc.id, score };\n  });\n\n  return scored.filter(item => item.score > 0).sort((a, b) => b.score - a.score);\n}\n"],"mappings":";AAKA,MAAM,UAAU;AAChB,MAAM,SAAS;;AAGf,SAAS,SAAS,MAAwB;CACxC,OAAO,KACJ,YAAY,CAAC,CACb,MAAM,aAAa,CAAC,CACpB,OAAO,OAAO;AACnB;;;;;;;;;;;;AAaA,SAAgB,SAAS,OAAe,MAA+C;CACrF,MAAM,aAAa,CAAC,GAAG,IAAI,IAAI,SAAS,KAAK,CAAC,CAAC;CAC/C,IAAI,WAAW,WAAW,KAAK,KAAK,WAAW,GAAG,OAAO,CAAC;CAE1D,MAAM,YAAY,KAAK,KAAI,SAAQ;EAAE,IAAI,IAAI;EAAI,OAAO,SAAS,IAAI,IAAI;CAAE,EAAE;CAC7E,MAAM,SACJ,UAAU,QAAQ,KAAK,MAAM,MAAM,EAAE,MAAM,QAAQ,CAAC,IAAI,UAAU,UAAU;CAG9E,MAAM,qBAAK,IAAI,IAAoB;CACnC,KAAK,MAAM,QAAQ,YACjB,GAAG,IACD,MACA,UAAU,QAAO,MAAK,EAAE,MAAM,SAAS,IAAI,CAAC,CAAC,CAAC,MAChD;CAGF,MAAM,IAAI,UAAU;CAqBpB,OAnBe,UAAU,KAAI,QAAO;EAClC,MAAM,MAAM,IAAI,MAAM,UAAU;EAChC,IAAI,QAAQ;EAEZ,KAAK,MAAM,QAAQ,YAAY;GAC7B,MAAM,KAAK,IAAI,MAAM,QAAO,MAAK,MAAM,IAAI,CAAC,CAAC;GAC7C,IAAI,OAAO,GAAG;GAEd,MAAM,UAAU,GAAG,IAAI,IAAI,KAAK;GAEhC,MAAM,MAAM,KAAK,IAAI,KAAK,IAAI,UAAU,OAAQ,UAAU,GAAI;GAC9D,MAAM,YAAY,KAAM;GACxB,MAAM,cAAc,KAAK,WAAW,IAAI,SAAS,UAAU,MAAM;GACjE,SAAS,OAAO,YAAY;EAC9B;EAEA,OAAO;GAAE,IAAI,IAAI;GAAI;EAAM;CAC7B,CAEY,CAAC,CAAC,QAAO,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAC/E"}

package/esm/rag/hybrid/hybrid-rank.d.mts

@@ -0,0 +1,33 @@
+import { RankedItem } from "./rrf.mjs";
+import { LexicalDoc } from "./bm25.mjs";
+
+//#region ../@warlock.js/ai/src/rag/hybrid/hybrid-rank.d.ts
+/**
+ * Hybrid rank (A4) — fuse a dense (vector) ranking with a BM25 lexical
+ * ranking over the same candidate set via Reciprocal Rank Fusion. Dense
+ * retrieval captures semantic similarity; BM25 captures exact-term
+ * matches dense embeddings miss (names, ids, rare tokens). Fusing both
+ * beats either alone for keyword-heavy queries.
+ *
+ * `dense` is the vector retriever's result in rank order; `candidates`
+ * supplies the text for the lexical pass (typically the same over-fetched
+ * set). Returns the fused ranking, highest score first.
+ *
+ * @example
+ * const fused = hybridRank({
+ *   query: "invoice 8842 refund",
+ *   dense: vectorHits,                 // [{ id }, ...] in similarity order
+ *   candidates: vectorHits.map(h => ({ id: h.id, text: h.text })),
+ * });
+ */
+declare function hybridRank(params: {
+  query: string;
+  dense: ReadonlyArray<{
+    id: string;
+  }>;
+  candidates: ReadonlyArray<LexicalDoc>;
+  k?: number;
+}): RankedItem[];
+//#endregion
+export { hybridRank };
+//# sourceMappingURL=hybrid-rank.d.mts.map

package/esm/rag/hybrid/hybrid-rank.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-rank.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/hybrid/hybrid-rank.ts"],"mappings":";;;;;;AAqBA;;;;;;;;;;;;;;;;iBAAgB,UAAA,CAAW,MAAA;EACzB,KAAA;EACA,KAAA,EAAO,aAAA;IAAgB,EAAA;EAAA;EACvB,UAAA,EAAY,aAAA,CAAc,UAAA;EAC1B,CAAA;AAAA,IACE,UAAA"}

package/esm/rag/hybrid/hybrid-rank.mjs

@@ -0,0 +1,29 @@
+import { reciprocalRankFusion } from "./rrf.mjs";
+import { bm25Rank } from "./bm25.mjs";
+
+//#region ../@warlock.js/ai/src/rag/hybrid/hybrid-rank.ts
+/**
+* Hybrid rank (A4) — fuse a dense (vector) ranking with a BM25 lexical
+* ranking over the same candidate set via Reciprocal Rank Fusion. Dense
+* retrieval captures semantic similarity; BM25 captures exact-term
+* matches dense embeddings miss (names, ids, rare tokens). Fusing both
+* beats either alone for keyword-heavy queries.
+*
+* `dense` is the vector retriever's result in rank order; `candidates`
+* supplies the text for the lexical pass (typically the same over-fetched
+* set). Returns the fused ranking, highest score first.
+*
+* @example
+* const fused = hybridRank({
+*   query: "invoice 8842 refund",
+*   dense: vectorHits,                 // [{ id }, ...] in similarity order
+*   candidates: vectorHits.map(h => ({ id: h.id, text: h.text })),
+* });
+*/
+function hybridRank(params) {
+	return reciprocalRankFusion([params.dense.map((d) => d.id), bm25Rank(params.query, params.candidates).map((r) => r.id)], params.k);
+}
+
+//#endregion
+export { hybridRank };
+//# sourceMappingURL=hybrid-rank.mjs.map

package/esm/rag/hybrid/hybrid-rank.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-rank.mjs","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/hybrid/hybrid-rank.ts"],"sourcesContent":["import { bm25Rank, type LexicalDoc } from \"./bm25\";\nimport { reciprocalRankFusion, type RankedItem } from \"./rrf\";\n\n/**\n * Hybrid rank (A4) — fuse a dense (vector) ranking with a BM25 lexical\n * ranking over the same candidate set via Reciprocal Rank Fusion. Dense\n * retrieval captures semantic similarity; BM25 captures exact-term\n * matches dense embeddings miss (names, ids, rare tokens). Fusing both\n * beats either alone for keyword-heavy queries.\n *\n * `dense` is the vector retriever's result in rank order; `candidates`\n * supplies the text for the lexical pass (typically the same over-fetched\n * set). Returns the fused ranking, highest score first.\n *\n * @example\n * const fused = hybridRank({\n *   query: \"invoice 8842 refund\",\n *   dense: vectorHits,                 // [{ id }, ...] in similarity order\n *   candidates: vectorHits.map(h => ({ id: h.id, text: h.text })),\n * });\n */\nexport function hybridRank(params: {\n  query: string;\n  dense: ReadonlyArray<{ id: string }>;\n  candidates: ReadonlyArray<LexicalDoc>;\n  k?: number;\n}): RankedItem[] {\n  const denseIds = params.dense.map(d => d.id);\n  const lexicalIds = bm25Rank(params.query, params.candidates).map(r => r.id);\n\n  return reciprocalRankFusion([denseIds, lexicalIds], params.k);\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,WAAW,QAKV;CAIf,OAAO,qBAAqB,CAHX,OAAO,MAAM,KAAI,MAAK,EAAE,EAGL,GAFjB,SAAS,OAAO,OAAO,OAAO,UAAU,CAAC,CAAC,KAAI,MAAK,EAAE,EAExB,CAAC,GAAG,OAAO,CAAC;AAC9D"}

package/esm/rag/hybrid/rrf.d.mts

@@ -0,0 +1,25 @@
+//#region ../@warlock.js/ai/src/rag/hybrid/rrf.d.ts
+/** One item's id paired with a fused relevance score. */
+type RankedItem = {
+  id: string;
+  score: number;
+};
+/**
+ * Reciprocal Rank Fusion (A4) — combine several independently-ranked
+ * lists of ids into one consensus ranking. Each list contributes
+ * `1 / (k + rank)` to an id's score (rank is 0-based within that list), so
+ * an id near the top of multiple lists rises even if no single list ranks
+ * it first. The classic fusion for hybrid (dense + lexical) retrieval
+ * because it needs no score calibration between the lists.
+ *
+ * `k` (default 60, the standard) dampens the contribution of lower ranks.
+ * Returns ids sorted by fused score, highest first.
+ *
+ * @example
+ * reciprocalRankFusion([["a", "b", "c"], ["b", "a"]]);
+ * // → [{ id: "b", ... }, { id: "a", ... }, { id: "c", ... }]
+ */
+declare function reciprocalRankFusion(rankedLists: ReadonlyArray<ReadonlyArray<string>>, k?: number): RankedItem[];
+//#endregion
+export { RankedItem, reciprocalRankFusion };
+//# sourceMappingURL=rrf.d.mts.map

package/esm/rag/hybrid/rrf.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rrf.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/hybrid/rrf.ts"],"mappings":";;KACY,UAAA;EAAe,EAAA;EAAY,KAAK;AAAA;AAAA;AAiB5C;;;;;;;;;;;;;;AAjB4C,iBAiB5B,oBAAA,CACd,WAAA,EAAa,aAAA,CAAc,aAAA,WAC3B,CAAA,YACC,UAAA"}

package/esm/rag/hybrid/rrf.mjs

@@ -0,0 +1,30 @@
+//#region ../@warlock.js/ai/src/rag/hybrid/rrf.ts
+/**
+* Reciprocal Rank Fusion (A4) — combine several independently-ranked
+* lists of ids into one consensus ranking. Each list contributes
+* `1 / (k + rank)` to an id's score (rank is 0-based within that list), so
+* an id near the top of multiple lists rises even if no single list ranks
+* it first. The classic fusion for hybrid (dense + lexical) retrieval
+* because it needs no score calibration between the lists.
+*
+* `k` (default 60, the standard) dampens the contribution of lower ranks.
+* Returns ids sorted by fused score, highest first.
+*
+* @example
+* reciprocalRankFusion([["a", "b", "c"], ["b", "a"]]);
+* // → [{ id: "b", ... }, { id: "a", ... }, { id: "c", ... }]
+*/
+function reciprocalRankFusion(rankedLists, k = 60) {
+	const scores = /* @__PURE__ */ new Map();
+	for (const list of rankedLists) list.forEach((id, rank) => {
+		scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank));
+	});
+	return [...scores.entries()].map(([id, score]) => ({
+		id,
+		score
+	})).sort((a, b) => b.score - a.score);
+}
+
+//#endregion
+export { reciprocalRankFusion };
+//# sourceMappingURL=rrf.mjs.map

package/esm/rag/hybrid/rrf.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"rrf.mjs","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/rag/hybrid/rrf.ts"],"sourcesContent":["/** One item's id paired with a fused relevance score. */\nexport type RankedItem = { id: string; score: number };\n\n/**\n * Reciprocal Rank Fusion (A4) — combine several independently-ranked\n * lists of ids into one consensus ranking. Each list contributes\n * `1 / (k + rank)` to an id's score (rank is 0-based within that list), so\n * an id near the top of multiple lists rises even if no single list ranks\n * it first. The classic fusion for hybrid (dense + lexical) retrieval\n * because it needs no score calibration between the lists.\n *\n * `k` (default 60, the standard) dampens the contribution of lower ranks.\n * Returns ids sorted by fused score, highest first.\n *\n * @example\n * reciprocalRankFusion([[\"a\", \"b\", \"c\"], [\"b\", \"a\"]]);\n * // → [{ id: \"b\", ... }, { id: \"a\", ... }, { id: \"c\", ... }]\n */\nexport function reciprocalRankFusion(\n  rankedLists: ReadonlyArray<ReadonlyArray<string>>,\n  k = 60,\n): RankedItem[] {\n  const scores = new Map<string, number>();\n\n  for (const list of rankedLists) {\n    list.forEach((id, rank) => {\n      scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank));\n    });\n  }\n\n  return [...scores.entries()]\n    .map(([id, score]) => ({ id, score }))\n    .sort((a, b) => b.score - a.score);\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAkBA,SAAgB,qBACd,aACA,IAAI,IACU;CACd,MAAM,yBAAS,IAAI,IAAoB;CAEvC,KAAK,MAAM,QAAQ,aACjB,KAAK,SAAS,IAAI,SAAS;EACzB,OAAO,IAAI,KAAK,OAAO,IAAI,EAAE,KAAK,KAAK,KAAK,IAAI,KAAK;CACvD,CAAC;CAGH,OAAO,CAAC,GAAG,OAAO,QAAQ,CAAC,CAAC,CACzB,KAAK,CAAC,IAAI,YAAY;EAAE;EAAI;CAAM,EAAE,CAAC,CACrC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACrC"}

package/esm/rag/index.d.mts

@@ -0,0 +1,15 @@
+import { Citation, RetrieveOptions, RetrieveResult, RetrievedChunk } from "./contracts/citation.type.mjs";
+import { RagReranker } from "./rerank/reranker.contract.mjs";
+import { Chunk, ChunkOptions, ChunkType } from "./contracts/chunk-options.type.mjs";
+import { RagDocument } from "./contracts/rag-document.type.mjs";
+import { Rag, RagAsToolOptions, RagConfig } from "./contracts/rag-config.type.mjs";
+import { rag } from "./rag.mjs";
+import { chunk } from "./chunk/chunk.mjs";
+import { VectorStore } from "./store/vector-store.contract.mjs";
+import { cacheVectorStore } from "./store/cache-vector-store.mjs";
+import { KeywordRerankerOptions, keywordReranker } from "./rerank/keyword-reranker.mjs";
+import { LlmRerankerOptions, llmReranker } from "./rerank/llm-reranker.mjs";
+import { RankedItem, reciprocalRankFusion } from "./hybrid/rrf.mjs";
+import { LexicalDoc, bm25Rank } from "./hybrid/bm25.mjs";
+import { hybridRank } from "./hybrid/hybrid-rank.mjs";
+import { MultiQueryOptions, multiQuery } from "./transforms/multi-query.mjs";

package/esm/rag/index.mjs

@@ -0,0 +1,11 @@
+import { chunk } from "./chunk/chunk.mjs";
+import { cacheVectorStore } from "./store/cache-vector-store.mjs";
+import { rag } from "./rag.mjs";
+import { keywordReranker } from "./rerank/keyword-reranker.mjs";
+import { llmReranker } from "./rerank/llm-reranker.mjs";
+import { reciprocalRankFusion } from "./hybrid/rrf.mjs";
+import { bm25Rank } from "./hybrid/bm25.mjs";
+import { hybridRank } from "./hybrid/hybrid-rank.mjs";
+import { multiQuery } from "./transforms/multi-query.mjs";
+
+export {  };

package/esm/rag/rag.d.mts

@@ -0,0 +1,38 @@
+import { Rag, RagConfig } from "./contracts/rag-config.type.mjs";
+
+//#region ../@warlock.js/ai/src/rag/rag.d.ts
+/**
+ * Create a RAG pipeline: **chunk → embed → vector store → retrieve →
+ * rerank → cite**, reusing the app's `ai.embedder` for embedding, a
+ * `@warlock.js/cache` `CacheDriver` as the vector store, and the
+ * composite-as-tool engine to expose retrieval as a tool.
+ *
+ * Resolution is loud at construction (mirroring `memory()`):
+ * - `embedder` is required — a provider with no embedder must be caught
+ *   here, not at first index.
+ * - `store` falls back to `ai.config({ defaultStore })`; if neither
+ *   resolves, construction throws.
+ *
+ * `retrieve()` is return-only — it never auto-injects into a prompt; the
+ * caller formats the cited chunks (or uses `asTool()` for the agent loop).
+ * The reranker is OFF by default (cosine-only) unless `config.reranker`
+ * is set.
+ *
+ * @example
+ * import { ai } from "@warlock.js/ai";
+ * import { MemoryCacheDriver } from "@warlock.js/cache";
+ *
+ * const kb = ai.rag({
+ *   name: "docs",
+ *   embedder: openai.embedder({ name: "text-embedding-3-small" }),
+ *   store: new MemoryCacheDriver(),
+ *   chunk: { type: "markdown", size: 800, overlap: 120 },
+ * });
+ *
+ * await kb.index([{ id: "guide", text: longMarkdown, metadata: { url: "/guide" } }]);
+ * const { chunks } = await kb.retrieve("how do I configure caching?", { topK: 4 });
+ */
+declare function rag(config: RagConfig): Rag;
+//#endregion
+export { rag };
+//# sourceMappingURL=rag.d.mts.map

package/esm/rag/rag.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rag.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/rag/rag.ts"],"mappings":";;;;;AAyDA;;;;;;;;AAA2C;;;;;;;;;;;;;;;;;;;;;iBAA3B,GAAA,CAAI,MAAA,EAAQ,SAAA,GAAY,GAAG"}