@warlock.js/core 4.5.0 → 4.6.1

package/esm/ai/src/planner/snapshot.mjs

@@ -0,0 +1,95 @@
+import { PlannerFailedError } from "../errors/planner-failed-error.mjs";
+import { PlannerDriftError } from "../errors/planner-drift-error.mjs";
+import "../errors/index.mjs";
+import { resolveDefaultSnapshotStore } from "../config.mjs";
+
+//#region ../@warlock.js/ai/src/planner/snapshot.ts
+/**
+* Resolve the effective {@link SnapshotStore}: the planner's own
+* `durable.store` wins; absent that, fall back to the global default set
+* via `ai.config({ defaultSnapshotStore })`.
+*
+* The global default is typed for the supervisor snapshot shape, but
+* every store impl keys purely by `runId` and round-trips whatever
+* envelope it is handed — so it serves a `PlannerSnapshot` just as well.
+* The cast re-tags the shape at this single boundary (Option B); the
+* planner only ever hands it a `PlannerSnapshot`.
+*/
+function resolveSnapshotStore(durable) {
+	return durable?.store ?? resolveDefaultSnapshotStore();
+}
+/**
+* Write the current run state to the resolved snapshot store. No-op
+* (returns `{ ok: true }`) when neither `durable.store` nor the global
+* `defaultSnapshotStore` is configured — the common non-durable path.
+* Failures are returned as `{ ok: false }` rather than thrown so the
+* engine can surface them via logs without aborting the run.
+*/
+async function persistPlannerSnapshot(params) {
+	const store = resolveSnapshotStore(params.durable);
+	if (!store) return { ok: true };
+	const snapshot = {
+		runId: params.runId,
+		plannerName: params.plannerName,
+		signature: params.signature,
+		version: params.version,
+		goal: params.goal,
+		plan: params.plan,
+		executedSteps: params.executedSteps,
+		usage: params.usage,
+		children: params.children,
+		replanCount: params.replanCount,
+		status: params.status,
+		startedAt: params.startedAt,
+		savedAt: (/* @__PURE__ */ new Date()).toISOString()
+	};
+	try {
+		await store.save(snapshot);
+		return { ok: true };
+	} catch (error) {
+		return {
+			ok: false,
+			error
+		};
+	}
+}
+/**
+* Delete a persisted snapshot — used after a successful run when
+* `durable.deleteOnComplete` is set. Never throws. No-op (ok) when no
+* store is configured.
+*/
+async function deletePlannerSnapshot(params) {
+	const store = resolveSnapshotStore(params.durable);
+	if (!store) return { ok: true };
+	try {
+		await store.delete(params.runId);
+		return { ok: true };
+	} catch (error) {
+		return {
+			ok: false,
+			error
+		};
+	}
+}
+/**
+* Load a persisted snapshot for `resume()` and run the drift check.
+* Throws `PlannerFailedError` when no store is configured or when the run
+* is missing; throws `PlannerDriftError` when the stored signature
+* doesn't match the current definition (unless `force` is set).
+*/
+async function loadPlannerSnapshotForResume(params) {
+	const store = resolveSnapshotStore(params.durable);
+	if (!store) throw new PlannerFailedError(`ai.planner("${params.plannerName}"): no durable store configured — set \`durable: { store }\` on the config or call \`ai.config({ defaultSnapshotStore })\` at boot before calling resume()`, { context: { runId: params.runId } });
+	const snapshot = await store.load(params.runId) ?? null;
+	if (!snapshot) throw new PlannerFailedError(`ai.planner("${params.plannerName}"): no snapshot for runId "${params.runId}"`, { context: { runId: params.runId } });
+	if (!params.options?.force && snapshot.signature !== params.signature) throw new PlannerDriftError(`ai.planner("${params.plannerName}") signature drift on resume`, {
+		savedSignature: snapshot.signature,
+		currentSignature: params.signature,
+		runId: params.runId
+	});
+	return snapshot;
+}
+
+//#endregion
+export { deletePlannerSnapshot, loadPlannerSnapshotForResume, persistPlannerSnapshot };
+//# sourceMappingURL=snapshot.mjs.map

package/esm/ai/src/planner/snapshot.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot.mjs","names":[],"sources":["../../../../../../../../../@warlock.js/ai/src/planner/snapshot.ts"],"sourcesContent":["import { resolveDefaultSnapshotStore } from \"../config\";\nimport type { SnapshotStore } from \"../contracts/orchestrator/snapshot-store.contract\";\nimport type { PlannerResumeOptions } from \"../contracts/planner/planner-execute-options.type\";\nimport type { PlannerPlan } from \"../contracts/planner/planner-plan.type\";\nimport type { PlannerStepSnapshot } from \"../contracts/planner/planner-result.type\";\nimport type {\n  PlannerSnapshot,\n  PlannerSnapshotStatus,\n} from \"../contracts/planner/planner-snapshot.type\";\nimport type { BaseReport } from \"../contracts/result/base-report.type\";\nimport type { Usage } from \"../contracts/result/usage.type\";\nimport { PlannerDriftError, PlannerFailedError } from \"../errors\";\n\n/**\n * The planner's `durable` config, narrowed to the fields the snapshot\n * helpers read.\n */\nexport type PlannerDurableConfig = {\n  store?: SnapshotStore<PlannerSnapshot>;\n  deleteOnComplete?: boolean;\n};\n\n/**\n * Resolve the effective {@link SnapshotStore}: the planner's own\n * `durable.store` wins; absent that, fall back to the global default set\n * via `ai.config({ defaultSnapshotStore })`.\n *\n * The global default is typed for the supervisor snapshot shape, but\n * every store impl keys purely by `runId` and round-trips whatever\n * envelope it is handed — so it serves a `PlannerSnapshot` just as well.\n * The cast re-tags the shape at this single boundary (Option B); the\n * planner only ever hands it a `PlannerSnapshot`.\n */\nfunction resolveSnapshotStore(\n  durable: PlannerDurableConfig | undefined,\n): SnapshotStore<PlannerSnapshot> | undefined {\n  return (\n    durable?.store ??\n    (resolveDefaultSnapshotStore() as SnapshotStore<PlannerSnapshot> | undefined)\n  );\n}\n\nexport type PersistPlannerParams = {\n  durable: PlannerDurableConfig | undefined;\n  runId: string;\n  plannerName: string;\n  signature: string;\n  version?: string;\n  goal: string;\n  plan: PlannerPlan;\n  executedSteps: PlannerStepSnapshot[];\n  usage: Usage;\n  children: BaseReport[];\n  replanCount: number;\n  status: PlannerSnapshotStatus;\n  startedAt: string;\n};\n\nexport type PersistOutcome = { ok: true } | { ok: false; error: unknown };\n\n/**\n * Write the current run state to the resolved snapshot store. No-op\n * (returns `{ ok: true }`) when neither `durable.store` nor the global\n * `defaultSnapshotStore` is configured — the common non-durable path.\n * Failures are returned as `{ ok: false }` rather than thrown so the\n * engine can surface them via logs without aborting the run.\n */\nexport async function persistPlannerSnapshot(\n  params: PersistPlannerParams,\n): Promise<PersistOutcome> {\n  const store = resolveSnapshotStore(params.durable);\n\n  if (!store) {\n    return { ok: true };\n  }\n\n  const snapshot: PlannerSnapshot = {\n    runId: params.runId,\n    plannerName: params.plannerName,\n    signature: params.signature,\n    version: params.version,\n    goal: params.goal,\n    plan: params.plan,\n    executedSteps: params.executedSteps,\n    usage: params.usage,\n    children: params.children,\n    replanCount: params.replanCount,\n    status: params.status,\n    startedAt: params.startedAt,\n    savedAt: new Date().toISOString(),\n  };\n\n  try {\n    await store.save(snapshot);\n\n    return { ok: true };\n  } catch (error) {\n    return { ok: false, error };\n  }\n}\n\n/**\n * Delete a persisted snapshot — used after a successful run when\n * `durable.deleteOnComplete` is set. Never throws. No-op (ok) when no\n * store is configured.\n */\nexport async function deletePlannerSnapshot(params: {\n  durable: PlannerDurableConfig | undefined;\n  runId: string;\n}): Promise<PersistOutcome> {\n  const store = resolveSnapshotStore(params.durable);\n\n  if (!store) {\n    return { ok: true };\n  }\n\n  try {\n    await store.delete(params.runId);\n\n    return { ok: true };\n  } catch (error) {\n    return { ok: false, error };\n  }\n}\n\n/**\n * Load a persisted snapshot for `resume()` and run the drift check.\n * Throws `PlannerFailedError` when no store is configured or when the run\n * is missing; throws `PlannerDriftError` when the stored signature\n * doesn't match the current definition (unless `force` is set).\n */\nexport async function loadPlannerSnapshotForResume(params: {\n  durable: PlannerDurableConfig | undefined;\n  plannerName: string;\n  signature: string;\n  runId: string;\n  options?: PlannerResumeOptions<unknown>;\n}): Promise<PlannerSnapshot> {\n  const store = resolveSnapshotStore(params.durable);\n\n  if (!store) {\n    throw new PlannerFailedError(\n      `ai.planner(\"${params.plannerName}\"): no durable store configured — set \\`durable: { store }\\` on the config or call \\`ai.config({ defaultSnapshotStore })\\` at boot before calling resume()`,\n      { context: { runId: params.runId } },\n    );\n  }\n\n  const snapshot = (await store.load(params.runId)) ?? null;\n\n  if (!snapshot) {\n    throw new PlannerFailedError(\n      `ai.planner(\"${params.plannerName}\"): no snapshot for runId \"${params.runId}\"`,\n      { context: { runId: params.runId } },\n    );\n  }\n\n  if (!params.options?.force && snapshot.signature !== params.signature) {\n    throw new PlannerDriftError(\n      `ai.planner(\"${params.plannerName}\") signature drift on resume`,\n      {\n        savedSignature: snapshot.signature,\n        currentSignature: params.signature,\n        runId: params.runId,\n      },\n    );\n  }\n\n  return snapshot;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAiCA,SAAS,qBACP,SAC4C;CAC5C,OACE,SAAS,SACR,4BAA4B;AAEjC;;;;;;;;AA2BA,eAAsB,uBACpB,QACyB;CACzB,MAAM,QAAQ,qBAAqB,OAAO,OAAO;CAEjD,IAAI,CAAC,OACH,OAAO,EAAE,IAAI,KAAK;CAGpB,MAAM,WAA4B;EAChC,OAAO,OAAO;EACd,aAAa,OAAO;EACpB,WAAW,OAAO;EAClB,SAAS,OAAO;EAChB,MAAM,OAAO;EACb,MAAM,OAAO;EACb,eAAe,OAAO;EACtB,OAAO,OAAO;EACd,UAAU,OAAO;EACjB,aAAa,OAAO;EACpB,QAAQ,OAAO;EACf,WAAW,OAAO;EAClB,0BAAS,IAAI,KAAK,EAAC,CAAC,YAAY;CAClC;CAEA,IAAI;EACF,MAAM,MAAM,KAAK,QAAQ;EAEzB,OAAO,EAAE,IAAI,KAAK;CACpB,SAAS,OAAO;EACd,OAAO;GAAE,IAAI;GAAO;EAAM;CAC5B;AACF;;;;;;AAOA,eAAsB,sBAAsB,QAGhB;CAC1B,MAAM,QAAQ,qBAAqB,OAAO,OAAO;CAEjD,IAAI,CAAC,OACH,OAAO,EAAE,IAAI,KAAK;CAGpB,IAAI;EACF,MAAM,MAAM,OAAO,OAAO,KAAK;EAE/B,OAAO,EAAE,IAAI,KAAK;CACpB,SAAS,OAAO;EACd,OAAO;GAAE,IAAI;GAAO;EAAM;CAC5B;AACF;;;;;;;AAQA,eAAsB,6BAA6B,QAMtB;CAC3B,MAAM,QAAQ,qBAAqB,OAAO,OAAO;CAEjD,IAAI,CAAC,OACH,MAAM,IAAI,mBACR,eAAe,OAAO,YAAY,6JAClC,EAAE,SAAS,EAAE,OAAO,OAAO,MAAM,EAAE,CACrC;CAGF,MAAM,WAAY,MAAM,MAAM,KAAK,OAAO,KAAK,KAAM;CAErD,IAAI,CAAC,UACH,MAAM,IAAI,mBACR,eAAe,OAAO,YAAY,6BAA6B,OAAO,MAAM,IAC5E,EAAE,SAAS,EAAE,OAAO,OAAO,MAAM,EAAE,CACrC;CAGF,IAAI,CAAC,OAAO,SAAS,SAAS,SAAS,cAAc,OAAO,WAC1D,MAAM,IAAI,kBACR,eAAe,OAAO,YAAY,+BAClC;EACE,gBAAgB,SAAS;EACzB,kBAAkB,OAAO;EACzB,OAAO,OAAO;CAChB,CACF;CAGF,OAAO;AACT"}

package/esm/ai/src/rag/hybrid/bm25.mjs

@@ -0,0 +1,51 @@
+//#region ../@warlock.js/ai/src/rag/hybrid/bm25.ts
+const BM25_K1 = 1.5;
+const BM25_B = .75;
+/** Lowercase + split on non-word characters; drop empties. */
+function tokenize(text) {
+	return text.toLowerCase().split(/[^a-z0-9]+/i).filter(Boolean);
+}
+/**
+* Rank `docs` against `query` with BM25 (A4) — the lexical half of hybrid
+* retrieval. Scores keyword overlap with TF saturation (`k1`) and length
+* normalization (`b`) over the candidate set, so an exact-term match
+* surfaces even when dense embeddings miss it. Returns docs sorted by
+* score (highest first); zero-score docs are dropped.
+*
+* Operates over the supplied candidate set (typically the dense retriever's
+* over-fetch), so it needs no global corpus index — ideal for fusing with
+* a vector ranking via {@link reciprocalRankFusion}.
+*/
+function bm25Rank(query, docs) {
+	const queryTerms = [...new Set(tokenize(query))];
+	if (queryTerms.length === 0 || docs.length === 0) return [];
+	const tokenized = docs.map((doc) => ({
+		id: doc.id,
+		terms: tokenize(doc.text)
+	}));
+	const avgLen = tokenized.reduce((sum, d) => sum + d.terms.length, 0) / tokenized.length || 1;
+	const df = /* @__PURE__ */ new Map();
+	for (const term of queryTerms) df.set(term, tokenized.filter((d) => d.terms.includes(term)).length);
+	const n = tokenized.length;
+	return tokenized.map((doc) => {
+		const len = doc.terms.length || 1;
+		let score = 0;
+		for (const term of queryTerms) {
+			const tf = doc.terms.filter((t) => t === term).length;
+			if (tf === 0) continue;
+			const docFreq = df.get(term) ?? 0;
+			const idf = Math.log(1 + (n - docFreq + .5) / (docFreq + .5));
+			const numerator = tf * 2.5;
+			const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * (len / avgLen));
+			score += idf * (numerator / denominator);
+		}
+		return {
+			id: doc.id,
+			score
+		};
+	}).filter((item) => item.score > 0).sort((a, b) => b.score - a.score);
+}
+
+//#endregion
+export { bm25Rank };
+//# sourceMappingURL=bm25.mjs.map

package/esm/ai/src/rag/hybrid/bm25.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/hybrid/bm25.ts"],"sourcesContent":["import type { RankedItem } from \"./rrf\";\n\n/** A document to score lexically. */\nexport type LexicalDoc = { id: string; text: string };\n\nconst BM25_K1 = 1.5;\nconst BM25_B = 0.75;\n\n/** Lowercase + split on non-word characters; drop empties. */\nfunction tokenize(text: string): string[] {\n  return text\n    .toLowerCase()\n    .split(/[^a-z0-9]+/i)\n    .filter(Boolean);\n}\n\n/**\n * Rank `docs` against `query` with BM25 (A4) — the lexical half of hybrid\n * retrieval. Scores keyword overlap with TF saturation (`k1`) and length\n * normalization (`b`) over the candidate set, so an exact-term match\n * surfaces even when dense embeddings miss it. Returns docs sorted by\n * score (highest first); zero-score docs are dropped.\n *\n * Operates over the supplied candidate set (typically the dense retriever's\n * over-fetch), so it needs no global corpus index — ideal for fusing with\n * a vector ranking via {@link reciprocalRankFusion}.\n */\nexport function bm25Rank(query: string, docs: ReadonlyArray<LexicalDoc>): RankedItem[] {\n  const queryTerms = [...new Set(tokenize(query))];\n  if (queryTerms.length === 0 || docs.length === 0) return [];\n\n  const tokenized = docs.map(doc => ({ id: doc.id, terms: tokenize(doc.text) }));\n  const avgLen =\n    tokenized.reduce((sum, d) => sum + d.terms.length, 0) / tokenized.length || 1;\n\n  // Document frequency per query term, across the candidate set.\n  const df = new Map<string, number>();\n  for (const term of queryTerms) {\n    df.set(\n      term,\n      tokenized.filter(d => d.terms.includes(term)).length,\n    );\n  }\n\n  const n = tokenized.length;\n\n  const scored = tokenized.map(doc => {\n    const len = doc.terms.length || 1;\n    let score = 0;\n\n    for (const term of queryTerms) {\n      const tf = doc.terms.filter(t => t === term).length;\n      if (tf === 0) continue;\n\n      const docFreq = df.get(term) ?? 0;\n      // BM25 idf (with the +1 to keep it non-negative).\n      const idf = Math.log(1 + (n - docFreq + 0.5) / (docFreq + 0.5));\n      const numerator = tf * (BM25_K1 + 1);\n      const denominator = tf + BM25_K1 * (1 - BM25_B + BM25_B * (len / avgLen));\n      score += idf * (numerator / denominator);\n    }\n\n    return { id: doc.id, score };\n  });\n\n  return scored.filter(item => item.score > 0).sort((a, b) => b.score - a.score);\n}\n"],"mappings":";AAKA,MAAM,UAAU;AAChB,MAAM,SAAS;;AAGf,SAAS,SAAS,MAAwB;CACxC,OAAO,KACJ,YAAY,CAAC,CACb,MAAM,aAAa,CAAC,CACpB,OAAO,OAAO;AACnB;;;;;;;;;;;;AAaA,SAAgB,SAAS,OAAe,MAA+C;CACrF,MAAM,aAAa,CAAC,GAAG,IAAI,IAAI,SAAS,KAAK,CAAC,CAAC;CAC/C,IAAI,WAAW,WAAW,KAAK,KAAK,WAAW,GAAG,OAAO,CAAC;CAE1D,MAAM,YAAY,KAAK,KAAI,SAAQ;EAAE,IAAI,IAAI;EAAI,OAAO,SAAS,IAAI,IAAI;CAAE,EAAE;CAC7E,MAAM,SACJ,UAAU,QAAQ,KAAK,MAAM,MAAM,EAAE,MAAM,QAAQ,CAAC,IAAI,UAAU,UAAU;CAG9E,MAAM,qBAAK,IAAI,IAAoB;CACnC,KAAK,MAAM,QAAQ,YACjB,GAAG,IACD,MACA,UAAU,QAAO,MAAK,EAAE,MAAM,SAAS,IAAI,CAAC,CAAC,CAAC,MAChD;CAGF,MAAM,IAAI,UAAU;CAqBpB,OAnBe,UAAU,KAAI,QAAO;EAClC,MAAM,MAAM,IAAI,MAAM,UAAU;EAChC,IAAI,QAAQ;EAEZ,KAAK,MAAM,QAAQ,YAAY;GAC7B,MAAM,KAAK,IAAI,MAAM,QAAO,MAAK,MAAM,IAAI,CAAC,CAAC;GAC7C,IAAI,OAAO,GAAG;GAEd,MAAM,UAAU,GAAG,IAAI,IAAI,KAAK;GAEhC,MAAM,MAAM,KAAK,IAAI,KAAK,IAAI,UAAU,OAAQ,UAAU,GAAI;GAC9D,MAAM,YAAY,KAAM;GACxB,MAAM,cAAc,KAAK,WAAW,IAAI,SAAS,UAAU,MAAM;GACjE,SAAS,OAAO,YAAY;EAC9B;EAEA,OAAO;GAAE,IAAI,IAAI;GAAI;EAAM;CAC7B,CAEY,CAAC,CAAC,QAAO,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAC/E"}

package/esm/ai/src/rag/hybrid/hybrid-rank.mjs

@@ -0,0 +1,29 @@
+import { reciprocalRankFusion } from "./rrf.mjs";
+import { bm25Rank } from "./bm25.mjs";
+
+//#region ../@warlock.js/ai/src/rag/hybrid/hybrid-rank.ts
+/**
+* Hybrid rank (A4) — fuse a dense (vector) ranking with a BM25 lexical
+* ranking over the same candidate set via Reciprocal Rank Fusion. Dense
+* retrieval captures semantic similarity; BM25 captures exact-term
+* matches dense embeddings miss (names, ids, rare tokens). Fusing both
+* beats either alone for keyword-heavy queries.
+*
+* `dense` is the vector retriever's result in rank order; `candidates`
+* supplies the text for the lexical pass (typically the same over-fetched
+* set). Returns the fused ranking, highest score first.
+*
+* @example
+* const fused = hybridRank({
+*   query: "invoice 8842 refund",
+*   dense: vectorHits,                 // [{ id }, ...] in similarity order
+*   candidates: vectorHits.map(h => ({ id: h.id, text: h.text })),
+* });
+*/
+function hybridRank(params) {
+	return reciprocalRankFusion([params.dense.map((d) => d.id), bm25Rank(params.query, params.candidates).map((r) => r.id)], params.k);
+}
+
+//#endregion
+export { hybridRank };
+//# sourceMappingURL=hybrid-rank.mjs.map

package/esm/ai/src/rag/hybrid/hybrid-rank.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-rank.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/hybrid/hybrid-rank.ts"],"sourcesContent":["import { bm25Rank, type LexicalDoc } from \"./bm25\";\nimport { reciprocalRankFusion, type RankedItem } from \"./rrf\";\n\n/**\n * Hybrid rank (A4) — fuse a dense (vector) ranking with a BM25 lexical\n * ranking over the same candidate set via Reciprocal Rank Fusion. Dense\n * retrieval captures semantic similarity; BM25 captures exact-term\n * matches dense embeddings miss (names, ids, rare tokens). Fusing both\n * beats either alone for keyword-heavy queries.\n *\n * `dense` is the vector retriever's result in rank order; `candidates`\n * supplies the text for the lexical pass (typically the same over-fetched\n * set). Returns the fused ranking, highest score first.\n *\n * @example\n * const fused = hybridRank({\n *   query: \"invoice 8842 refund\",\n *   dense: vectorHits,                 // [{ id }, ...] in similarity order\n *   candidates: vectorHits.map(h => ({ id: h.id, text: h.text })),\n * });\n */\nexport function hybridRank(params: {\n  query: string;\n  dense: ReadonlyArray<{ id: string }>;\n  candidates: ReadonlyArray<LexicalDoc>;\n  k?: number;\n}): RankedItem[] {\n  const denseIds = params.dense.map(d => d.id);\n  const lexicalIds = bm25Rank(params.query, params.candidates).map(r => r.id);\n\n  return reciprocalRankFusion([denseIds, lexicalIds], params.k);\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,WAAW,QAKV;CAIf,OAAO,qBAAqB,CAHX,OAAO,MAAM,KAAI,MAAK,EAAE,EAGL,GAFjB,SAAS,OAAO,OAAO,OAAO,UAAU,CAAC,CAAC,KAAI,MAAK,EAAE,EAExB,CAAC,GAAG,OAAO,CAAC;AAC9D"}

package/esm/ai/src/rag/hybrid/rrf.mjs

@@ -0,0 +1,30 @@
+//#region ../@warlock.js/ai/src/rag/hybrid/rrf.ts
+/**
+* Reciprocal Rank Fusion (A4) — combine several independently-ranked
+* lists of ids into one consensus ranking. Each list contributes
+* `1 / (k + rank)` to an id's score (rank is 0-based within that list), so
+* an id near the top of multiple lists rises even if no single list ranks
+* it first. The classic fusion for hybrid (dense + lexical) retrieval
+* because it needs no score calibration between the lists.
+*
+* `k` (default 60, the standard) dampens the contribution of lower ranks.
+* Returns ids sorted by fused score, highest first.
+*
+* @example
+* reciprocalRankFusion([["a", "b", "c"], ["b", "a"]]);
+* // → [{ id: "b", ... }, { id: "a", ... }, { id: "c", ... }]
+*/
+function reciprocalRankFusion(rankedLists, k = 60) {
+	const scores = /* @__PURE__ */ new Map();
+	for (const list of rankedLists) list.forEach((id, rank) => {
+		scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank));
+	});
+	return [...scores.entries()].map(([id, score]) => ({
+		id,
+		score
+	})).sort((a, b) => b.score - a.score);
+}
+
+//#endregion
+export { reciprocalRankFusion };
+//# sourceMappingURL=rrf.mjs.map

package/esm/ai/src/rag/hybrid/rrf.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"rrf.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/hybrid/rrf.ts"],"sourcesContent":["/** One item's id paired with a fused relevance score. */\nexport type RankedItem = { id: string; score: number };\n\n/**\n * Reciprocal Rank Fusion (A4) — combine several independently-ranked\n * lists of ids into one consensus ranking. Each list contributes\n * `1 / (k + rank)` to an id's score (rank is 0-based within that list), so\n * an id near the top of multiple lists rises even if no single list ranks\n * it first. The classic fusion for hybrid (dense + lexical) retrieval\n * because it needs no score calibration between the lists.\n *\n * `k` (default 60, the standard) dampens the contribution of lower ranks.\n * Returns ids sorted by fused score, highest first.\n *\n * @example\n * reciprocalRankFusion([[\"a\", \"b\", \"c\"], [\"b\", \"a\"]]);\n * // → [{ id: \"b\", ... }, { id: \"a\", ... }, { id: \"c\", ... }]\n */\nexport function reciprocalRankFusion(\n  rankedLists: ReadonlyArray<ReadonlyArray<string>>,\n  k = 60,\n): RankedItem[] {\n  const scores = new Map<string, number>();\n\n  for (const list of rankedLists) {\n    list.forEach((id, rank) => {\n      scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank));\n    });\n  }\n\n  return [...scores.entries()]\n    .map(([id, score]) => ({ id, score }))\n    .sort((a, b) => b.score - a.score);\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAkBA,SAAgB,qBACd,aACA,IAAI,IACU;CACd,MAAM,yBAAS,IAAI,IAAoB;CAEvC,KAAK,MAAM,QAAQ,aACjB,KAAK,SAAS,IAAI,SAAS;EACzB,OAAO,IAAI,KAAK,OAAO,IAAI,EAAE,KAAK,KAAK,KAAK,IAAI,KAAK;CACvD,CAAC;CAGH,OAAO,CAAC,GAAG,OAAO,QAAQ,CAAC,CAAC,CACzB,KAAK,CAAC,IAAI,YAAY;EAAE;EAAI;CAAM,EAAE,CAAC,CACrC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACrC"}

package/esm/ai/src/rag/index.mjs

@@ -1,7 +1,18 @@
 import { chunk } from "./chunk/chunk.mjs";
 import { cacheVectorStore } from "./store/cache-vector-store.mjs";
 import { rag } from "./rag.mjs";
+import { pgVectorStore, vectorLiteral } from "./store/pg-vector-store.mjs";
+import { loadText } from "./loaders/load-text.mjs";
+import { loadHtml } from "./loaders/load-html.mjs";
+import { loadWeb } from "./loaders/load-web.mjs";
+import { PDF_PARSE_INSTALL_INSTRUCTIONS } from "./loaders/errors.mjs";
+import { loadPdf } from "./loaders/load-pdf.mjs";
+import "./loaders/index.mjs";
 import { keywordReranker } from "./rerank/keyword-reranker.mjs";
 import { llmReranker } from "./rerank/llm-reranker.mjs";
+import { reciprocalRankFusion } from "./hybrid/rrf.mjs";
+import { bm25Rank } from "./hybrid/bm25.mjs";
+import { hybridRank } from "./hybrid/hybrid-rank.mjs";
+import { multiQuery } from "./transforms/multi-query.mjs";
 
 export {  };

package/esm/ai/src/rag/loaders/errors.mjs

@@ -0,0 +1,25 @@
+//#region ../@warlock.js/ai/src/rag/loaders/errors.ts
+/**
+* Loader error surface. The only loader-specific failure is a missing
+* OPTIONAL peer (`pdf-parse`), which — like the moderation detector's
+* missing `openai` peer — is an *infrastructure* fault, not a content
+* problem, so {@link loadPdf} throws a plain `Error` carrying the curated
+* {@link PDF_PARSE_INSTALL_INSTRUCTIONS} rather than an `AIError`. Mirrors
+* the guard's `OPENAI_INSTALL_INSTRUCTIONS` / ai-panoptic's
+* `LANGFUSE_INSTALL_INSTRUCTIONS`.
+*/
+/**
+* Curated install string thrown by {@link loadPdf} on first call when the
+* `pdf-parse` peer is absent. Surfaced instead of a raw
+* module-resolution stack trace.
+*/
+const PDF_PARSE_INSTALL_INSTRUCTIONS = `
+The @warlock.js/ai PDF loader requires the optional "pdf-parse" peer.
+Install it with:
+
+  npm install pdf-parse
+`.trim();
+
+//#endregion
+export { PDF_PARSE_INSTALL_INSTRUCTIONS };
+//# sourceMappingURL=errors.mjs.map

package/esm/ai/src/rag/loaders/errors.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/loaders/errors.ts"],"sourcesContent":["/**\n * Loader error surface. The only loader-specific failure is a missing\n * OPTIONAL peer (`pdf-parse`), which — like the moderation detector's\n * missing `openai` peer — is an *infrastructure* fault, not a content\n * problem, so {@link loadPdf} throws a plain `Error` carrying the curated\n * {@link PDF_PARSE_INSTALL_INSTRUCTIONS} rather than an `AIError`. Mirrors\n * the guard's `OPENAI_INSTALL_INSTRUCTIONS` / ai-panoptic's\n * `LANGFUSE_INSTALL_INSTRUCTIONS`.\n */\n\n/**\n * Curated install string thrown by {@link loadPdf} on first call when the\n * `pdf-parse` peer is absent. Surfaced instead of a raw\n * module-resolution stack trace.\n */\nexport const PDF_PARSE_INSTALL_INSTRUCTIONS = `\nThe @warlock.js/ai PDF loader requires the optional \"pdf-parse\" peer.\nInstall it with:\n\n  npm install pdf-parse\n`.trim();\n"],"mappings":";;;;;;;;;;;;;;;AAeA,MAAa,iCAAiC;;;;;EAK5C,KAAK"}

package/esm/ai/src/rag/loaders/index.mjs

@@ -0,0 +1,7 @@
+import { loadText } from "./load-text.mjs";
+import { loadHtml } from "./load-html.mjs";
+import { loadWeb } from "./load-web.mjs";
+import { PDF_PARSE_INSTALL_INSTRUCTIONS } from "./errors.mjs";
+import { loadPdf } from "./load-pdf.mjs";
+
+export {  };

package/esm/ai/src/rag/loaders/load-html.mjs

@@ -0,0 +1,138 @@
+//#region ../@warlock.js/ai/src/rag/loaders/load-html.ts
+/** Default `id` when the caller supplies none. */
+const DEFAULT_ID = "document";
+/**
+* Elements whose *content* is not human-readable text and must be removed
+* wholesale (open tag → close tag → everything in between) before tags are
+* stripped. `script` / `style` would otherwise leak code into the chunked
+* text; `noscript` / `template` / `head` / `svg` are non-prose noise.
+*/
+const STRIPPED_ELEMENTS = [
+	"script",
+	"style",
+	"noscript",
+	"template",
+	"head",
+	"svg"
+];
+/**
+* Block-level tags that imply a line break in the readable text. Replacing
+* them with `\n` BEFORE the generic tag strip keeps paragraph / list / table
+* structure (so the recursive splitter still sees `\n\n` boundaries) instead
+* of collapsing the whole page onto one line.
+*/
+const BLOCK_TAGS = /<\/?(?:p|div|section|article|header|footer|main|aside|nav|h[1-6]|ul|ol|li|table|tr|td|th|thead|tbody|blockquote|pre|hr|br)\b[^>]*>/gi;
+/** Named HTML entities common in prose. Numeric entities are decoded generically. */
+const NAMED_ENTITIES = {
+	amp: "&",
+	lt: "<",
+	gt: ">",
+	quot: "\"",
+	apos: "'",
+	nbsp: " ",
+	copy: "©",
+	reg: "®",
+	trade: "™",
+	hellip: "…",
+	mdash: "—",
+	ndash: "–",
+	lsquo: "‘",
+	rsquo: "’",
+	ldquo: "“",
+	rdquo: "”",
+	laquo: "«",
+	raquo: "»",
+	middot: "·",
+	bull: "•"
+};
+/**
+* Decode the HTML entities that survive tag stripping: named (`&amp;`),
+* decimal (`&#169;`), and hex (`&#xA9;`). Unknown named entities are left
+* verbatim rather than dropped, so unusual markup never silently loses text.
+*/
+function decodeEntities(text) {
+	return text.replace(/&(#x?[0-9a-f]+|[a-z][a-z0-9]*);/gi, (match, body) => {
+		if (body[0] === "#") {
+			const codePoint = body[1] === "x" || body[1] === "X" ? Number.parseInt(body.slice(2), 16) : Number.parseInt(body.slice(1), 10);
+			if (Number.isNaN(codePoint) || codePoint < 0 || codePoint > 1114111) return match;
+			try {
+				return String.fromCodePoint(codePoint);
+			} catch {
+				return match;
+			}
+		}
+		return NAMED_ENTITIES[body.toLowerCase()] ?? match;
+	});
+}
+/**
+* Pull the `<title>` text out of the document, decoded and trimmed, or
+* `undefined` when there is none. Read BEFORE `<head>` is stripped.
+*/
+function extractTitle(html) {
+	const match = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
+	if (!match) return;
+	const title = decodeEntities(match[1]).replace(/\s+/g, " ").trim();
+	return title.length > 0 ? title : void 0;
+}
+/**
+* Strip HTML markup down to readable plain text — a lightweight,
+* dependency-free pass (no DOM parser): drop comments and non-prose elements
+* (`script` / `style` / `head` / `svg` / …) content-and-all, convert block
+* tags to line breaks to preserve paragraph structure, remove every
+* remaining tag, decode entities, then collapse runs of whitespace while
+* keeping blank-line paragraph separators.
+*/
+function htmlToText(html) {
+	let text = html;
+	text = text.replace(/<!--[\s\S]*?-->/g, " ");
+	for (const tag of STRIPPED_ELEMENTS) {
+		const element = new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*?<\\/${tag}>`, "gi");
+		text = text.replace(element, " ");
+		text = text.replace(new RegExp(`<\\/?${tag}\\b[^>]*>`, "gi"), " ");
+	}
+	text = text.replace(BLOCK_TAGS, "\n");
+	text = text.replace(/<[^>]+>/g, "");
+	text = decodeEntities(text);
+	text = text.replace(/[^\S\n]+/g, " ").replace(/[ \t]*\n[ \t]*/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+	return text;
+}
+/**
+* Load an HTML string into a single {@link RagDocument} of readable text.
+* Scripts, styles, and other non-prose elements are dropped content-and-all,
+* block tags become line breaks (so paragraph structure survives for the
+* splitter), remaining tags are stripped, and HTML entities are decoded — a
+* lightweight regex pass, no heavy DOM dependency.
+*
+* The document's `metadata.title` is set from the page's `<title>` (unless
+* the caller overrode it), and `metadata.loader` is `"html"`. The output is
+* the exact shape `index()` consumes.
+*
+* @example
+* const kb = ai.rag({ embedder, store });
+* await kb.index(loadHtml(rawHtmlString, { id: "landing-page" }));
+*
+* @param html - The raw HTML markup.
+* @param options - Shared `id` / `metadata` / `tags` ({@link LoadHtmlOptions}).
+* @returns A {@link RagLoaderResult} (one document) ready for `rag.index()`.
+*/
+function loadHtml(html, options = {}) {
+	const id = options.id ?? DEFAULT_ID;
+	const title = extractTitle(html);
+	const text = htmlToText(html);
+	if (text.length === 0) return [];
+	return [{
+		id,
+		text,
+		metadata: {
+			source: id,
+			loader: "html",
+			...title !== void 0 ? { title } : {},
+			...options.metadata
+		},
+		tags: options.tags
+	}];
+}
+
+//#endregion
+export { extractTitle, htmlToText, loadHtml };
+//# sourceMappingURL=load-html.mjs.map

package/esm/ai/src/rag/loaders/load-html.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-html.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/loaders/load-html.ts"],"sourcesContent":["import type { RagDocument } from \"../contracts/rag-document.type\";\nimport type { LoadHtmlOptions, RagLoaderResult } from \"./loader.type\";\n\n/** Default `id` when the caller supplies none. */\nconst DEFAULT_ID = \"document\";\n\n/**\n * Elements whose *content* is not human-readable text and must be removed\n * wholesale (open tag → close tag → everything in between) before tags are\n * stripped. `script` / `style` would otherwise leak code into the chunked\n * text; `noscript` / `template` / `head` / `svg` are non-prose noise.\n */\nconst STRIPPED_ELEMENTS = [\n  \"script\",\n  \"style\",\n  \"noscript\",\n  \"template\",\n  \"head\",\n  \"svg\",\n];\n\n/**\n * Block-level tags that imply a line break in the readable text. Replacing\n * them with `\\n` BEFORE the generic tag strip keeps paragraph / list / table\n * structure (so the recursive splitter still sees `\\n\\n` boundaries) instead\n * of collapsing the whole page onto one line.\n */\nconst BLOCK_TAGS =\n  /<\\/?(?:p|div|section|article|header|footer|main|aside|nav|h[1-6]|ul|ol|li|table|tr|td|th|thead|tbody|blockquote|pre|hr|br)\\b[^>]*>/gi;\n\n/** Named HTML entities common in prose. Numeric entities are decoded generically. */\nconst NAMED_ENTITIES: Record<string, string> = {\n  amp: \"&\",\n  lt: \"<\",\n  gt: \">\",\n  quot: '\"',\n  apos: \"'\",\n  nbsp: \" \",\n  copy: \"©\",\n  reg: \"®\",\n  trade: \"™\",\n  hellip: \"…\",\n  mdash: \"—\",\n  ndash: \"–\",\n  lsquo: \"‘\",\n  rsquo: \"’\",\n  ldquo: \"“\",\n  rdquo: \"”\",\n  laquo: \"«\",\n  raquo: \"»\",\n  middot: \"·\",\n  bull: \"•\",\n};\n\n/**\n * Decode the HTML entities that survive tag stripping: named (`&amp;`),\n * decimal (`&#169;`), and hex (`&#xA9;`). Unknown named entities are left\n * verbatim rather than dropped, so unusual markup never silently loses text.\n */\nfunction decodeEntities(text: string): string {\n  return text.replace(/&(#x?[0-9a-f]+|[a-z][a-z0-9]*);/gi, (match, body: string) => {\n    if (body[0] === \"#\") {\n      const codePoint =\n        body[1] === \"x\" || body[1] === \"X\"\n          ? Number.parseInt(body.slice(2), 16)\n          : Number.parseInt(body.slice(1), 10);\n\n      if (Number.isNaN(codePoint) || codePoint < 0 || codePoint > 0x10ffff) {\n        return match;\n      }\n\n      try {\n        return String.fromCodePoint(codePoint);\n      } catch {\n        return match;\n      }\n    }\n\n    const named = NAMED_ENTITIES[body.toLowerCase()];\n\n    return named ?? match;\n  });\n}\n\n/**\n * Pull the `<title>` text out of the document, decoded and trimmed, or\n * `undefined` when there is none. Read BEFORE `<head>` is stripped.\n */\nfunction extractTitle(html: string): string | undefined {\n  const match = /<title[^>]*>([\\s\\S]*?)<\\/title>/i.exec(html);\n\n  if (!match) {\n    return undefined;\n  }\n\n  const title = decodeEntities(match[1]).replace(/\\s+/g, \" \").trim();\n\n  return title.length > 0 ? title : undefined;\n}\n\n/**\n * Strip HTML markup down to readable plain text — a lightweight,\n * dependency-free pass (no DOM parser): drop comments and non-prose elements\n * (`script` / `style` / `head` / `svg` / …) content-and-all, convert block\n * tags to line breaks to preserve paragraph structure, remove every\n * remaining tag, decode entities, then collapse runs of whitespace while\n * keeping blank-line paragraph separators.\n */\nfunction htmlToText(html: string): string {\n  let text = html;\n\n  // 1. Comments first — a commented-out `<script>` must not survive.\n  text = text.replace(/<!--[\\s\\S]*?-->/g, \" \");\n\n  // 2. Non-prose elements, content and all.\n  for (const tag of STRIPPED_ELEMENTS) {\n    const element = new RegExp(`<${tag}\\\\b[^>]*>[\\\\s\\\\S]*?<\\\\/${tag}>`, \"gi\");\n    text = text.replace(element, \" \");\n    // Defensively drop a self-closing / unterminated open tag too.\n    text = text.replace(new RegExp(`<\\\\/?${tag}\\\\b[^>]*>`, \"gi\"), \" \");\n  }\n\n  // 3. Block tags → newlines, so paragraph / list structure survives.\n  text = text.replace(BLOCK_TAGS, \"\\n\");\n\n  // 4. Every remaining tag → gone.\n  text = text.replace(/<[^>]+>/g, \"\");\n\n  // 5. Entities → characters.\n  text = decodeEntities(text);\n\n  // 6. Normalize whitespace: trim each line, drop blank runs to a single\n  //    blank line (a paragraph separator the recursive splitter honors).\n  text = text\n    .replace(/[^\\S\\n]+/g, \" \")\n    .replace(/[ \\t]*\\n[ \\t]*/g, \"\\n\")\n    .replace(/\\n{3,}/g, \"\\n\\n\")\n    .trim();\n\n  return text;\n}\n\n/**\n * Load an HTML string into a single {@link RagDocument} of readable text.\n * Scripts, styles, and other non-prose elements are dropped content-and-all,\n * block tags become line breaks (so paragraph structure survives for the\n * splitter), remaining tags are stripped, and HTML entities are decoded — a\n * lightweight regex pass, no heavy DOM dependency.\n *\n * The document's `metadata.title` is set from the page's `<title>` (unless\n * the caller overrode it), and `metadata.loader` is `\"html\"`. The output is\n * the exact shape `index()` consumes.\n *\n * @example\n * const kb = ai.rag({ embedder, store });\n * await kb.index(loadHtml(rawHtmlString, { id: \"landing-page\" }));\n *\n * @param html - The raw HTML markup.\n * @param options - Shared `id` / `metadata` / `tags` ({@link LoadHtmlOptions}).\n * @returns A {@link RagLoaderResult} (one document) ready for `rag.index()`.\n */\nexport function loadHtml(\n  html: string,\n  options: LoadHtmlOptions = {},\n): RagLoaderResult {\n  const id = options.id ?? DEFAULT_ID;\n  const title = extractTitle(html);\n  const text = htmlToText(html);\n\n  // An all-markup / empty page strips to nothing; emit no document so\n  // index() never receives a no-op record (matches loadText's behavior).\n  if (text.length === 0) {\n    return [];\n  }\n\n  // Derived keys (source, loader, title) sit UNDER the caller's metadata so\n  // an explicit override always wins.\n  const doc: RagDocument = {\n    id,\n    text,\n    metadata: {\n      source: id,\n      loader: \"html\",\n      ...(title !== undefined ? { title } : {}),\n      ...options.metadata,\n    },\n    tags: options.tags,\n  };\n\n  return [doc];\n}\n\n/** Internal — exported for the web loader so it shares the exact strip pass. */\nexport { htmlToText, extractTitle };\n"],"mappings":";;AAIA,MAAM,aAAa;;;;;;;AAQnB,MAAM,oBAAoB;CACxB;CACA;CACA;CACA;CACA;CACA;AACF;;;;;;;AAQA,MAAM,aACJ;;AAGF,MAAM,iBAAyC;CAC7C,KAAK;CACL,IAAI;CACJ,IAAI;CACJ,MAAM;CACN,MAAM;CACN,MAAM;CACN,MAAM;CACN,KAAK;CACL,OAAO;CACP,QAAQ;CACR,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,OAAO;CACP,QAAQ;CACR,MAAM;AACR;;;;;;AAOA,SAAS,eAAe,MAAsB;CAC5C,OAAO,KAAK,QAAQ,sCAAsC,OAAO,SAAiB;EAChF,IAAI,KAAK,OAAO,KAAK;GACnB,MAAM,YACJ,KAAK,OAAO,OAAO,KAAK,OAAO,MAC3B,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE,IACjC,OAAO,SAAS,KAAK,MAAM,CAAC,GAAG,EAAE;GAEvC,IAAI,OAAO,MAAM,SAAS,KAAK,YAAY,KAAK,YAAY,SAC1D,OAAO;GAGT,IAAI;IACF,OAAO,OAAO,cAAc,SAAS;GACvC,QAAQ;IACN,OAAO;GACT;EACF;EAIA,OAFc,eAAe,KAAK,YAAY,MAE9B;CAClB,CAAC;AACH;;;;;AAMA,SAAS,aAAa,MAAkC;CACtD,MAAM,QAAQ,mCAAmC,KAAK,IAAI;CAE1D,IAAI,CAAC,OACH;CAGF,MAAM,QAAQ,eAAe,MAAM,EAAE,CAAC,CAAC,QAAQ,QAAQ,GAAG,CAAC,CAAC,KAAK;CAEjE,OAAO,MAAM,SAAS,IAAI,QAAQ;AACpC;;;;;;;;;AAUA,SAAS,WAAW,MAAsB;CACxC,IAAI,OAAO;CAGX,OAAO,KAAK,QAAQ,oBAAoB,GAAG;CAG3C,KAAK,MAAM,OAAO,mBAAmB;EACnC,MAAM,UAAU,IAAI,OAAO,IAAI,IAAI,yBAAyB,IAAI,IAAI,IAAI;EACxE,OAAO,KAAK,QAAQ,SAAS,GAAG;EAEhC,OAAO,KAAK,QAAQ,IAAI,OAAO,QAAQ,IAAI,YAAY,IAAI,GAAG,GAAG;CACnE;CAGA,OAAO,KAAK,QAAQ,YAAY,IAAI;CAGpC,OAAO,KAAK,QAAQ,YAAY,EAAE;CAGlC,OAAO,eAAe,IAAI;CAI1B,OAAO,KACJ,QAAQ,aAAa,GAAG,CAAC,CACzB,QAAQ,mBAAmB,IAAI,CAAC,CAChC,QAAQ,WAAW,MAAM,CAAC,CAC1B,KAAK;CAER,OAAO;AACT;;;;;;;;;;;;;;;;;;;;AAqBA,SAAgB,SACd,MACA,UAA2B,CAAC,GACX;CACjB,MAAM,KAAK,QAAQ,MAAM;CACzB,MAAM,QAAQ,aAAa,IAAI;CAC/B,MAAM,OAAO,WAAW,IAAI;CAI5B,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAiBV,OAAO,CAAC;EAXN;EACA;EACA,UAAU;GACR,QAAQ;GACR,QAAQ;GACR,GAAI,UAAU,SAAY,EAAE,MAAM,IAAI,CAAC;GACvC,GAAG,QAAQ;EACb;EACA,MAAM,QAAQ;CAGN,CAAC;AACb"}

package/esm/ai/src/rag/loaders/load-pdf.mjs

@@ -0,0 +1,150 @@
+import { PDF_PARSE_INSTALL_INSTRUCTIONS } from "./errors.mjs";
+
+//#region ../@warlock.js/ai/src/rag/loaders/load-pdf.ts
+/** Default `id` when the caller supplies none. */
+const DEFAULT_ID = "document";
+let pdfParse;
+let isModuleExists;
+let loadingPromise;
+/**
+* Settle the lazy import of `pdf-parse` once, concurrency-safe. A bare
+* `catch` flips the flag to `false`; the curated
+* {@link PDF_PARSE_INSTALL_INSTRUCTIONS} surfaces at first
+* {@link loadPdf} call, never a raw module-resolution stack trace. Mirrors
+* the guard moderation detector's `loadOpenAi`.
+*/
+function loadPdfParse() {
+	if (isModuleExists !== void 0) return Promise.resolve();
+	if (loadingPromise) return loadingPromise;
+	loadingPromise = (async () => {
+		try {
+			const mod = await import("pdf-parse");
+			pdfParse = mod.default ?? mod;
+			isModuleExists = typeof pdfParse === "function";
+		} catch {
+			isModuleExists = false;
+		}
+	})();
+	return loadingPromise;
+}
+/**
+* Coerce a {@link RagDocument}-compatible binary input into a `Buffer` for
+* `pdf-parse`. Accepts a Node `Buffer`, an `ArrayBuffer`, or a typed array
+* (`Uint8Array`) — the shapes a file read / fetch body hands back.
+*/
+function toBuffer(input) {
+	if (Buffer.isBuffer(input)) return input;
+	if (input instanceof ArrayBuffer) return Buffer.from(input);
+	return Buffer.from(input.buffer, input.byteOffset, input.byteLength);
+}
+/**
+* Load a PDF's bytes into {@link RagDocument}(s) via the OPTIONAL `pdf-parse`
+* peer. The peer is resolved lazily on the FIRST call (not at import) so
+* importing `@warlock.js/ai` never forces it to be installed; when it is
+* absent the curated {@link PDF_PARSE_INSTALL_INSTRUCTIONS} is thrown as a
+* plain `Error` (a missing optional peer is an infrastructure fault, not a
+* content problem).
+*
+* By default the whole PDF becomes a single document carrying
+* `metadata.pageCount`. With `perPage: true`, each page becomes its own
+* document (`id` suffixed `#p<n>`, `metadata.page` set) so citations stay
+* page-precise. Document `metadata.title` comes from the PDF info
+* dictionary's `Title` (unless overridden), and `metadata.loader` is
+* `"pdf"`. The output is the exact shape `index()` consumes.
+*
+* @example
+* import { readFile } from "node:fs/promises";
+* const kb = ai.rag({ embedder, store });
+* await kb.index(await loadPdf(await readFile("guide.pdf"), { id: "guide" }));
+*
+* @example
+* // One document per page for page-precise citations:
+* await kb.index(await loadPdf(bytes, { id: "manual", perPage: true }));
+*
+* @param input - The PDF bytes (`Buffer`, `ArrayBuffer`, or `Uint8Array`).
+* @param options - `perPage` plus shared `id` / `metadata` / `tags`
+*   ({@link LoadPdfOptions}).
+* @returns A {@link RagLoaderResult} ready for `rag.index()`.
+* @throws {Error} carrying {@link PDF_PARSE_INSTALL_INSTRUCTIONS} when the
+*   `pdf-parse` peer is not installed.
+*/
+async function loadPdf(input, options = {}) {
+	await loadPdfParse();
+	if (!isModuleExists || !pdfParse) throw new Error(PDF_PARSE_INSTALL_INSTRUCTIONS);
+	const id = options.id ?? DEFAULT_ID;
+	if (options.perPage ?? false) return loadPerPage(input, id, options);
+	const parsed = await pdfParse(toBuffer(input));
+	const text = parsed.text.trim();
+	const title = parsed.info?.Title?.trim();
+	if (text.length === 0) return [];
+	return [{
+		id,
+		text,
+		metadata: {
+			source: id,
+			loader: "pdf",
+			pageCount: parsed.numpages,
+			...title ? { title } : {},
+			...options.metadata
+		},
+		tags: options.tags
+	}];
+}
+/**
+* Per-page variant: render each page separately via `pdf-parse`'s
+* `pagerender` hook, accumulating one document per non-empty page. Each
+* carries `metadata.page` (1-based) and `metadata.pageCount`, and its id is
+* the base id suffixed `#p<n>` so every page-document is distinctly
+* identified for citation.
+*
+* `pdf-parse` calls `pagerender` once per page in document order and
+* `await`s the returned string, so capturing each page's joined text content
+* here gives reliable page boundaries the concatenated `text` lacks.
+*/
+async function loadPerPage(input, id, options) {
+	const pages = [];
+	const parsed = await pdfParse(toBuffer(input), { pagerender: async (page) => {
+		const rendered = await renderPage(page);
+		pages.push(rendered);
+		return rendered;
+	} });
+	const title = parsed.info?.Title?.trim();
+	const docs = [];
+	pages.forEach((pageText, index) => {
+		const text = pageText.trim();
+		if (text.length === 0) return;
+		const pageNumber = index + 1;
+		docs.push({
+			id: `${id}#p${pageNumber}`,
+			text,
+			metadata: {
+				source: id,
+				loader: "pdf",
+				page: pageNumber,
+				pageCount: parsed.numpages,
+				...title ? { title } : {},
+				...options.metadata
+			},
+			tags: options.tags
+		});
+	});
+	return docs;
+}
+/**
+* Join a single page's text-layer items in reading order, inserting a space
+* between items so adjacent words do not run together. Mirrors the essence
+* of `pdf-parse`'s default renderer without depending on its internals, so
+* the per-page hook stays stable across `pdf-parse` versions. A page with no
+* text layer (scanned image) renders to an empty string and is dropped.
+*/
+async function renderPage(page) {
+	if (typeof page?.getTextContent !== "function") return "";
+	return (await page.getTextContent({
+		normalizeWhitespace: true,
+		disableCombineTextItems: false
+	})).items.map((item) => item.str).join(" ").replace(/\s+/g, " ").trim();
+}
+
+//#endregion
+export { loadPdf };
+//# sourceMappingURL=load-pdf.mjs.map

package/esm/ai/src/rag/loaders/load-pdf.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-pdf.mjs","names":[],"sources":["../../../../../../../../../../@warlock.js/ai/src/rag/loaders/load-pdf.ts"],"sourcesContent":["import type { RagDocument } from \"../contracts/rag-document.type\";\nimport { PDF_PARSE_INSTALL_INSTRUCTIONS } from \"./errors\";\nimport type { LoadPdfOptions, RagLoaderResult } from \"./loader.type\";\n\n/** Default `id` when the caller supplies none. */\nconst DEFAULT_ID = \"document\";\n\n/**\n * The slice of `pdf-parse`'s result we consume. The peer returns more\n * (`info`, `metadata`, `version`); we only need the extracted `text` and\n * page count, so we type just those to keep the dependency at arm's length.\n */\ntype PdfParseResult = {\n  /** Concatenated text of every page. */\n  text: string;\n  /** Number of pages in the document. */\n  numpages: number;\n  /** Document info dictionary — `Title` lifted into metadata when present. */\n  info?: { Title?: string } & Record<string, unknown>;\n};\n\n/** The `pdf-parse` module's callable default export. */\ntype PdfParseFn = (\n  data: Buffer | Uint8Array,\n  options?: {\n    /**\n     * Per-page renderer `pdf-parse` calls once per page in document order and\n     * `await`s — may return the page text synchronously or as a promise.\n     */\n    pagerender?: (page: unknown) => string | Promise<string>;\n  },\n) => Promise<PdfParseResult>;\n\n// ============================================================\n// Lazily-loaded pdf-parse (OPTIONAL peer)\n// ============================================================\n\nlet pdfParse: PdfParseFn | undefined;\nlet isModuleExists: boolean | undefined;\nlet loadingPromise: Promise<void> | undefined;\n\n/**\n * Settle the lazy import of `pdf-parse` once, concurrency-safe. A bare\n * `catch` flips the flag to `false`; the curated\n * {@link PDF_PARSE_INSTALL_INSTRUCTIONS} surfaces at first\n * {@link loadPdf} call, never a raw module-resolution stack trace. Mirrors\n * the guard moderation detector's `loadOpenAi`.\n */\nfunction loadPdfParse(): Promise<void> {\n  if (isModuleExists !== undefined) {\n    return Promise.resolve();\n  }\n\n  if (loadingPromise) {\n    return loadingPromise;\n  }\n\n  loadingPromise = (async () => {\n    try {\n      // Literal specifier so `vi.mock(\"pdf-parse\")` can intercept it in tests.\n      // Typed via the ambient `pdf-parse` shim in this directory, so the bare\n      // import resolves even though the OPTIONAL peer is not a dependency.\n      const mod = (await import(\"pdf-parse\")) as {\n        default?: PdfParseFn;\n      } & Partial<PdfParseFn>;\n      // pdf-parse ships CommonJS — the callable is `module.exports`, surfaced\n      // as `default` under ESM interop. Fall back to the namespace itself for\n      // bundlers that hoist the callable to the top level.\n      pdfParse = mod.default ?? (mod as unknown as PdfParseFn);\n      isModuleExists = typeof pdfParse === \"function\";\n    } catch {\n      isModuleExists = false;\n    }\n  })();\n\n  return loadingPromise;\n}\n\n/**\n * Coerce a {@link RagDocument}-compatible binary input into a `Buffer` for\n * `pdf-parse`. Accepts a Node `Buffer`, an `ArrayBuffer`, or a typed array\n * (`Uint8Array`) — the shapes a file read / fetch body hands back.\n */\nfunction toBuffer(input: Buffer | ArrayBuffer | Uint8Array): Buffer {\n  if (Buffer.isBuffer(input)) {\n    return input;\n  }\n\n  if (input instanceof ArrayBuffer) {\n    return Buffer.from(input);\n  }\n\n  return Buffer.from(input.buffer, input.byteOffset, input.byteLength);\n}\n\n/**\n * Load a PDF's bytes into {@link RagDocument}(s) via the OPTIONAL `pdf-parse`\n * peer. The peer is resolved lazily on the FIRST call (not at import) so\n * importing `@warlock.js/ai` never forces it to be installed; when it is\n * absent the curated {@link PDF_PARSE_INSTALL_INSTRUCTIONS} is thrown as a\n * plain `Error` (a missing optional peer is an infrastructure fault, not a\n * content problem).\n *\n * By default the whole PDF becomes a single document carrying\n * `metadata.pageCount`. With `perPage: true`, each page becomes its own\n * document (`id` suffixed `#p<n>`, `metadata.page` set) so citations stay\n * page-precise. Document `metadata.title` comes from the PDF info\n * dictionary's `Title` (unless overridden), and `metadata.loader` is\n * `\"pdf\"`. The output is the exact shape `index()` consumes.\n *\n * @example\n * import { readFile } from \"node:fs/promises\";\n * const kb = ai.rag({ embedder, store });\n * await kb.index(await loadPdf(await readFile(\"guide.pdf\"), { id: \"guide\" }));\n *\n * @example\n * // One document per page for page-precise citations:\n * await kb.index(await loadPdf(bytes, { id: \"manual\", perPage: true }));\n *\n * @param input - The PDF bytes (`Buffer`, `ArrayBuffer`, or `Uint8Array`).\n * @param options - `perPage` plus shared `id` / `metadata` / `tags`\n *   ({@link LoadPdfOptions}).\n * @returns A {@link RagLoaderResult} ready for `rag.index()`.\n * @throws {Error} carrying {@link PDF_PARSE_INSTALL_INSTRUCTIONS} when the\n *   `pdf-parse` peer is not installed.\n */\nexport async function loadPdf(\n  input: Buffer | ArrayBuffer | Uint8Array,\n  options: LoadPdfOptions = {},\n): Promise<RagLoaderResult> {\n  await loadPdfParse();\n\n  if (!isModuleExists || !pdfParse) {\n    throw new Error(PDF_PARSE_INSTALL_INSTRUCTIONS);\n  }\n\n  const id = options.id ?? DEFAULT_ID;\n  const perPage = options.perPage ?? false;\n\n  if (perPage) {\n    return loadPerPage(input, id, options);\n  }\n\n  const parsed = await pdfParse(toBuffer(input));\n  const text = parsed.text.trim();\n  const title = parsed.info?.Title?.trim();\n\n  // An image-only / empty PDF extracts no text — emit nothing so index()\n  // never receives a no-op record.\n  if (text.length === 0) {\n    return [];\n  }\n\n  const doc: RagDocument = {\n    id,\n    text,\n    metadata: {\n      source: id,\n      loader: \"pdf\",\n      pageCount: parsed.numpages,\n      ...(title ? { title } : {}),\n      ...options.metadata,\n    },\n    tags: options.tags,\n  };\n\n  return [doc];\n}\n\n/** One page of a parsed PDF — the text-layer item list `pagerender` sees. */\ntype PdfPage = {\n  getTextContent: (\n    options?: unknown,\n  ) => Promise<{ items: { str: string }[] }>;\n};\n\n/**\n * Per-page variant: render each page separately via `pdf-parse`'s\n * `pagerender` hook, accumulating one document per non-empty page. Each\n * carries `metadata.page` (1-based) and `metadata.pageCount`, and its id is\n * the base id suffixed `#p<n>` so every page-document is distinctly\n * identified for citation.\n *\n * `pdf-parse` calls `pagerender` once per page in document order and\n * `await`s the returned string, so capturing each page's joined text content\n * here gives reliable page boundaries the concatenated `text` lacks.\n */\nasync function loadPerPage(\n  input: Buffer | ArrayBuffer | Uint8Array,\n  id: string,\n  options: LoadPdfOptions,\n): Promise<RagDocument[]> {\n  const pages: string[] = [];\n\n  const parsed = await pdfParse!(toBuffer(input), {\n    pagerender: async (page: unknown): Promise<string> => {\n      const rendered = await renderPage(page as PdfPage);\n      pages.push(rendered);\n      return rendered;\n    },\n  });\n\n  const title = parsed.info?.Title?.trim();\n  const docs: RagDocument[] = [];\n\n  pages.forEach((pageText, index) => {\n    const text = pageText.trim();\n\n    if (text.length === 0) {\n      return;\n    }\n\n    const pageNumber = index + 1;\n\n    docs.push({\n      id: `${id}#p${pageNumber}`,\n      text,\n      metadata: {\n        source: id,\n        loader: \"pdf\",\n        page: pageNumber,\n        pageCount: parsed.numpages,\n        ...(title ? { title } : {}),\n        ...options.metadata,\n      },\n      tags: options.tags,\n    });\n  });\n\n  return docs;\n}\n\n/**\n * Join a single page's text-layer items in reading order, inserting a space\n * between items so adjacent words do not run together. Mirrors the essence\n * of `pdf-parse`'s default renderer without depending on its internals, so\n * the per-page hook stays stable across `pdf-parse` versions. A page with no\n * text layer (scanned image) renders to an empty string and is dropped.\n */\nasync function renderPage(page: PdfPage): Promise<string> {\n  if (typeof page?.getTextContent !== \"function\") {\n    return \"\";\n  }\n\n  const content = await page.getTextContent({\n    normalizeWhitespace: true,\n    disableCombineTextItems: false,\n  });\n\n  return content.items\n    .map((item) => item.str)\n    .join(\" \")\n    .replace(/\\s+/g, \" \")\n    .trim();\n}\n"],"mappings":";;;;AAKA,MAAM,aAAa;AAgCnB,IAAI;AACJ,IAAI;AACJ,IAAI;;;;;;;;AASJ,SAAS,eAA8B;CACrC,IAAI,mBAAmB,QACrB,OAAO,QAAQ,QAAQ;CAGzB,IAAI,gBACF,OAAO;CAGT,kBAAkB,YAAY;EAC5B,IAAI;GAIF,MAAM,MAAO,MAAM,OAAO;GAM1B,WAAW,IAAI,WAAY;GAC3B,iBAAiB,OAAO,aAAa;EACvC,QAAQ;GACN,iBAAiB;EACnB;CACF,EAAC,CAAE;CAEH,OAAO;AACT;;;;;;AAOA,SAAS,SAAS,OAAkD;CAClE,IAAI,OAAO,SAAS,KAAK,GACvB,OAAO;CAGT,IAAI,iBAAiB,aACnB,OAAO,OAAO,KAAK,KAAK;CAG1B,OAAO,OAAO,KAAK,MAAM,QAAQ,MAAM,YAAY,MAAM,UAAU;AACrE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiCA,eAAsB,QACpB,OACA,UAA0B,CAAC,GACD;CAC1B,MAAM,aAAa;CAEnB,IAAI,CAAC,kBAAkB,CAAC,UACtB,MAAM,IAAI,MAAM,8BAA8B;CAGhD,MAAM,KAAK,QAAQ,MAAM;CAGzB,IAFgB,QAAQ,WAAW,OAGjC,OAAO,YAAY,OAAO,IAAI,OAAO;CAGvC,MAAM,SAAS,MAAM,SAAS,SAAS,KAAK,CAAC;CAC7C,MAAM,OAAO,OAAO,KAAK,KAAK;CAC9B,MAAM,QAAQ,OAAO,MAAM,OAAO,KAAK;CAIvC,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAgBV,OAAO,CAAC;EAZN;EACA;EACA,UAAU;GACR,QAAQ;GACR,QAAQ;GACR,WAAW,OAAO;GAClB,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;GACzB,GAAG,QAAQ;EACb;EACA,MAAM,QAAQ;CAGN,CAAC;AACb;;;;;;;;;;;;AAoBA,eAAe,YACb,OACA,IACA,SACwB;CACxB,MAAM,QAAkB,CAAC;CAEzB,MAAM,SAAS,MAAM,SAAU,SAAS,KAAK,GAAG,EAC9C,YAAY,OAAO,SAAmC;EACpD,MAAM,WAAW,MAAM,WAAW,IAAe;EACjD,MAAM,KAAK,QAAQ;EACnB,OAAO;CACT,EACF,CAAC;CAED,MAAM,QAAQ,OAAO,MAAM,OAAO,KAAK;CACvC,MAAM,OAAsB,CAAC;CAE7B,MAAM,SAAS,UAAU,UAAU;EACjC,MAAM,OAAO,SAAS,KAAK;EAE3B,IAAI,KAAK,WAAW,GAClB;EAGF,MAAM,aAAa,QAAQ;EAE3B,KAAK,KAAK;GACR,IAAI,GAAG,GAAG,IAAI;GACd;GACA,UAAU;IACR,QAAQ;IACR,QAAQ;IACR,MAAM;IACN,WAAW,OAAO;IAClB,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;IACzB,GAAG,QAAQ;GACb;GACA,MAAM,QAAQ;EAChB,CAAC;CACH,CAAC;CAED,OAAO;AACT;;;;;;;;AASA,eAAe,WAAW,MAAgC;CACxD,IAAI,OAAO,MAAM,mBAAmB,YAClC,OAAO;CAQT,QAAO,MALe,KAAK,eAAe;EACxC,qBAAqB;EACrB,yBAAyB;CAC3B,CAAC,EAEa,CAAC,MACZ,KAAK,SAAS,KAAK,GAAG,CAAC,CACvB,KAAK,GAAG,CAAC,CACT,QAAQ,QAAQ,GAAG,CAAC,CACpB,KAAK;AACV"}