@devilsdev/rag-core 3.6.0 → 3.7.0

package/dist/all.d.ts

@@ -532,6 +532,7 @@ export { MemoryRetriever } from "./types/connectors/memory-retriever";
 export { PgVectorRetriever } from "./types/retrieval/pgvector-retriever";
 export { QdrantRetriever } from "./types/retrieval/qdrant-retriever";
 export { PineconeRetriever } from "./types/retrieval/pinecone-retriever";
+export { WeaviateRetriever } from "./types/retrieval/weaviate-retriever";
 export { OpenAIConnector } from "./types/connectors/openai-connector";
 export { AnthropicConnector } from "./types/connectors/anthropic-connector";
 export { CohereConnector } from "./types/connectors/cohere-connector";

package/dist/index.cjs

@@ -59,32 +59,44 @@ var require_logger = __commonJS({
         return false;
       }
     };
-    var baseLogger = pino({
-      name: "rag-pipeline-utils",
-      level: getLogLevel(),
-      formatters: {
-        level: (label) => {
-          return { level: label };
+    var getDestination = (pretty) => {
+      if (pretty || typeof pino.destination !== "function") return void 0;
+      const stdout = process.stdout;
+      if (!stdout || stdout.write !== stdout.constructor.prototype.write) {
+        return void 0;
+      }
+      return pino.destination({ dest: stdout.fd || 1, sync: true });
+    };
+    var prettyPrint = shouldUsePrettyPrint();
+    var baseLogger = pino(
+      {
+        name: "rag-pipeline-utils",
+        level: getLogLevel(),
+        formatters: {
+          level: (label) => {
+            return { level: label };
+          },
+          log: (object) => {
+            if (object.correlationId) {
+              object.correlationId = object.correlationId;
+            }
+            return object;
+          }
         },
-        log: (object) => {
-          if (object.correlationId) {
-            object.correlationId = object.correlationId;
+        // Pretty print in development or when LOG_PRETTY=true
+        ...prettyPrint && {
+          transport: {
+            target: "pino-pretty",
+            options: {
+              colorize: true,
+              translateTime: "HH:MM:ss",
+              ignore: "pid,hostname"
+            }
           }
-          return object;
         }
       },
-      // Pretty print in development or when LOG_PRETTY=true
-      ...shouldUsePrettyPrint() && {
-        transport: {
-          target: "pino-pretty",
-          options: {
-            colorize: true,
-            translateTime: "HH:MM:ss",
-            ignore: "pid,hostname"
-          }
-        }
-      }
-    });
+      getDestination(prettyPrint)
+    );
     function normalizeMeta(meta) {
       if (meta == null) return {};
       if (meta instanceof Error) {
@@ -1358,7 +1370,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@devilsdev/rag-pipeline-utils",
-      version: "3.6.0",
+      version: "3.7.0",
       description: "Composable retrieval-augmented generation (RAG) for Node.js: grounded citations, evaluation, guardrails, hybrid retrieval and a CLI.",
       main: "dist/index.cjs",
       type: "commonjs",
@@ -1397,7 +1409,7 @@ var require_package = __commonJS({
         "./cli": "./bin/cli.js"
       },
       bin: {
-        "rag-pipeline": "./bin/cli.js"
+        "rag-pipeline": "bin/cli.js"
       },
       engines: {
         node: ">=22"
@@ -1499,7 +1511,7 @@ var require_package = __commonJS({
       },
       repository: {
         type: "git",
-        url: "https://github.com/DevilsDev/rag-pipeline-utils.git"
+        url: "git+https://github.com/DevilsDev/rag-pipeline-utils.git"
       },
       keywords: [
         "rag",
@@ -1557,7 +1569,7 @@ var require_package = __commonJS({
       "lint-staged": {
         "src/**/*.{js,ts,jsx,tsx}": "eslint --fix",
         "{__tests__,scripts,bin,docs-site,examples,contracts}/**/*.{js,jsx,ts,tsx,mjs,cjs}": "prettier --write --ignore-unknown",
-        "*.{js,mjs,cjs}": "prettier --write --ignore-unknown",
+        "./*.{js,mjs,cjs}": "prettier --write --ignore-unknown",
         "**/*.{json,md,mdx,yml,yaml,css,scss,html}": "prettier --write --ignore-unknown"
       },
       devDependencies: {
@@ -1626,10 +1638,10 @@ var require_package = __commonJS({
         "why-is-node-running": "^3.2.2"
       },
       dependencies: {
-        "@devilsdev/rag-connectors": "3.6.0",
-        "@devilsdev/rag-core": "3.6.0",
-        "@devilsdev/rag-eval": "3.6.0",
-        "@devilsdev/rag-guardrails": "3.6.0",
+        "@devilsdev/rag-connectors": "3.7.0",
+        "@devilsdev/rag-core": "3.7.0",
+        "@devilsdev/rag-eval": "3.7.0",
+        "@devilsdev/rag-guardrails": "3.7.0",
         ajv: "^8.17.1",
         axios: "^1.21.0",
         chokidar: "^4.0.3",
@@ -2586,8 +2598,8 @@ var require_scoring = __commonJS({
       };
       for (const w of words) {
         const last = run[run.length - 1];
-        const isNumberWord = w in NUMBER_WORDS || w in NUMBER_SCALES || w === "one" || w === "and" && last in NUMBER_SCALES;
-        if (isNumberWord) {
+        const isNumberWord2 = w in NUMBER_WORDS || w in NUMBER_SCALES || w === "one" || w === "and" && last in NUMBER_SCALES;
+        if (isNumberWord2) {
           run.push(w);
         } else {
           flush();
@@ -2596,22 +2608,304 @@ var require_scoring = __commonJS({
       flush();
       return { values, parts: [...values, ...parts] };
     }
+    var UNITS = (() => {
+      const table = {};
+      const add = (kind, size, name, words) => {
+        for (const w of words.split(" ")) table[w] = [kind, size, name];
+      };
+      add("mass", 1e-3, "milligram", "mg milligram milligrams");
+      add("mass", 1, "gram", "g gram grams gramme grammes");
+      add("mass", 1e3, "kilogram", "kg kgs kilogram kilograms kilo kilos");
+      add("mass", 453.59237, "pound", "lb lbs pound pounds");
+      add("mass", 28.349523125, "ounce", "oz ounce ounces");
+      add(
+        "length",
+        1e-3,
+        "millimetre",
+        "mm millimetre millimetres millimeter millimeters"
+      );
+      add(
+        "length",
+        0.01,
+        "centimetre",
+        "cm centimetre centimetres centimeter centimeters"
+      );
+      add("length", 1, "metre", "metre metres meter meters");
+      add(
+        "length",
+        1e3,
+        "kilometre",
+        "km kilometre kilometres kilometer kilometers"
+      );
+      add("length", 1609.344, "mile", "mile miles");
+      add("length", 0.3048, "foot", "ft foot feet");
+      add("length", 0.0254, "inch", "inch inches");
+      add("duration", 1, "second", "sec secs second seconds");
+      add("duration", 60, "minute", "min mins minute minutes");
+      add("duration", 3600, "hour", "hr hrs hour hours");
+      add("duration", 86400, "day", "day days");
+      add("duration", 604800, "week", "wk wks week weeks");
+      add("calendar", 1, "month", "month months");
+      add("calendar", 12, "year", "yr yrs year years");
+      add("percent", 1, "percent", "percent");
+      return table;
+    })();
+    var FRACTION_WORDS = {
+      half: 50,
+      quarter: 25,
+      quarters: 25,
+      third: 100 / 3,
+      thirds: 100 / 3
+    };
+    var QUANTITY_TOKEN_RE = new RegExp("\\d+(?:[.,:]\\d+)*|\\p{L}+|%", "gu");
+    function digitValue(raw) {
+      if (raw.includes(":")) return null;
+      if (/^\d{1,3}(,\d{3})+(\.\d+)?$/.test(raw))
+        return Number(raw.replace(/,/g, ""));
+      if (raw.includes(",")) return null;
+      const n = Number(raw);
+      return Number.isFinite(n) ? n : null;
+    }
+    var isNumberWord = (w) => w in NUMBER_WORDS || w in NUMBER_SCALES || w === "one";
+    function quantities(text) {
+      const items = [];
+      const unitWords = /* @__PURE__ */ new Set();
+      const tokens = [...String(text || "").matchAll(QUANTITY_TOKEN_RE)].map(
+        (m) => ({
+          t: m[0].toLowerCase(),
+          start: m.index,
+          end: m.index + m[0].length
+        })
+      );
+      for (const tok of tokens) if (UNITS[tok.t]) unitWords.add(UNITS[tok.t][2]);
+      const unitAt = (j) => {
+        const tok = tokens[j];
+        if (!tok) return null;
+        if (tok.t === "%") return [["percent", 1, "percent"], j];
+        if (tok.t === "per" && tokens[j + 1] && tokens[j + 1].t === "cent") {
+          return [["percent", 1, "percent"], j + 1];
+        }
+        return UNITS[tok.t] ? [UNITS[tok.t], j] : null;
+      };
+      const push = (kind, value, unit, number, from, to, words) => items.push({
+        kind,
+        value,
+        unit,
+        number: String(number),
+        start: tokens[from].start,
+        end: tokens[to].end,
+        words
+      });
+      for (let i = 0; i < tokens.length; i++) {
+        const t = tokens[i].t;
+        if (t === "half" && tokens[i + 1] && (tokens[i + 1].t === "a" || tokens[i + 1].t === "an")) {
+          const u2 = unitAt(i + 2);
+          if (u2 && u2[0][0] !== "percent") {
+            push(u2[0][0], 0.5 * u2[0][1], u2[0][2], 0.5, i, u2[1], true);
+            i = u2[1];
+            continue;
+          }
+        }
+        if (t in FRACTION_WORDS) {
+          const prev = tokens[i - 1] && tokens[i - 1].t;
+          const count = t === "half" ? 1 : prev in NUMBER_WORDS ? NUMBER_WORDS[prev] : prev === "a" || prev === "one" ? 1 : null;
+          if (count !== null && tokens[i + 1] && tokens[i + 1].t === "of") {
+            const from = t === "half" ? i : i - 1;
+            push(
+              "percent",
+              count * FRACTION_WORDS[t],
+              "percent",
+              count * FRACTION_WORDS[t],
+              from,
+              i,
+              true
+            );
+            items[items.length - 1].fraction = true;
+            if (t === "third" || t === "thirds")
+              items[items.length - 1].approx = true;
+            continue;
+          }
+        }
+        let value = null;
+        let end = i;
+        let words = false;
+        if (/^\d/.test(t)) {
+          value = digitValue(t);
+        } else if (isNumberWord(t)) {
+          let j = i;
+          const run = [];
+          while (tokens[j] && (isNumberWord(tokens[j].t) || tokens[j].t === "and" && tokens[j + 1] && isNumberWord(tokens[j + 1].t) && run.length)) {
+            if (tokens[j].t !== "and") run.push(tokens[j].t);
+            j++;
+          }
+          if (run.length) {
+            value = parseNumberWords(run);
+            end = j - 1;
+            words = true;
+          }
+        }
+        if (value === null) continue;
+        let next = end + 1;
+        if (tokens[next] && tokens[next].t === "and" && tokens[next + 1] && tokens[next + 1].t === "a" && tokens[next + 2] && tokens[next + 2].t === "half") {
+          value += 0.5;
+          next += 3;
+        }
+        const u = unitAt(next);
+        if (u) {
+          push(u[0][0], value * u[0][1], u[0][2], value, i, u[1], words);
+          i = u[1];
+        } else {
+          i = end;
+        }
+      }
+      return { items, unitWords };
+    }
+    var docQuantityCache = /* @__PURE__ */ new Map();
+    var claimQuantityCache = /* @__PURE__ */ new Map();
+    function cachedQuantities(cache, text) {
+      const key = String(text || "");
+      let hit = cache.get(key);
+      if (!hit) {
+        hit = quantities(key);
+        if (cache.size >= 256) cache.clear();
+        cache.set(key, hit);
+      }
+      return hit;
+    }
+    var docQuantities = (text) => cachedQuantities(docQuantityCache, text);
+    function sameQuantity(a, b) {
+      const tolerance = a.approx || b.approx ? 1 : Math.max(Math.abs(a.value), Math.abs(b.value)) * 1e-9;
+      return Math.abs(a.value - b.value) <= tolerance;
+    }
+    function clockHours(text) {
+      const hours24 = /* @__PURE__ */ new Set();
+      const hours12 = /* @__PURE__ */ new Set();
+      for (const m of String(text || "").matchAll(
+        // one-digit minutes too ("7:0-23:0", as some listings write hours)
+        /(?<!\d)(\d{1,2}):([0-5]?\d)(?!\d)/g
+      )) {
+        const h = Number(m[1]);
+        if (h > 23) continue;
+        hours24.add(h);
+        hours12.add(h % 12 === 0 ? 12 : h % 12);
+      }
+      return { hours24, hours12 };
+    }
+    function checkClockTimes(claimText, docText) {
+      const spans = [];
+      const matched = [];
+      const missing = [];
+      const { hours24, hours12 } = clockHours(docText);
+      if (hours24.size === 0) return { spans, matched, missing };
+      const tokens = [...String(claimText || "").matchAll(QUANTITY_TOKEN_RE)].map(
+        (m) => ({
+          t: m[0].toLowerCase(),
+          start: m.index,
+          end: m.index + m[0].length
+        })
+      );
+      for (let i = 0; i < tokens.length; i++) {
+        if (!/^\d{1,2}$/.test(tokens[i].t)) continue;
+        const h = Number(tokens[i].t);
+        const next = tokens[i + 1] && tokens[i + 1].t;
+        const prev = tokens[i - 1] && tokens[i - 1].t;
+        if ((next === "am" || next === "pm") && h >= 1 && h <= 12) {
+          const h24 = h % 12 + (next === "pm" ? 12 : 0);
+          const span = [tokens[i].start, tokens[i + 1].end];
+          if (hours24.has(h24)) matched.push(span);
+          else missing.push(`time:${h}${next}`);
+          spans.push(span);
+        } else if (["at", "from", "to", "until", "till"].includes(prev) && h >= 1 && h <= 12 && // "open from 9 to 5.", not "up to 8 people" or "at 11 Main Street"
+        endsTime(claimText, tokens[i].end, next)) {
+          const span = [tokens[i].start, tokens[i].end];
+          if (hours12.has(h) || hours24.has(h)) matched.push(span);
+          else missing.push(`time:${h}`);
+          spans.push(span);
+        }
+      }
+      return { spans, matched, missing };
+    }
+    var AFTER_HOUR = new Set(
+      "to until till and or on every each daily sharp then".split(" ")
+    );
+    function endsTime(text, end, next) {
+      if (next === void 0 || /^\s*[.,;:!?)]/.test(String(text).slice(end, end + 3))) {
+        return true;
+      }
+      return AFTER_HOUR.has(next);
+    }
+    function withoutSpans(text, spans) {
+      let out = String(text || "");
+      for (const [start, end] of [...spans].sort((a, b) => b[0] - a[0])) {
+        out = out.slice(0, start) + " ".repeat(end - start) + out.slice(end);
+      }
+      return out;
+    }
+    function footnotedYears(text) {
+      const out = [];
+      for (const m of String(text || "").matchAll(
+        new RegExp("(?<![\\d.])(1[89]\\d\\d|20\\d\\d|2100)\\.\\d{1,2}(?=\\s+\\p{Lu})", "gu")
+      )) {
+        out.push(String(Number(m[1])));
+      }
+      return out;
+    }
     function checkNumericConsistency(claimText, docText) {
-      const claimNumbers = [
-        ...new Set(extractNumbers(withoutReferences(claimText)).values),
+      const claim = withoutReferences(claimText);
+      const docParts = /* @__PURE__ */ new Set([
+        ...extractNumbers(docText).parts,
+        ...footnotedYears(docText)
+      ]);
+      const doc = docQuantities(docText);
+      const spans = [];
+      const matched = [];
+      const missing = [];
+      const claimNumbers = [];
+      for (const q of cachedQuantities(claimQuantityCache, claim).items) {
+        const sameKind = doc.items.filter((d) => d.kind === q.kind);
+        const equal = sameKind.filter((d) => sameQuantity(d, q));
+        const found = equal.length > 0 || // the number with the same unit word elsewhere ("20 days" in "20
+        // vacation days"), or with no other unit of this kind ("seven days a
+        // week" against a source with "7:00")
+        docParts.has(q.number) && (doc.unitWords.has(q.unit) || !sameKind.some((d) => d.number === q.number));
+        if (sameKind.length === 0 && !q.words && !found) continue;
+        spans.push([q.start, q.end]);
+        const converted = (d) => d.unit !== q.unit || Boolean(d.fraction) !== Boolean(q.fraction);
+        if (equal.length > 0 && equal.every(converted)) {
+          matched.push([q.start, q.end]);
+        }
+        claimNumbers.push(`${q.kind}:${q.number} ${q.unit}`);
+        if (!found && (sameKind.length > 0 || !q.words))
+          missing.push(`${q.kind}:${q.number} ${q.unit}`);
+      }
+      const clock = checkClockTimes(claim, docText);
+      spans.push(...clock.spans);
+      matched.push(...clock.matched);
+      missing.push(...clock.missing);
+      const plain = [
+        ...new Set(extractNumbers(withoutSpans(claim, spans)).values),
         ...claimMonths(claimText).map((m) => `month:${m}`),
         ...identifiers(claimText)
       ];
+      claimNumbers.push(...plain, ...clock.missing);
+      const matchedWords = [
+        ...new Set(matched.flatMap(([a, b]) => tokenize(claim.slice(a, b))))
+      ];
       if (claimNumbers.length === 0) {
-        return { consistent: true, claimNumbers, missing: [] };
+        return { consistent: true, claimNumbers, missing: [], matchedWords };
       }
       const docNumbers = /* @__PURE__ */ new Set([
-        ...extractNumbers(docText).parts,
+        ...docParts,
         ...docMonths(docText).map((m) => `month:${m}`),
         ...identifiers(docText)
       ]);
-      const missing = claimNumbers.filter((n) => !docNumbers.has(n));
-      return { consistent: missing.length === 0, claimNumbers, missing };
+      missing.push(...plain.filter((n) => !docNumbers.has(n)));
+      return {
+        consistent: missing.length === 0,
+        claimNumbers,
+        missing,
+        matchedWords
+      };
     }
     var REFERENCE_WORDS = "document|documents|doc|docs|source|sources|table|tables|figure|figures|fig|section|sections|step|steps|study|studies|appendix|chapter|page|pages|claim|claims|passage|passages|item|items|reference|ref|part|exhibit|equation|eq|line|row|column|note|footnote|paragraph";
     var REFERENCE_RE = new RegExp(
@@ -2687,6 +2981,14 @@ var require_scoring = __commonJS({
         " "
       )
     );
+    var ELLIPTICAL_START = new Set(
+      "not never was were is are be been did does do has have had can could will would should may might must shall".split(" ")
+    );
+    var FRAMING_WORDS = new Set(
+      "according based in on given as from per under after before during".split(
+        " "
+      )
+    );
     function clauseTokens(text) {
       const out = [];
       for (const part of String(text || "").split(/[,;:()[\]]+/)) {
@@ -2729,22 +3031,32 @@ var require_scoring = __commonJS({
           contentTokens(tokens.filter((t) => !NEGATION_WORDS.has(t))).map(stem)
         )
       ];
-      const claimClauses = clauseTokens(claimText).map((tokens) => ({ tokens, words: words(tokens) })).filter((c) => c.words.length > 0);
+      let claimClauses = clauseTokens(claimText).map((tokens) => ({ tokens, words: words(tokens) })).filter((c) => c.words.length > 0);
+      if (claimClauses.length > 1 && FRAMING_WORDS.has(claimClauses[0].tokens[0]) && !isNegated(claimClauses[0].tokens)) {
+        claimClauses = claimClauses.slice(1);
+      }
       const claimNegated = claimClauses.some((c) => isNegated(c.tokens));
       if (claimClauses.length === 0) {
         return { consistent: true, claimNegated, sourceSentence: null };
       }
       const sentences = splitSentences(docText).map((sentence) => {
-        const clauses = clauseTokens(sentence).map((tokens) => ({
+        const clauses = clauseTokens(sentence).map((tokens, i) => ({
           negated: isNegated(tokens),
-          stems: new Set(tokens.map(stem))
+          stems: new Set(tokens.map(stem)),
+          // "... but was not better than Y", "... but did not reduce Y": no
+          // subject of its own, so it continues the first clause's (3.7)
+          continues: i > 0 && ELLIPTICAL_START.has(tokens[0])
         }));
         const stems = new Set(clauses.flatMap((c) => [...c.stems]));
         return { sentence, clauses, stems };
       });
       const agrees = (clause, sentence) => {
         const negated = isNegated(clause.tokens);
-        const distinctive = clause.words.map((w) => sentence.clauses.filter((c) => c.stems.has(w))).filter((owners) => owners.length === 1).map((owners) => owners[0]);
+        let distinctive = clause.words.map((w) => sentence.clauses.filter((c) => c.stems.has(w))).filter((owners) => owners.length === 1).map((owners) => owners[0]);
+        const first = sentence.clauses[0];
+        if (distinctive.some((c) => c.continues)) {
+          distinctive = distinctive.filter((c) => c !== first);
+        }
         if (distinctive.length > 0) {
           return distinctive.every((c) => c.negated === negated);
         }
@@ -2784,15 +3096,25 @@ var require_scoring = __commonJS({
       const docSet = docTokens instanceof Set ? docTokens : new Set(docTokens);
       let matched = 0;
       let total = 0;
+      const unmatched = [];
       for (const token of claim) {
         const weight = idfWeights ? idfWeights.get(token) ?? 1 : 1;
         total += weight;
         if (docSet.has(token)) matched += weight;
+        else unmatched.push([token, weight]);
       }
-      let score = total > 0 ? matched / total : 0;
+      if (matched === 0) return 0;
       const { claimText, docText } = options || {};
-      if (score > 0 && typeof claimText === "string" && typeof docText === "string") {
-        const numeric = checkNumericConsistency(claimText, docText);
+      const texts = typeof claimText === "string" && typeof docText === "string";
+      const numeric = texts ? checkNumericConsistency(claimText, docText) : null;
+      if (numeric && numeric.matchedWords.length) {
+        const restated = new Set(numeric.matchedWords);
+        for (const [token, weight] of unmatched) {
+          if (restated.has(token)) matched += weight;
+        }
+      }
+      let score = total > 0 ? matched / total : 0;
+      if (texts) {
         if (!numeric.consistent) {
           const share = (numeric.claimNumbers.length - numeric.missing.length) / numeric.claimNumbers.length;
           score *= NUMERIC_MISMATCH_FACTOR * share;
@@ -5352,6 +5674,8 @@ var require_connector_utils = __commonJS({
         `${provider} API key required. Set ${envVar} or pass the apiKey option.`
       );
       err.code = "MISSING_API_KEY";
+      err.provider = provider;
+      err.envVar = envVar;
       return err;
     }
     function signalOptions(signal) {
@@ -5411,6 +5735,7 @@ var require_connector_utils = __commonJS({
         message = `${provider} request failed: ${err.message}`;
       }
       const error = new Error(message);
+      error.provider = provider;
       if (err.code) error.code = err.code;
       if (response) {
         error.status = response.status;

package/dist/types/cli/services/instrument.d.ts

@@ -1,6 +1,8 @@
 /**
  * @param {Object} components - { embedder, retriever, reranker, llm }
- * @returns {{ components: Object, timings: Array<{stage: string, ms: number}>, captured: { results: Array|null } }}
+ * @returns {{ components: Object, timings: Array<{stage: string, ms: number}>, captured: { results: Array|null, error: * } }}
+ *   `captured.error` is the last error a stage threw: the pipeline reports
+ *   only its message, and the CLI words a refused key from the error itself.
  */
 export function instrumentComponents(components: any): {
     components: any;
@@ -10,5 +12,6 @@ export function instrumentComponents(components: any): {
     }>;
     captured: {
         results: any[] | null;
+        error: any;
     };
 };

package/dist/types/cli/services/rag-service.d.ts

@@ -94,7 +94,8 @@ export function query({ config, configDir, question, topK, components, timeout,
  * @param {object} params
  * @param {object} params.config
  * @param {string} params.configDir
- * @returns {{ pipeline: object, hasLlm: boolean }}
+ * @returns {{ pipeline: object, hasLlm: boolean, captured: { error: * } }}
+ *   `captured.error` is the last error a stage threw (see instrument.js)
  */
 export function createPipeline({ config, configDir }: {
     config: object;
@@ -102,6 +103,9 @@ export function createPipeline({ config, configDir }: {
 }): {
     pipeline: object;
     hasLlm: boolean;
+    captured: {
+        error: any;
+    };
 };
 /**
  * Retrieval only (no llm): embed the question, retrieve and, if configured,

package/dist/types/config/component-factory.d.ts

@@ -42,6 +42,7 @@ export namespace BUILT_INS {
         memory: (cfg: any) => import("../connectors/memory-retriever").MemoryRetriever;
         pgvector: (cfg: any) => import("../retrieval/pgvector-retriever").PgVectorRetriever;
         qdrant: (cfg: any) => import("../retrieval/qdrant-retriever").QdrantRetriever;
+        weaviate: (cfg: any) => import("../retrieval/weaviate-retriever").WeaviateRetriever;
         pinecone: (cfg: any) => import("../retrieval/pinecone-retriever").PineconeRetriever;
     };
     namespace llm {

package/dist/types/connectors/openai-connector.d.ts

@@ -62,6 +62,13 @@ export class OpenAIConnector extends BaseConnector {
      * @private
      */
     private _request;
+    /**
+     * Name the provider on an HTTP error from the SDK, as the axios-based
+     * connectors do, so a caller such as the CLI can say which key was refused.
+     * A custom baseURL never reads OPENAI_API_KEY, so it gets no envVar.
+     * @private
+     */
+    private _tagError;
     /**
      * Generate a chat completion using the provided query and optional RAG context.
      *

package/dist/types/retrieval/pinecone-retriever.d.ts

@@ -1,8 +1,9 @@
 /**
  * Retriever on a Pinecone serverless index over its REST API, with no
- * client package. The index must exist, with a dense vector of your
- * embedding model's dimension; use the cosine metric for the scores the
- * other retrievers return.
+ * client package. The index must exist: with a dense vector of your
+ * embedding model's dimension, or with a text field Pinecone embeds itself
+ * (then documents need no vector and retrieve() takes the query text). Use
+ * the cosine metric for the scores the other retrievers return.
  */
 export class PineconeRetriever {
     /**
@@ -13,6 +14,7 @@ export class PineconeRetriever {
      * @param {string} [options.namespace='rag_documents'] - A namespace this retriever owns
      * @param {'auto'|'documents'|'vectors'} [options.api='auto'] - Document index (Documents API) or vector index; auto reads it from the index description
      * @param {string} [options.vectorField] - The document index's dense_vector field (default: the schema's)
+     * @param {string} [options.textField] - The document index's text field that Pinecone embeds (integrated embedding; default: the schema's, when the index has no dense_vector field)
      * @param {number} [options.freshnessTimeoutMs=30000] - How long a read waits for this retriever's writes to be indexed; 0 to not wait
      * @param {string} [options.cacheKey] - Cache identity; change it when the index changes
      * @param {Function} [options.fetch] - fetch implementation (defaults to the global)
@@ -24,6 +26,7 @@ export class PineconeRetriever {
         namespace?: string;
         api?: "auto" | "documents" | "vectors";
         vectorField?: string;
+        textField?: string;
         freshnessTimeoutMs?: number;
         cacheKey?: string;
         fetch?: Function;
@@ -36,6 +39,7 @@ export class PineconeRetriever {
     _host: string;
     _api: "auto" | "documents" | "vectors";
     _vectorField: string;
+    _textField: string;
     _fetch: any;
     _resolving: any;
     _writeLsn: number;
@@ -47,14 +51,28 @@ export class PineconeRetriever {
      * @returns {string|null}
      */
     getCacheKey(role: "retriever"): string | null;
+    /**
+     * Whether the index embeds text itself (integrated embedding): then
+     * store() needs no vectors and retrieve() takes the query text, so no
+     * embedder is needed. The CLI checks this before requiring an embedder.
+     * Reads the index description on first use.
+     * @param {{ signal?: AbortSignal }} [options]
+     * @returns {Promise<boolean>}
+     */
+    embedsText({ signal }?: {
+        signal?: AbortSignal;
+    }): Promise<boolean>;
     /**
      * One REST call; resolves the parsed body and the response headers.
      * @private
      */
     private _call;
+    /** True on a document index that embeds text itself. @private */
+    private get _integrated();
     /**
      * The index's host and kind, from its description (once): a document
-     * index's dense_vector field and dimension, or a vector index's.
+     * index's dense_vector field and dimension or the text field it embeds,
+     * or a vector index's dimension.
      * @private
      */
     private _resolve;
@@ -68,32 +86,47 @@ export class PineconeRetriever {
     /**
      * A search, repeated until it reflects this retriever's writes (when the
      * index reports log sequence numbers) or `freshnessTimeoutMs` passes.
-     * Resolves `[{ id, score, fields }]`.
+     * `query` is a vector, or the query text on an index that embeds text
+     * itself. Resolves `[{ id, score, fields }]`.
      * @private
      */
     private _search;
     /** Waits until this retriever's writes are indexed (for list and size). @private */
     private _waitForWrites;
     /**
-     * Inserts or replaces documents.
-     * @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
+     * What list() and size() search with: a unit vector of the index's
+     * dimension, or a fixed text on an index that embeds text itself.
+     * @private
+     */
+    private _probe;
+    /**
+     * Inserts or replaces documents. On an index that embeds text itself,
+     * Pinecone embeds `content` (required there) and `vector` is ignored.
+     * @param {Array<{ id: string, vector?: number[], content?: string, metadata?: Object }>} docs
+     *   `vector` is required unless the index embeds text itself
      * @returns {Promise<void>}
      */
     store(docs: Array<{
         id: string;
-        vector: number[];
+        vector?: number[];
         content?: string;
         metadata?: any;
     }>): Promise<void>;
+    /** store() on an index that embeds text itself. @private */
+    private _storeText;
     /**
-     * The documents most similar to a query vector.
-     * @param {number[]|{ queryVector?: number[], topK?: number, signal?: AbortSignal }} request
-     *   A vector, or an object with one (required; optional in the type to
-     *   match the retriever contract)
-     * @param {number} [k=10] - Results when `request` is a vector
+     * The documents most similar to a query vector, or to the query text on
+     * an index that embeds text itself (Pinecone embeds the query; a
+     * `queryVector` is then ignored).
+     * @param {number[]|string|{ query?: string, queryVector?: number[], topK?: number, signal?: AbortSignal }} request
+     *   A vector, the query text, or an object with them: `queryVector`, or
+     *   `query` on an index that embeds text itself (required; optional in
+     *   the type to match the retriever contract)
+     * @param {number} [k=10] - Results when `request` is a vector or text
      * @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
      */
-    retrieve(request: number[] | {
+    retrieve(request: number[] | string | {
+        query?: string;
         queryVector?: number[];
         topK?: number;
         signal?: AbortSignal;

package/dist/types/retrieval/weaviate-retriever.d.ts

@@ -0,0 +1,120 @@
+/**
+ * Retriever on Weaviate over its REST and GraphQL APIs, with no client
+ * package. Creates the collection on first store; similarity is cosine.
+ */
+export class WeaviateRetriever {
+    /**
+     * @param {Object} [options]
+     * @param {string} [options.url='http://localhost:8080'] - Weaviate's REST endpoint (without /v1)
+     * @param {string} [options.apiKey] - Sent as `Authorization: Bearer <key>`
+     * @param {string} [options.collection='RagDocuments'] - Class name; the first letter is upper-cased, as Weaviate does
+     * @param {number} [options.dimensions] - Expected vector size; defaults to the collection's or the first stored vector's
+     * @param {boolean} [options.createCollection=true] - Create the collection (and any missing properties) on first store
+     * @param {string} [options.cacheKey] - Cache identity; change it when the collection changes
+     * @param {Function} [options.fetch] - fetch implementation (defaults to the global)
+     */
+    constructor(options?: {
+        url?: string;
+        apiKey?: string;
+        collection?: string;
+        dimensions?: number;
+        createCollection?: boolean;
+        cacheKey?: string;
+        fetch?: Function;
+    });
+    url: any;
+    collection: string;
+    dimensions: number;
+    createCollection: boolean;
+    cacheKey: string;
+    _apiKey: string;
+    _fetch: any;
+    _ready: any;
+    _exists: boolean;
+    /**
+     * Cache identity for createRagPipeline({ cache }): none unless you set
+     * `cacheKey`, since other clients can change the collection.
+     * @param {'retriever'} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: "retriever"): string | null;
+    /** The message without the API key, should a server ever echo it. */
+    _redact(message: any): string;
+    /**
+     * One REST call; resolves the parsed body.
+     * @private
+     * @param {string} method
+     * @param {string} path - After /v1
+     * @param {Object} [body]
+     * @param {{ signal?: AbortSignal, allow404?: boolean }} [options] - allow404 resolves null on 404
+     * @returns {Promise<any>}
+     */
+    private _request;
+    /** A GraphQL query; resolves `data`, throws on GraphQL errors. */
+    _graphql(query: any, signal: any): Promise<any>;
+    /** The collection's schema, or null when it doesn't exist. */
+    _schema(signal: any): Promise<any>;
+    /** Whether the collection exists; only a yes is remembered. */
+    _collectionExists(signal: any): Promise<boolean>;
+    _ensureCollection(): any;
+    /**
+     * Inserts or replaces documents.
+     * @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
+     * @returns {Promise<void>}
+     */
+    store(docs: Array<{
+        id: string;
+        vector: number[];
+        content?: string;
+        metadata?: any;
+    }>): Promise<void>;
+    /**
+     * The documents most similar to a query vector.
+     * @param {number[]|{ queryVector?: number[], topK?: number, signal?: AbortSignal }} request
+     *   A vector, or an object with one (required; optional in the type to
+     *   match the retriever contract)
+     * @param {number} [k=10] - Results when `request` is a vector
+     * @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
+     */
+    retrieve(request: number[] | {
+        queryVector?: number[];
+        topK?: number;
+        signal?: AbortSignal;
+    }, k?: number): Promise<Array<{
+        id: string;
+        score: number;
+        content: string;
+        metadata: any;
+    }>>;
+    /**
+     * Deletes the matching objects, a server-limited round at a time.
+     * @private
+     */
+    private _deleteWhere;
+    /**
+     * Deletes the chunks of one source (`metadata.source`), for re-ingesting.
+     * @param {string} source
+     */
+    removeBySource(source: string): Promise<void>;
+    /**
+     * Deletes documents by id.
+     * @param {string[]} ids
+     */
+    delete(ids: string[]): Promise<void>;
+    /** Deletes every document this class stored; the collection stays. */
+    clear(): Promise<void>;
+    /** @returns {Promise<number>} */
+    size(): Promise<number>;
+    /**
+     * Every document, without vectors.
+     * @param {{ limit?: number }} [options]
+     * @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
+     */
+    list({ limit }?: {
+        limit?: number;
+    }): Promise<Array<{
+        id: string;
+        content: string;
+        metadata: any;
+    }>>;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devilsdev/rag-core",
-  "version": "3.6.0",
+  "version": "3.7.0",
   "description": "createRagPipeline and its building blocks: retrieval, reranking, caching, cost tracking and model routing.",
   "keywords": [
     "rag",
@@ -13,7 +13,7 @@
   "homepage": "https://devilsdev.github.io/rag-pipeline-utils/",
   "repository": {
     "type": "git",
-    "url": "https://github.com/DevilsDev/rag-pipeline-utils.git"
+    "url": "git+https://github.com/DevilsDev/rag-pipeline-utils.git"
   },
   "bugs": {
     "url": "https://github.com/DevilsDev/rag-pipeline-utils/issues"