@devilsdev/rag-core 3.5.0 → 3.6.0

package/dist/all.d.ts

@@ -531,6 +531,7 @@ export { LocalEmbedder } from "./types/connectors/local-embedder";
 export { MemoryRetriever } from "./types/connectors/memory-retriever";
 export { PgVectorRetriever } from "./types/retrieval/pgvector-retriever";
 export { QdrantRetriever } from "./types/retrieval/qdrant-retriever";
+export { PineconeRetriever } from "./types/retrieval/pinecone-retriever";
 export { OpenAIConnector } from "./types/connectors/openai-connector";
 export { AnthropicConnector } from "./types/connectors/anthropic-connector";
 export { CohereConnector } from "./types/connectors/cohere-connector";

package/dist/index.cjs

@@ -1358,7 +1358,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@devilsdev/rag-pipeline-utils",
-      version: "3.5.0",
+      version: "3.6.0",
       description: "Composable retrieval-augmented generation (RAG) for Node.js: grounded citations, evaluation, guardrails, hybrid retrieval and a CLI.",
       main: "dist/index.cjs",
       type: "commonjs",
@@ -1376,6 +1376,7 @@ var require_package = __commonJS({
       exports: {
         ".": {
           types: "./dist/index.d.ts",
+          workerd: "./dist/index.cjs",
           require: "./dist/index.cjs",
           import: "./dist/index.mjs"
         },
@@ -1404,6 +1405,8 @@ var require_package = __commonJS({
       scripts: {
         build: "node scripts/build-packages.js --bundles && node scripts/build.js && node scripts/generate-types.js && node scripts/build-packages.js --types && node scripts/check-type-exports.js",
         "test:packages": "node scripts/test-packages.js",
+        "test:consumer": "node scripts/consumer-journeys.js",
+        "test:templates": "node scripts/test-templates.js",
         prepack: "npm run build && node scripts/pin-readme-assets.js",
         postpack: "node scripts/pin-readme-assets.js --restore",
         "docs:validate": "node scripts/validate-docs-examples.js",
@@ -1459,6 +1462,7 @@ var require_package = __commonJS({
         "bench:claude-citations": "node benchmarks/claude-citations.js",
         "bench:judge-more": "node benchmarks/judge-more.js",
         "bench:claude-latency": "node benchmarks/claude-latency.js",
+        "claude:usage": "node scripts/claude-usage-report.js && node benchmarks/render.js",
         dev: "node scripts/dev-tools.js menu",
         "dev:health": "node scripts/dev-tools.js health",
         "dev:setup": "node scripts/dev-tools.js setup",
@@ -1622,10 +1626,10 @@ var require_package = __commonJS({
         "why-is-node-running": "^3.2.2"
       },
       dependencies: {
-        "@devilsdev/rag-connectors": "3.5.0",
-        "@devilsdev/rag-core": "3.5.0",
-        "@devilsdev/rag-eval": "3.5.0",
-        "@devilsdev/rag-guardrails": "3.5.0",
+        "@devilsdev/rag-connectors": "3.6.0",
+        "@devilsdev/rag-core": "3.6.0",
+        "@devilsdev/rag-eval": "3.6.0",
+        "@devilsdev/rag-guardrails": "3.6.0",
         ajv: "^8.17.1",
         axios: "^1.21.0",
         chokidar: "^4.0.3",
@@ -1695,12 +1699,18 @@ var require_optional_peer = __commonJS({
         throw error;
       }
     }
+    function missingPeer(error, name, hint) {
+      if (!isMissing(error, name)) return error;
+      const err = new Error(hint);
+      err.code = "MISSING_DEPENDENCY";
+      return err;
+    }
     function isMissing(error, name) {
       if (!error || error.code !== "MODULE_NOT_FOUND") return false;
       const message = String(error.message || "");
       return message.startsWith(`Cannot find module '${name}'`) || message.startsWith(`Cannot find module "${name}"`);
     }
-    module2.exports = { loadOptionalPeer };
+    module2.exports = { loadOptionalPeer, missingPeer };
   }
 });
 
@@ -1710,16 +1720,19 @@ var require_otel = __commonJS({
     "use strict";
     var { evaluationScores } = require_scores();
     var { version: LIBRARY_VERSION } = require_package();
-    var { loadOptionalPeer } = require_optional_peer();
+    var { missingPeer } = require_optional_peer();
     var TRACER_NAME = "@devilsdev/rag-pipeline-utils";
     var PROVIDERS = /* @__PURE__ */ new Set(["openai", "anthropic", "cohere", "ollama"]);
     function loadOtelApi() {
-      return loadOptionalPeer(
-        "@opentelemetry/api",
-        () => require("@opentelemetry/api"),
-        // eslint-disable-line global-require
-        'telemetry needs the "@opentelemetry/api" package, an optional peer dependency: npm install @opentelemetry/api (with @devilsdev/rag-pipeline-utils/edge, pass it: telemetry: { api })'
-      );
+      try {
+        return require("@opentelemetry/api");
+      } catch (error) {
+        throw missingPeer(
+          error,
+          "@opentelemetry/api",
+          'telemetry needs the "@opentelemetry/api" package, an optional peer dependency: npm install @opentelemetry/api (with @devilsdev/rag-pipeline-utils/edge, pass it: telemetry: { api })'
+        );
+      }
     }
     function createTelemetry(option) {
       if (!option) return null;
@@ -2584,17 +2597,114 @@ var require_scoring = __commonJS({
       return { values, parts: [...values, ...parts] };
     }
     function checkNumericConsistency(claimText, docText) {
-      const claimNumbers = [...new Set(extractNumbers(claimText).values)];
+      const claimNumbers = [
+        ...new Set(extractNumbers(withoutReferences(claimText)).values),
+        ...claimMonths(claimText).map((m) => `month:${m}`),
+        ...identifiers(claimText)
+      ];
       if (claimNumbers.length === 0) {
         return { consistent: true, claimNumbers, missing: [] };
       }
-      const docNumbers = new Set(extractNumbers(docText).parts);
+      const docNumbers = /* @__PURE__ */ new Set([
+        ...extractNumbers(docText).parts,
+        ...docMonths(docText).map((m) => `month:${m}`),
+        ...identifiers(docText)
+      ]);
       const missing = claimNumbers.filter((n) => !docNumbers.has(n));
       return { consistent: missing.length === 0, claimNumbers, missing };
     }
+    var REFERENCE_WORDS = "document|documents|doc|docs|source|sources|table|tables|figure|figures|fig|section|sections|step|steps|study|studies|appendix|chapter|page|pages|claim|claims|passage|passages|item|items|reference|ref|part|exhibit|equation|eq|line|row|column|note|footnote|paragraph";
+    var REFERENCE_RE = new RegExp(
+      `\\b(${REFERENCE_WORDS})\\.?\\s+\\d+[a-z]?\\b`,
+      "giu"
+    );
+    var IDENTIFIER_RE = new RegExp("(\\p{L}[\\p{L}\\p{N}]*)-(\\d+)\\b", "gu");
+    function identifiers(text) {
+      if (!text || typeof text !== "string") return [];
+      return [
+        ...new Set(
+          [...text.matchAll(IDENTIFIER_RE)].map((m) => `id:${m[0].toLowerCase()}`)
+        )
+      ];
+    }
+    function withoutReferences(text) {
+      if (!text || typeof text !== "string") return text;
+      return text.replace(REFERENCE_RE, " ").replace(IDENTIFIER_RE, (_, word) => word);
+    }
+    var MONTHS = [
+      ["january", "jan"],
+      ["february", "feb"],
+      ["march", "mar"],
+      ["april", "apr"],
+      ["may"],
+      ["june", "jun"],
+      ["july", "jul"],
+      ["august", "aug"],
+      ["september", "sep", "sept"],
+      ["october", "oct"],
+      ["november", "nov"],
+      ["december", "dec"]
+    ];
+    var MONTH_OF = new Map(
+      MONTHS.flatMap((names, i) => names.map((n) => [n, i + 1]))
+    );
+    function claimMonths(text) {
+      const words = String(text || "").toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean);
+      const out = [];
+      words.forEach((w, i) => {
+        const m = MONTH_OF.get(w);
+        if (!m) return;
+        const near = [words[i - 1], words[i + 1]].some(
+          (x) => /^\d+$/.test(x || "")
+        );
+        if (near) out.push(m);
+      });
+      return [...new Set(out)];
+    }
+    function docMonths(text) {
+      const t = String(text || "");
+      const out = /* @__PURE__ */ new Set();
+      for (const w of t.toLowerCase().split(/[^\p{L}]+/u)) {
+        const m = MONTH_OF.get(w);
+        if (m) out.add(m);
+      }
+      for (const [, a, b] of t.matchAll(
+        /\b(\d{1,2})[/.-](\d{1,2})[/.-]\d{2,4}\b/g
+      )) {
+        for (const v of [Number(a), Number(b)]) if (v >= 1 && v <= 12) out.add(v);
+      }
+      for (const [, mm] of t.matchAll(/\b\d{4}-(\d{1,2})-\d{1,2}\b/g)) {
+        const v = Number(mm);
+        if (v >= 1 && v <= 12) out.add(v);
+      }
+      return [...out];
+    }
     var NEGATION_WORDS = new Set(
-      "not no never none nobody nothing nowhere neither nor cannot without dont doesnt didnt isnt arent wasnt werent wont wouldnt shouldnt couldnt cant hasnt havent hadnt mustnt neednt".split(" ")
+      "not no never none nobody nothing nowhere neither nor cannot without dont doesnt didnt isnt arent wasnt werent wont wouldnt shouldnt couldnt cant hasnt havent hadnt mustnt neednt nicht kein keine keinen keinem keiner keines nie niemals nunca jam\xE1s jamas nadie nada tampoco ni pas jamais rien aucun aucune".split(" ")
+    );
+    var CLAUSE_BOUNDARY_WORDS = new Set(
+      "but whereas although though while however yet and or except unlike".split(
+        " "
+      )
     );
+    function clauseTokens(text) {
+      const out = [];
+      for (const part of String(text || "").split(/[,;:()[\]]+/)) {
+        let current = [];
+        for (const t of tokenize(part)) {
+          if (CLAUSE_BOUNDARY_WORDS.has(t)) {
+            if (current.length) out.push(current);
+            current = [];
+          } else {
+            current.push(t);
+          }
+        }
+        if (current.length) out.push(current);
+      }
+      return out;
+    }
+    var stem = (t) => t.length > 4 ? t.replace(/(ing|ed|es|s|e)$/, "") : t;
+    var CONTINUING_NEGATIONS = /* @__PURE__ */ new Set(["nor", "ni"]);
     var NON_NEGATING_NEXT = {
       not: /* @__PURE__ */ new Set(["only", "just", "merely"]),
       no: /* @__PURE__ */ new Set(["doubt", "matter"]),
@@ -2605,6 +2715,7 @@ var require_scoring = __commonJS({
       for (let i = 0; i < tokens.length; i++) {
         const t = tokens[i];
         if (!NEGATION_WORDS.has(t)) continue;
+        if (CONTINUING_NEGATIONS.has(t) && count > 0) continue;
         if (NON_NEGATING_NEXT[t] && NON_NEGATING_NEXT[t].has(tokens[i + 1])) {
           continue;
         }
@@ -2613,38 +2724,62 @@ var require_scoring = __commonJS({
       return count % 2 === 1;
     }
     function checkPolarityConsistency(claimText, docText) {
-      const claimTokens = tokenize(claimText);
-      const claimNegated = isNegated(claimTokens);
-      const claimWords = [
+      const words = (tokens) => [
         ...new Set(
-          contentTokens(claimTokens.filter((t) => !NEGATION_WORDS.has(t)))
+          contentTokens(tokens.filter((t) => !NEGATION_WORDS.has(t))).map(stem)
         )
       ];
-      if (claimWords.length === 0) {
-        return { consistent: true, claimNegated, sourceSentence: null };
-      }
-      const candidates = splitSentences(docText).map((sentence) => {
-        const tokens = tokenize(sentence);
-        const set = new Set(tokens);
-        const support = claimWords.filter((t) => set.has(t)).length / claimWords.length;
-        return { sentence, tokens, support };
-      }).filter((c) => c.support > 0).sort((a, b) => b.support - a.support);
-      if (candidates.length === 0) {
+      const claimClauses = clauseTokens(claimText).map((tokens) => ({ tokens, words: words(tokens) })).filter((c) => c.words.length > 0);
+      const claimNegated = claimClauses.some((c) => isNegated(c.tokens));
+      if (claimClauses.length === 0) {
         return { consistent: true, claimNegated, sourceSentence: null };
       }
-      const best = candidates[0].support;
-      const compared = candidates.filter((c) => c.support === best);
-      const match = compared.find((c) => isNegated(c.tokens) === claimNegated);
-      return {
-        consistent: Boolean(match),
-        claimNegated,
-        sourceSentence: (match || compared[0]).sentence
+      const sentences = splitSentences(docText).map((sentence) => {
+        const clauses = clauseTokens(sentence).map((tokens) => ({
+          negated: isNegated(tokens),
+          stems: new Set(tokens.map(stem))
+        }));
+        const stems = new Set(clauses.flatMap((c) => [...c.stems]));
+        return { sentence, clauses, stems };
+      });
+      const agrees = (clause, sentence) => {
+        const negated = isNegated(clause.tokens);
+        const distinctive = clause.words.map((w) => sentence.clauses.filter((c) => c.stems.has(w))).filter((owners) => owners.length === 1).map((owners) => owners[0]);
+        if (distinctive.length > 0) {
+          return distinctive.every((c) => c.negated === negated);
+        }
+        const ranked = sentence.clauses.map((c) => ({
+          c,
+          support: clause.words.filter((w) => c.stems.has(w)).length
+        })).filter((r) => r.support > 0).sort((a, b) => b.support - a.support);
+        if (ranked.length === 0) return true;
+        const best = ranked[0].support;
+        return ranked.filter((r) => r.support === best).some((r) => r.c.negated === negated);
       };
+      let consistent = true;
+      let sourceSentence = null;
+      for (const clause of claimClauses) {
+        const candidates = sentences.map((s) => ({
+          ...s,
+          support: clause.words.filter((w) => s.stems.has(w)).length / clause.words.length
+        })).filter((s) => s.support > 0).sort((a, b) => b.support - a.support);
+        if (candidates.length === 0) continue;
+        const best = candidates[0].support;
+        const compared = candidates.filter((s) => s.support === best);
+        const match = compared.find((s) => agrees(clause, s));
+        if (!match) consistent = false;
+        if (sourceSentence === null || !match) {
+          sourceSentence = (match || compared[0]).sentence;
+        }
+      }
+      return { consistent, claimNegated, sourceSentence };
     }
     var POLARITY_MISMATCH_FACTOR = 0.4;
     var NUMERIC_MISMATCH_FACTOR = 0.4;
     function computeSupportScore(claimTokens, docTokens, idfWeights = null, options = {}) {
-      const claim = [...new Set(contentTokens(claimTokens))];
+      const rawClaim = options && options.claimText;
+      const fromText = typeof rawClaim === "string" && rawClaim !== withoutReferences(rawClaim) ? tokenize(withoutReferences(rawClaim)) : claimTokens;
+      const claim = [...new Set(contentTokens(fromText))];
       if (claim.length === 0) return 0;
       const docSet = docTokens instanceof Set ? docTokens : new Set(docTokens);
       let matched = 0;

package/dist/types/config/component-factory.d.ts

@@ -42,6 +42,7 @@ export namespace BUILT_INS {
         memory: (cfg: any) => import("../connectors/memory-retriever").MemoryRetriever;
         pgvector: (cfg: any) => import("../retrieval/pgvector-retriever").PgVectorRetriever;
         qdrant: (cfg: any) => import("../retrieval/qdrant-retriever").QdrantRetriever;
+        pinecone: (cfg: any) => import("../retrieval/pinecone-retriever").PineconeRetriever;
     };
     namespace llm {
         export function openai_1(cfg: any): import("../connectors/openai-connector").OpenAIConnector;

package/dist/types/retrieval/pinecone-retriever.d.ts

@@ -0,0 +1,141 @@
+/**
+ * Retriever on a Pinecone serverless index over its REST API, with no
+ * client package. The index must exist, with a dense vector of your
+ * embedding model's dimension; use the cosine metric for the scores the
+ * other retrievers return.
+ */
+export class PineconeRetriever {
+    /**
+     * @param {Object} options
+     * @param {string} [options.apiKey] - Defaults to PINECONE_API_KEY
+     * @param {string} [options.index] - The index name, described on first use
+     * @param {string} [options.host] - The index host (from the console), instead of or with `index`
+     * @param {string} [options.namespace='rag_documents'] - A namespace this retriever owns
+     * @param {'auto'|'documents'|'vectors'} [options.api='auto'] - Document index (Documents API) or vector index; auto reads it from the index description
+     * @param {string} [options.vectorField] - The document index's dense_vector field (default: the schema's)
+     * @param {number} [options.freshnessTimeoutMs=30000] - How long a read waits for this retriever's writes to be indexed; 0 to not wait
+     * @param {string} [options.cacheKey] - Cache identity; change it when the index changes
+     * @param {Function} [options.fetch] - fetch implementation (defaults to the global)
+     */
+    constructor(options?: {
+        apiKey?: string;
+        index?: string;
+        host?: string;
+        namespace?: string;
+        api?: "auto" | "documents" | "vectors";
+        vectorField?: string;
+        freshnessTimeoutMs?: number;
+        cacheKey?: string;
+        fetch?: Function;
+    });
+    index: string;
+    namespace: string;
+    freshnessTimeoutMs: number;
+    cacheKey: string;
+    _apiKey: string;
+    _host: string;
+    _api: "auto" | "documents" | "vectors";
+    _vectorField: string;
+    _fetch: any;
+    _resolving: any;
+    _writeLsn: number;
+    _dimension: any;
+    /**
+     * Cache identity for createRagPipeline({ cache }): none unless you set
+     * `cacheKey`, since other clients can change the index.
+     * @param {'retriever'} role
+     * @returns {string|null}
+     */
+    getCacheKey(role: "retriever"): string | null;
+    /**
+     * One REST call; resolves the parsed body and the response headers.
+     * @private
+     */
+    private _call;
+    /**
+     * The index's host and kind, from its description (once): a document
+     * index's dense_vector field and dimension, or a vector index's.
+     * @private
+     */
+    private _resolve;
+    /** @private */
+    private _describe;
+    _indexDimension: any;
+    /** A data-plane call; records the log sequence number of writes. @private */
+    private _data;
+    /** The Documents API path for this namespace. @private */
+    private _docs;
+    /**
+     * A search, repeated until it reflects this retriever's writes (when the
+     * index reports log sequence numbers) or `freshnessTimeoutMs` passes.
+     * Resolves `[{ id, score, fields }]`.
+     * @private
+     */
+    private _search;
+    /** Waits until this retriever's writes are indexed (for list and size). @private */
+    private _waitForWrites;
+    /**
+     * Inserts or replaces documents.
+     * @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
+     * @returns {Promise<void>}
+     */
+    store(docs: Array<{
+        id: string;
+        vector: number[];
+        content?: string;
+        metadata?: any;
+    }>): Promise<void>;
+    /**
+     * The documents most similar to a query vector.
+     * @param {number[]|{ queryVector?: number[], topK?: number, signal?: AbortSignal }} request
+     *   A vector, or an object with one (required; optional in the type to
+     *   match the retriever contract)
+     * @param {number} [k=10] - Results when `request` is a vector
+     * @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
+     */
+    retrieve(request: number[] | {
+        queryVector?: number[];
+        topK?: number;
+        signal?: AbortSignal;
+    }, k?: number): Promise<Array<{
+        id: string;
+        score: number;
+        content: string;
+        metadata: any;
+    }>>;
+    /** @private */
+    private _delete;
+    /**
+     * Deletes the chunks of one source (`metadata.source`), for re-ingesting.
+     * @param {string} source
+     */
+    removeBySource(source: string): Promise<void>;
+    /**
+     * Deletes documents by id.
+     * @param {string[]} ids
+     */
+    delete(ids: string[]): Promise<void>;
+    /** Deletes every document this retriever stored in its namespace. */
+    clear(): Promise<void>;
+    /**
+     * Every document, without vectors.
+     * @param {{ limit?: number }} [options]
+     * @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
+     */
+    list({ limit }?: {
+        limit?: number;
+    }): Promise<Array<{
+        id: string;
+        content: string;
+        metadata: any;
+    }>>;
+    /** @returns {Promise<number>} The documents this retriever stored */
+    size(): Promise<number>;
+    /** Documents API: fetch by filter, page by page. @private */
+    private _listDocuments;
+    /** Vectors API: list ids, then fetch them. @private */
+    private _listVectors;
+}
+/** The record id for a document id: itself when Pinecone accepts it. */
+export function recordId(id: any): string;
+export const API_VERSION: "2026-07";

package/dist/types/utils/optional-peer.d.ts

@@ -2,7 +2,10 @@
  * Load an optional peer dependency. `load` must be a literal
  * `() => require('name')`, so bundlers (esbuild, Wrangler, webpack) can see
  * and include the package; probing with require.resolve() instead fails in
- * bundles even when the package is there.
+ * bundles even when the package is there. A bundle of a project without the
+ * package fails to build, though, because the require isn't inside a
+ * `try`: loaders on the main entry's path use `missingPeer` with the
+ * require in their own `try` instead.
  *
  * Only the package itself being absent becomes a `MISSING_DEPENDENCY` error
  * carrying `hint`; any other failure, including a module missing inside the
@@ -14,3 +17,20 @@
  * @returns {any} The module
  */
 export function loadOptionalPeer(name: string, load: () => any, hint: string): any;
+/**
+ * For a loader that requires an optional peer inside its own `try`, which
+ * bundlers (esbuild, Wrangler) then include when the package is installed
+ * and leave out, without failing the build, when it isn't:
+ *
+ *   try {
+ *     return require('openai');
+ *   } catch (error) {
+ *     throw missingPeer(error, 'openai', hint);
+ *   }
+ *
+ * @param {Error} error - What require() threw
+ * @param {string} name - The package name
+ * @param {string} hint - The error message when the package is missing
+ * @returns {Error} A `MISSING_DEPENDENCY` error carrying `hint` when the package itself is absent; otherwise `error`
+ */
+export function missingPeer(error: Error, name: string, hint: string): Error;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devilsdev/rag-core",
-  "version": "3.5.0",
+  "version": "3.6.0",
   "description": "createRagPipeline and its building blocks: retrieval, reranking, caching, cost tracking and model routing.",
   "keywords": [
     "rag",
@@ -26,6 +26,7 @@
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
+      "workerd": "./dist/index.cjs",
       "require": "./dist/index.cjs",
       "import": "./dist/index.mjs"
     },