@devilsdev/rag-core 3.5.0 → 3.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/all.d.ts +2 -0
- package/dist/index.cjs +531 -71
- package/dist/types/cli/services/instrument.d.ts +4 -1
- package/dist/types/cli/services/rag-service.d.ts +5 -1
- package/dist/types/config/component-factory.d.ts +2 -0
- package/dist/types/connectors/openai-connector.d.ts +7 -0
- package/dist/types/retrieval/pinecone-retriever.d.ts +174 -0
- package/dist/types/retrieval/weaviate-retriever.d.ts +120 -0
- package/dist/types/utils/optional-peer.d.ts +21 -1
- package/package.json +3 -2
package/dist/all.d.ts
CHANGED
|
@@ -531,6 +531,8 @@ export { LocalEmbedder } from "./types/connectors/local-embedder";
|
|
|
531
531
|
export { MemoryRetriever } from "./types/connectors/memory-retriever";
|
|
532
532
|
export { PgVectorRetriever } from "./types/retrieval/pgvector-retriever";
|
|
533
533
|
export { QdrantRetriever } from "./types/retrieval/qdrant-retriever";
|
|
534
|
+
export { PineconeRetriever } from "./types/retrieval/pinecone-retriever";
|
|
535
|
+
export { WeaviateRetriever } from "./types/retrieval/weaviate-retriever";
|
|
534
536
|
export { OpenAIConnector } from "./types/connectors/openai-connector";
|
|
535
537
|
export { AnthropicConnector } from "./types/connectors/anthropic-connector";
|
|
536
538
|
export { CohereConnector } from "./types/connectors/cohere-connector";
|
package/dist/index.cjs
CHANGED
|
@@ -59,32 +59,44 @@ var require_logger = __commonJS({
|
|
|
59
59
|
return false;
|
|
60
60
|
}
|
|
61
61
|
};
|
|
62
|
-
var
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
62
|
+
var getDestination = (pretty) => {
|
|
63
|
+
if (pretty || typeof pino.destination !== "function") return void 0;
|
|
64
|
+
const stdout = process.stdout;
|
|
65
|
+
if (!stdout || stdout.write !== stdout.constructor.prototype.write) {
|
|
66
|
+
return void 0;
|
|
67
|
+
}
|
|
68
|
+
return pino.destination({ dest: stdout.fd || 1, sync: true });
|
|
69
|
+
};
|
|
70
|
+
var prettyPrint = shouldUsePrettyPrint();
|
|
71
|
+
var baseLogger = pino(
|
|
72
|
+
{
|
|
73
|
+
name: "rag-pipeline-utils",
|
|
74
|
+
level: getLogLevel(),
|
|
75
|
+
formatters: {
|
|
76
|
+
level: (label) => {
|
|
77
|
+
return { level: label };
|
|
78
|
+
},
|
|
79
|
+
log: (object) => {
|
|
80
|
+
if (object.correlationId) {
|
|
81
|
+
object.correlationId = object.correlationId;
|
|
82
|
+
}
|
|
83
|
+
return object;
|
|
84
|
+
}
|
|
68
85
|
},
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
86
|
+
// Pretty print in development or when LOG_PRETTY=true
|
|
87
|
+
...prettyPrint && {
|
|
88
|
+
transport: {
|
|
89
|
+
target: "pino-pretty",
|
|
90
|
+
options: {
|
|
91
|
+
colorize: true,
|
|
92
|
+
translateTime: "HH:MM:ss",
|
|
93
|
+
ignore: "pid,hostname"
|
|
94
|
+
}
|
|
72
95
|
}
|
|
73
|
-
return object;
|
|
74
96
|
}
|
|
75
97
|
},
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
transport: {
|
|
79
|
-
target: "pino-pretty",
|
|
80
|
-
options: {
|
|
81
|
-
colorize: true,
|
|
82
|
-
translateTime: "HH:MM:ss",
|
|
83
|
-
ignore: "pid,hostname"
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
});
|
|
98
|
+
getDestination(prettyPrint)
|
|
99
|
+
);
|
|
88
100
|
function normalizeMeta(meta) {
|
|
89
101
|
if (meta == null) return {};
|
|
90
102
|
if (meta instanceof Error) {
|
|
@@ -1358,7 +1370,7 @@ var require_package = __commonJS({
|
|
|
1358
1370
|
"package.json"(exports2, module2) {
|
|
1359
1371
|
module2.exports = {
|
|
1360
1372
|
name: "@devilsdev/rag-pipeline-utils",
|
|
1361
|
-
version: "3.
|
|
1373
|
+
version: "3.7.0",
|
|
1362
1374
|
description: "Composable retrieval-augmented generation (RAG) for Node.js: grounded citations, evaluation, guardrails, hybrid retrieval and a CLI.",
|
|
1363
1375
|
main: "dist/index.cjs",
|
|
1364
1376
|
type: "commonjs",
|
|
@@ -1376,6 +1388,7 @@ var require_package = __commonJS({
|
|
|
1376
1388
|
exports: {
|
|
1377
1389
|
".": {
|
|
1378
1390
|
types: "./dist/index.d.ts",
|
|
1391
|
+
workerd: "./dist/index.cjs",
|
|
1379
1392
|
require: "./dist/index.cjs",
|
|
1380
1393
|
import: "./dist/index.mjs"
|
|
1381
1394
|
},
|
|
@@ -1396,7 +1409,7 @@ var require_package = __commonJS({
|
|
|
1396
1409
|
"./cli": "./bin/cli.js"
|
|
1397
1410
|
},
|
|
1398
1411
|
bin: {
|
|
1399
|
-
"rag-pipeline": "
|
|
1412
|
+
"rag-pipeline": "bin/cli.js"
|
|
1400
1413
|
},
|
|
1401
1414
|
engines: {
|
|
1402
1415
|
node: ">=22"
|
|
@@ -1404,6 +1417,8 @@ var require_package = __commonJS({
|
|
|
1404
1417
|
scripts: {
|
|
1405
1418
|
build: "node scripts/build-packages.js --bundles && node scripts/build.js && node scripts/generate-types.js && node scripts/build-packages.js --types && node scripts/check-type-exports.js",
|
|
1406
1419
|
"test:packages": "node scripts/test-packages.js",
|
|
1420
|
+
"test:consumer": "node scripts/consumer-journeys.js",
|
|
1421
|
+
"test:templates": "node scripts/test-templates.js",
|
|
1407
1422
|
prepack: "npm run build && node scripts/pin-readme-assets.js",
|
|
1408
1423
|
postpack: "node scripts/pin-readme-assets.js --restore",
|
|
1409
1424
|
"docs:validate": "node scripts/validate-docs-examples.js",
|
|
@@ -1459,6 +1474,7 @@ var require_package = __commonJS({
|
|
|
1459
1474
|
"bench:claude-citations": "node benchmarks/claude-citations.js",
|
|
1460
1475
|
"bench:judge-more": "node benchmarks/judge-more.js",
|
|
1461
1476
|
"bench:claude-latency": "node benchmarks/claude-latency.js",
|
|
1477
|
+
"claude:usage": "node scripts/claude-usage-report.js && node benchmarks/render.js",
|
|
1462
1478
|
dev: "node scripts/dev-tools.js menu",
|
|
1463
1479
|
"dev:health": "node scripts/dev-tools.js health",
|
|
1464
1480
|
"dev:setup": "node scripts/dev-tools.js setup",
|
|
@@ -1495,7 +1511,7 @@ var require_package = __commonJS({
|
|
|
1495
1511
|
},
|
|
1496
1512
|
repository: {
|
|
1497
1513
|
type: "git",
|
|
1498
|
-
url: "https://github.com/DevilsDev/rag-pipeline-utils.git"
|
|
1514
|
+
url: "git+https://github.com/DevilsDev/rag-pipeline-utils.git"
|
|
1499
1515
|
},
|
|
1500
1516
|
keywords: [
|
|
1501
1517
|
"rag",
|
|
@@ -1553,7 +1569,7 @@ var require_package = __commonJS({
|
|
|
1553
1569
|
"lint-staged": {
|
|
1554
1570
|
"src/**/*.{js,ts,jsx,tsx}": "eslint --fix",
|
|
1555
1571
|
"{__tests__,scripts,bin,docs-site,examples,contracts}/**/*.{js,jsx,ts,tsx,mjs,cjs}": "prettier --write --ignore-unknown",
|
|
1556
|
-
"
|
|
1572
|
+
"./*.{js,mjs,cjs}": "prettier --write --ignore-unknown",
|
|
1557
1573
|
"**/*.{json,md,mdx,yml,yaml,css,scss,html}": "prettier --write --ignore-unknown"
|
|
1558
1574
|
},
|
|
1559
1575
|
devDependencies: {
|
|
@@ -1622,10 +1638,10 @@ var require_package = __commonJS({
|
|
|
1622
1638
|
"why-is-node-running": "^3.2.2"
|
|
1623
1639
|
},
|
|
1624
1640
|
dependencies: {
|
|
1625
|
-
"@devilsdev/rag-connectors": "3.
|
|
1626
|
-
"@devilsdev/rag-core": "3.
|
|
1627
|
-
"@devilsdev/rag-eval": "3.
|
|
1628
|
-
"@devilsdev/rag-guardrails": "3.
|
|
1641
|
+
"@devilsdev/rag-connectors": "3.7.0",
|
|
1642
|
+
"@devilsdev/rag-core": "3.7.0",
|
|
1643
|
+
"@devilsdev/rag-eval": "3.7.0",
|
|
1644
|
+
"@devilsdev/rag-guardrails": "3.7.0",
|
|
1629
1645
|
ajv: "^8.17.1",
|
|
1630
1646
|
axios: "^1.21.0",
|
|
1631
1647
|
chokidar: "^4.0.3",
|
|
@@ -1695,12 +1711,18 @@ var require_optional_peer = __commonJS({
|
|
|
1695
1711
|
throw error;
|
|
1696
1712
|
}
|
|
1697
1713
|
}
|
|
1714
|
+
function missingPeer(error, name, hint) {
|
|
1715
|
+
if (!isMissing(error, name)) return error;
|
|
1716
|
+
const err = new Error(hint);
|
|
1717
|
+
err.code = "MISSING_DEPENDENCY";
|
|
1718
|
+
return err;
|
|
1719
|
+
}
|
|
1698
1720
|
function isMissing(error, name) {
|
|
1699
1721
|
if (!error || error.code !== "MODULE_NOT_FOUND") return false;
|
|
1700
1722
|
const message = String(error.message || "");
|
|
1701
1723
|
return message.startsWith(`Cannot find module '${name}'`) || message.startsWith(`Cannot find module "${name}"`);
|
|
1702
1724
|
}
|
|
1703
|
-
module2.exports = { loadOptionalPeer };
|
|
1725
|
+
module2.exports = { loadOptionalPeer, missingPeer };
|
|
1704
1726
|
}
|
|
1705
1727
|
});
|
|
1706
1728
|
|
|
@@ -1710,16 +1732,19 @@ var require_otel = __commonJS({
|
|
|
1710
1732
|
"use strict";
|
|
1711
1733
|
var { evaluationScores } = require_scores();
|
|
1712
1734
|
var { version: LIBRARY_VERSION } = require_package();
|
|
1713
|
-
var {
|
|
1735
|
+
var { missingPeer } = require_optional_peer();
|
|
1714
1736
|
var TRACER_NAME = "@devilsdev/rag-pipeline-utils";
|
|
1715
1737
|
var PROVIDERS = /* @__PURE__ */ new Set(["openai", "anthropic", "cohere", "ollama"]);
|
|
1716
1738
|
function loadOtelApi() {
|
|
1717
|
-
|
|
1718
|
-
"@opentelemetry/api"
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1739
|
+
try {
|
|
1740
|
+
return require("@opentelemetry/api");
|
|
1741
|
+
} catch (error) {
|
|
1742
|
+
throw missingPeer(
|
|
1743
|
+
error,
|
|
1744
|
+
"@opentelemetry/api",
|
|
1745
|
+
'telemetry needs the "@opentelemetry/api" package, an optional peer dependency: npm install @opentelemetry/api (with @devilsdev/rag-pipeline-utils/edge, pass it: telemetry: { api })'
|
|
1746
|
+
);
|
|
1747
|
+
}
|
|
1723
1748
|
}
|
|
1724
1749
|
function createTelemetry(option) {
|
|
1725
1750
|
if (!option) return null;
|
|
@@ -2573,8 +2598,8 @@ var require_scoring = __commonJS({
|
|
|
2573
2598
|
};
|
|
2574
2599
|
for (const w of words) {
|
|
2575
2600
|
const last = run[run.length - 1];
|
|
2576
|
-
const
|
|
2577
|
-
if (
|
|
2601
|
+
const isNumberWord2 = w in NUMBER_WORDS || w in NUMBER_SCALES || w === "one" || w === "and" && last in NUMBER_SCALES;
|
|
2602
|
+
if (isNumberWord2) {
|
|
2578
2603
|
run.push(w);
|
|
2579
2604
|
} else {
|
|
2580
2605
|
flush();
|
|
@@ -2583,18 +2608,405 @@ var require_scoring = __commonJS({
|
|
|
2583
2608
|
flush();
|
|
2584
2609
|
return { values, parts: [...values, ...parts] };
|
|
2585
2610
|
}
|
|
2611
|
+
var UNITS = (() => {
|
|
2612
|
+
const table = {};
|
|
2613
|
+
const add = (kind, size, name, words) => {
|
|
2614
|
+
for (const w of words.split(" ")) table[w] = [kind, size, name];
|
|
2615
|
+
};
|
|
2616
|
+
add("mass", 1e-3, "milligram", "mg milligram milligrams");
|
|
2617
|
+
add("mass", 1, "gram", "g gram grams gramme grammes");
|
|
2618
|
+
add("mass", 1e3, "kilogram", "kg kgs kilogram kilograms kilo kilos");
|
|
2619
|
+
add("mass", 453.59237, "pound", "lb lbs pound pounds");
|
|
2620
|
+
add("mass", 28.349523125, "ounce", "oz ounce ounces");
|
|
2621
|
+
add(
|
|
2622
|
+
"length",
|
|
2623
|
+
1e-3,
|
|
2624
|
+
"millimetre",
|
|
2625
|
+
"mm millimetre millimetres millimeter millimeters"
|
|
2626
|
+
);
|
|
2627
|
+
add(
|
|
2628
|
+
"length",
|
|
2629
|
+
0.01,
|
|
2630
|
+
"centimetre",
|
|
2631
|
+
"cm centimetre centimetres centimeter centimeters"
|
|
2632
|
+
);
|
|
2633
|
+
add("length", 1, "metre", "metre metres meter meters");
|
|
2634
|
+
add(
|
|
2635
|
+
"length",
|
|
2636
|
+
1e3,
|
|
2637
|
+
"kilometre",
|
|
2638
|
+
"km kilometre kilometres kilometer kilometers"
|
|
2639
|
+
);
|
|
2640
|
+
add("length", 1609.344, "mile", "mile miles");
|
|
2641
|
+
add("length", 0.3048, "foot", "ft foot feet");
|
|
2642
|
+
add("length", 0.0254, "inch", "inch inches");
|
|
2643
|
+
add("duration", 1, "second", "sec secs second seconds");
|
|
2644
|
+
add("duration", 60, "minute", "min mins minute minutes");
|
|
2645
|
+
add("duration", 3600, "hour", "hr hrs hour hours");
|
|
2646
|
+
add("duration", 86400, "day", "day days");
|
|
2647
|
+
add("duration", 604800, "week", "wk wks week weeks");
|
|
2648
|
+
add("calendar", 1, "month", "month months");
|
|
2649
|
+
add("calendar", 12, "year", "yr yrs year years");
|
|
2650
|
+
add("percent", 1, "percent", "percent");
|
|
2651
|
+
return table;
|
|
2652
|
+
})();
|
|
2653
|
+
var FRACTION_WORDS = {
|
|
2654
|
+
half: 50,
|
|
2655
|
+
quarter: 25,
|
|
2656
|
+
quarters: 25,
|
|
2657
|
+
third: 100 / 3,
|
|
2658
|
+
thirds: 100 / 3
|
|
2659
|
+
};
|
|
2660
|
+
var QUANTITY_TOKEN_RE = new RegExp("\\d+(?:[.,:]\\d+)*|\\p{L}+|%", "gu");
|
|
2661
|
+
function digitValue(raw) {
|
|
2662
|
+
if (raw.includes(":")) return null;
|
|
2663
|
+
if (/^\d{1,3}(,\d{3})+(\.\d+)?$/.test(raw))
|
|
2664
|
+
return Number(raw.replace(/,/g, ""));
|
|
2665
|
+
if (raw.includes(",")) return null;
|
|
2666
|
+
const n = Number(raw);
|
|
2667
|
+
return Number.isFinite(n) ? n : null;
|
|
2668
|
+
}
|
|
2669
|
+
var isNumberWord = (w) => w in NUMBER_WORDS || w in NUMBER_SCALES || w === "one";
|
|
2670
|
+
function quantities(text) {
|
|
2671
|
+
const items = [];
|
|
2672
|
+
const unitWords = /* @__PURE__ */ new Set();
|
|
2673
|
+
const tokens = [...String(text || "").matchAll(QUANTITY_TOKEN_RE)].map(
|
|
2674
|
+
(m) => ({
|
|
2675
|
+
t: m[0].toLowerCase(),
|
|
2676
|
+
start: m.index,
|
|
2677
|
+
end: m.index + m[0].length
|
|
2678
|
+
})
|
|
2679
|
+
);
|
|
2680
|
+
for (const tok of tokens) if (UNITS[tok.t]) unitWords.add(UNITS[tok.t][2]);
|
|
2681
|
+
const unitAt = (j) => {
|
|
2682
|
+
const tok = tokens[j];
|
|
2683
|
+
if (!tok) return null;
|
|
2684
|
+
if (tok.t === "%") return [["percent", 1, "percent"], j];
|
|
2685
|
+
if (tok.t === "per" && tokens[j + 1] && tokens[j + 1].t === "cent") {
|
|
2686
|
+
return [["percent", 1, "percent"], j + 1];
|
|
2687
|
+
}
|
|
2688
|
+
return UNITS[tok.t] ? [UNITS[tok.t], j] : null;
|
|
2689
|
+
};
|
|
2690
|
+
const push = (kind, value, unit, number, from, to, words) => items.push({
|
|
2691
|
+
kind,
|
|
2692
|
+
value,
|
|
2693
|
+
unit,
|
|
2694
|
+
number: String(number),
|
|
2695
|
+
start: tokens[from].start,
|
|
2696
|
+
end: tokens[to].end,
|
|
2697
|
+
words
|
|
2698
|
+
});
|
|
2699
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
2700
|
+
const t = tokens[i].t;
|
|
2701
|
+
if (t === "half" && tokens[i + 1] && (tokens[i + 1].t === "a" || tokens[i + 1].t === "an")) {
|
|
2702
|
+
const u2 = unitAt(i + 2);
|
|
2703
|
+
if (u2 && u2[0][0] !== "percent") {
|
|
2704
|
+
push(u2[0][0], 0.5 * u2[0][1], u2[0][2], 0.5, i, u2[1], true);
|
|
2705
|
+
i = u2[1];
|
|
2706
|
+
continue;
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
if (t in FRACTION_WORDS) {
|
|
2710
|
+
const prev = tokens[i - 1] && tokens[i - 1].t;
|
|
2711
|
+
const count = t === "half" ? 1 : prev in NUMBER_WORDS ? NUMBER_WORDS[prev] : prev === "a" || prev === "one" ? 1 : null;
|
|
2712
|
+
if (count !== null && tokens[i + 1] && tokens[i + 1].t === "of") {
|
|
2713
|
+
const from = t === "half" ? i : i - 1;
|
|
2714
|
+
push(
|
|
2715
|
+
"percent",
|
|
2716
|
+
count * FRACTION_WORDS[t],
|
|
2717
|
+
"percent",
|
|
2718
|
+
count * FRACTION_WORDS[t],
|
|
2719
|
+
from,
|
|
2720
|
+
i,
|
|
2721
|
+
true
|
|
2722
|
+
);
|
|
2723
|
+
items[items.length - 1].fraction = true;
|
|
2724
|
+
if (t === "third" || t === "thirds")
|
|
2725
|
+
items[items.length - 1].approx = true;
|
|
2726
|
+
continue;
|
|
2727
|
+
}
|
|
2728
|
+
}
|
|
2729
|
+
let value = null;
|
|
2730
|
+
let end = i;
|
|
2731
|
+
let words = false;
|
|
2732
|
+
if (/^\d/.test(t)) {
|
|
2733
|
+
value = digitValue(t);
|
|
2734
|
+
} else if (isNumberWord(t)) {
|
|
2735
|
+
let j = i;
|
|
2736
|
+
const run = [];
|
|
2737
|
+
while (tokens[j] && (isNumberWord(tokens[j].t) || tokens[j].t === "and" && tokens[j + 1] && isNumberWord(tokens[j + 1].t) && run.length)) {
|
|
2738
|
+
if (tokens[j].t !== "and") run.push(tokens[j].t);
|
|
2739
|
+
j++;
|
|
2740
|
+
}
|
|
2741
|
+
if (run.length) {
|
|
2742
|
+
value = parseNumberWords(run);
|
|
2743
|
+
end = j - 1;
|
|
2744
|
+
words = true;
|
|
2745
|
+
}
|
|
2746
|
+
}
|
|
2747
|
+
if (value === null) continue;
|
|
2748
|
+
let next = end + 1;
|
|
2749
|
+
if (tokens[next] && tokens[next].t === "and" && tokens[next + 1] && tokens[next + 1].t === "a" && tokens[next + 2] && tokens[next + 2].t === "half") {
|
|
2750
|
+
value += 0.5;
|
|
2751
|
+
next += 3;
|
|
2752
|
+
}
|
|
2753
|
+
const u = unitAt(next);
|
|
2754
|
+
if (u) {
|
|
2755
|
+
push(u[0][0], value * u[0][1], u[0][2], value, i, u[1], words);
|
|
2756
|
+
i = u[1];
|
|
2757
|
+
} else {
|
|
2758
|
+
i = end;
|
|
2759
|
+
}
|
|
2760
|
+
}
|
|
2761
|
+
return { items, unitWords };
|
|
2762
|
+
}
|
|
2763
|
+
var docQuantityCache = /* @__PURE__ */ new Map();
|
|
2764
|
+
var claimQuantityCache = /* @__PURE__ */ new Map();
|
|
2765
|
+
function cachedQuantities(cache, text) {
|
|
2766
|
+
const key = String(text || "");
|
|
2767
|
+
let hit = cache.get(key);
|
|
2768
|
+
if (!hit) {
|
|
2769
|
+
hit = quantities(key);
|
|
2770
|
+
if (cache.size >= 256) cache.clear();
|
|
2771
|
+
cache.set(key, hit);
|
|
2772
|
+
}
|
|
2773
|
+
return hit;
|
|
2774
|
+
}
|
|
2775
|
+
var docQuantities = (text) => cachedQuantities(docQuantityCache, text);
|
|
2776
|
+
function sameQuantity(a, b) {
|
|
2777
|
+
const tolerance = a.approx || b.approx ? 1 : Math.max(Math.abs(a.value), Math.abs(b.value)) * 1e-9;
|
|
2778
|
+
return Math.abs(a.value - b.value) <= tolerance;
|
|
2779
|
+
}
|
|
2780
|
+
function clockHours(text) {
|
|
2781
|
+
const hours24 = /* @__PURE__ */ new Set();
|
|
2782
|
+
const hours12 = /* @__PURE__ */ new Set();
|
|
2783
|
+
for (const m of String(text || "").matchAll(
|
|
2784
|
+
// one-digit minutes too ("7:0-23:0", as some listings write hours)
|
|
2785
|
+
/(?<!\d)(\d{1,2}):([0-5]?\d)(?!\d)/g
|
|
2786
|
+
)) {
|
|
2787
|
+
const h = Number(m[1]);
|
|
2788
|
+
if (h > 23) continue;
|
|
2789
|
+
hours24.add(h);
|
|
2790
|
+
hours12.add(h % 12 === 0 ? 12 : h % 12);
|
|
2791
|
+
}
|
|
2792
|
+
return { hours24, hours12 };
|
|
2793
|
+
}
|
|
2794
|
+
function checkClockTimes(claimText, docText) {
|
|
2795
|
+
const spans = [];
|
|
2796
|
+
const matched = [];
|
|
2797
|
+
const missing = [];
|
|
2798
|
+
const { hours24, hours12 } = clockHours(docText);
|
|
2799
|
+
if (hours24.size === 0) return { spans, matched, missing };
|
|
2800
|
+
const tokens = [...String(claimText || "").matchAll(QUANTITY_TOKEN_RE)].map(
|
|
2801
|
+
(m) => ({
|
|
2802
|
+
t: m[0].toLowerCase(),
|
|
2803
|
+
start: m.index,
|
|
2804
|
+
end: m.index + m[0].length
|
|
2805
|
+
})
|
|
2806
|
+
);
|
|
2807
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
2808
|
+
if (!/^\d{1,2}$/.test(tokens[i].t)) continue;
|
|
2809
|
+
const h = Number(tokens[i].t);
|
|
2810
|
+
const next = tokens[i + 1] && tokens[i + 1].t;
|
|
2811
|
+
const prev = tokens[i - 1] && tokens[i - 1].t;
|
|
2812
|
+
if ((next === "am" || next === "pm") && h >= 1 && h <= 12) {
|
|
2813
|
+
const h24 = h % 12 + (next === "pm" ? 12 : 0);
|
|
2814
|
+
const span = [tokens[i].start, tokens[i + 1].end];
|
|
2815
|
+
if (hours24.has(h24)) matched.push(span);
|
|
2816
|
+
else missing.push(`time:${h}${next}`);
|
|
2817
|
+
spans.push(span);
|
|
2818
|
+
} else if (["at", "from", "to", "until", "till"].includes(prev) && h >= 1 && h <= 12 && // "open from 9 to 5.", not "up to 8 people" or "at 11 Main Street"
|
|
2819
|
+
endsTime(claimText, tokens[i].end, next)) {
|
|
2820
|
+
const span = [tokens[i].start, tokens[i].end];
|
|
2821
|
+
if (hours12.has(h) || hours24.has(h)) matched.push(span);
|
|
2822
|
+
else missing.push(`time:${h}`);
|
|
2823
|
+
spans.push(span);
|
|
2824
|
+
}
|
|
2825
|
+
}
|
|
2826
|
+
return { spans, matched, missing };
|
|
2827
|
+
}
|
|
2828
|
+
var AFTER_HOUR = new Set(
|
|
2829
|
+
"to until till and or on every each daily sharp then".split(" ")
|
|
2830
|
+
);
|
|
2831
|
+
function endsTime(text, end, next) {
|
|
2832
|
+
if (next === void 0 || /^\s*[.,;:!?)]/.test(String(text).slice(end, end + 3))) {
|
|
2833
|
+
return true;
|
|
2834
|
+
}
|
|
2835
|
+
return AFTER_HOUR.has(next);
|
|
2836
|
+
}
|
|
2837
|
+
function withoutSpans(text, spans) {
|
|
2838
|
+
let out = String(text || "");
|
|
2839
|
+
for (const [start, end] of [...spans].sort((a, b) => b[0] - a[0])) {
|
|
2840
|
+
out = out.slice(0, start) + " ".repeat(end - start) + out.slice(end);
|
|
2841
|
+
}
|
|
2842
|
+
return out;
|
|
2843
|
+
}
|
|
2844
|
+
function footnotedYears(text) {
|
|
2845
|
+
const out = [];
|
|
2846
|
+
for (const m of String(text || "").matchAll(
|
|
2847
|
+
new RegExp("(?<![\\d.])(1[89]\\d\\d|20\\d\\d|2100)\\.\\d{1,2}(?=\\s+\\p{Lu})", "gu")
|
|
2848
|
+
)) {
|
|
2849
|
+
out.push(String(Number(m[1])));
|
|
2850
|
+
}
|
|
2851
|
+
return out;
|
|
2852
|
+
}
|
|
2586
2853
|
function checkNumericConsistency(claimText, docText) {
|
|
2587
|
-
const
|
|
2854
|
+
const claim = withoutReferences(claimText);
|
|
2855
|
+
const docParts = /* @__PURE__ */ new Set([
|
|
2856
|
+
...extractNumbers(docText).parts,
|
|
2857
|
+
...footnotedYears(docText)
|
|
2858
|
+
]);
|
|
2859
|
+
const doc = docQuantities(docText);
|
|
2860
|
+
const spans = [];
|
|
2861
|
+
const matched = [];
|
|
2862
|
+
const missing = [];
|
|
2863
|
+
const claimNumbers = [];
|
|
2864
|
+
for (const q of cachedQuantities(claimQuantityCache, claim).items) {
|
|
2865
|
+
const sameKind = doc.items.filter((d) => d.kind === q.kind);
|
|
2866
|
+
const equal = sameKind.filter((d) => sameQuantity(d, q));
|
|
2867
|
+
const found = equal.length > 0 || // the number with the same unit word elsewhere ("20 days" in "20
|
|
2868
|
+
// vacation days"), or with no other unit of this kind ("seven days a
|
|
2869
|
+
// week" against a source with "7:00")
|
|
2870
|
+
docParts.has(q.number) && (doc.unitWords.has(q.unit) || !sameKind.some((d) => d.number === q.number));
|
|
2871
|
+
if (sameKind.length === 0 && !q.words && !found) continue;
|
|
2872
|
+
spans.push([q.start, q.end]);
|
|
2873
|
+
const converted = (d) => d.unit !== q.unit || Boolean(d.fraction) !== Boolean(q.fraction);
|
|
2874
|
+
if (equal.length > 0 && equal.every(converted)) {
|
|
2875
|
+
matched.push([q.start, q.end]);
|
|
2876
|
+
}
|
|
2877
|
+
claimNumbers.push(`${q.kind}:${q.number} ${q.unit}`);
|
|
2878
|
+
if (!found && (sameKind.length > 0 || !q.words))
|
|
2879
|
+
missing.push(`${q.kind}:${q.number} ${q.unit}`);
|
|
2880
|
+
}
|
|
2881
|
+
const clock = checkClockTimes(claim, docText);
|
|
2882
|
+
spans.push(...clock.spans);
|
|
2883
|
+
matched.push(...clock.matched);
|
|
2884
|
+
missing.push(...clock.missing);
|
|
2885
|
+
const plain = [
|
|
2886
|
+
...new Set(extractNumbers(withoutSpans(claim, spans)).values),
|
|
2887
|
+
...claimMonths(claimText).map((m) => `month:${m}`),
|
|
2888
|
+
...identifiers(claimText)
|
|
2889
|
+
];
|
|
2890
|
+
claimNumbers.push(...plain, ...clock.missing);
|
|
2891
|
+
const matchedWords = [
|
|
2892
|
+
...new Set(matched.flatMap(([a, b]) => tokenize(claim.slice(a, b))))
|
|
2893
|
+
];
|
|
2588
2894
|
if (claimNumbers.length === 0) {
|
|
2589
|
-
return { consistent: true, claimNumbers, missing: [] };
|
|
2590
|
-
}
|
|
2591
|
-
const docNumbers = new Set(
|
|
2592
|
-
|
|
2593
|
-
|
|
2895
|
+
return { consistent: true, claimNumbers, missing: [], matchedWords };
|
|
2896
|
+
}
|
|
2897
|
+
const docNumbers = /* @__PURE__ */ new Set([
|
|
2898
|
+
...docParts,
|
|
2899
|
+
...docMonths(docText).map((m) => `month:${m}`),
|
|
2900
|
+
...identifiers(docText)
|
|
2901
|
+
]);
|
|
2902
|
+
missing.push(...plain.filter((n) => !docNumbers.has(n)));
|
|
2903
|
+
return {
|
|
2904
|
+
consistent: missing.length === 0,
|
|
2905
|
+
claimNumbers,
|
|
2906
|
+
missing,
|
|
2907
|
+
matchedWords
|
|
2908
|
+
};
|
|
2909
|
+
}
|
|
2910
|
+
var REFERENCE_WORDS = "document|documents|doc|docs|source|sources|table|tables|figure|figures|fig|section|sections|step|steps|study|studies|appendix|chapter|page|pages|claim|claims|passage|passages|item|items|reference|ref|part|exhibit|equation|eq|line|row|column|note|footnote|paragraph";
|
|
2911
|
+
var REFERENCE_RE = new RegExp(
|
|
2912
|
+
`\\b(${REFERENCE_WORDS})\\.?\\s+\\d+[a-z]?\\b`,
|
|
2913
|
+
"giu"
|
|
2914
|
+
);
|
|
2915
|
+
var IDENTIFIER_RE = new RegExp("(\\p{L}[\\p{L}\\p{N}]*)-(\\d+)\\b", "gu");
|
|
2916
|
+
function identifiers(text) {
|
|
2917
|
+
if (!text || typeof text !== "string") return [];
|
|
2918
|
+
return [
|
|
2919
|
+
...new Set(
|
|
2920
|
+
[...text.matchAll(IDENTIFIER_RE)].map((m) => `id:${m[0].toLowerCase()}`)
|
|
2921
|
+
)
|
|
2922
|
+
];
|
|
2923
|
+
}
|
|
2924
|
+
function withoutReferences(text) {
|
|
2925
|
+
if (!text || typeof text !== "string") return text;
|
|
2926
|
+
return text.replace(REFERENCE_RE, " ").replace(IDENTIFIER_RE, (_, word) => word);
|
|
2927
|
+
}
|
|
2928
|
+
var MONTHS = [
|
|
2929
|
+
["january", "jan"],
|
|
2930
|
+
["february", "feb"],
|
|
2931
|
+
["march", "mar"],
|
|
2932
|
+
["april", "apr"],
|
|
2933
|
+
["may"],
|
|
2934
|
+
["june", "jun"],
|
|
2935
|
+
["july", "jul"],
|
|
2936
|
+
["august", "aug"],
|
|
2937
|
+
["september", "sep", "sept"],
|
|
2938
|
+
["october", "oct"],
|
|
2939
|
+
["november", "nov"],
|
|
2940
|
+
["december", "dec"]
|
|
2941
|
+
];
|
|
2942
|
+
var MONTH_OF = new Map(
|
|
2943
|
+
MONTHS.flatMap((names, i) => names.map((n) => [n, i + 1]))
|
|
2944
|
+
);
|
|
2945
|
+
function claimMonths(text) {
|
|
2946
|
+
const words = String(text || "").toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean);
|
|
2947
|
+
const out = [];
|
|
2948
|
+
words.forEach((w, i) => {
|
|
2949
|
+
const m = MONTH_OF.get(w);
|
|
2950
|
+
if (!m) return;
|
|
2951
|
+
const near = [words[i - 1], words[i + 1]].some(
|
|
2952
|
+
(x) => /^\d+$/.test(x || "")
|
|
2953
|
+
);
|
|
2954
|
+
if (near) out.push(m);
|
|
2955
|
+
});
|
|
2956
|
+
return [...new Set(out)];
|
|
2957
|
+
}
|
|
2958
|
+
function docMonths(text) {
|
|
2959
|
+
const t = String(text || "");
|
|
2960
|
+
const out = /* @__PURE__ */ new Set();
|
|
2961
|
+
for (const w of t.toLowerCase().split(/[^\p{L}]+/u)) {
|
|
2962
|
+
const m = MONTH_OF.get(w);
|
|
2963
|
+
if (m) out.add(m);
|
|
2964
|
+
}
|
|
2965
|
+
for (const [, a, b] of t.matchAll(
|
|
2966
|
+
/\b(\d{1,2})[/.-](\d{1,2})[/.-]\d{2,4}\b/g
|
|
2967
|
+
)) {
|
|
2968
|
+
for (const v of [Number(a), Number(b)]) if (v >= 1 && v <= 12) out.add(v);
|
|
2969
|
+
}
|
|
2970
|
+
for (const [, mm] of t.matchAll(/\b\d{4}-(\d{1,2})-\d{1,2}\b/g)) {
|
|
2971
|
+
const v = Number(mm);
|
|
2972
|
+
if (v >= 1 && v <= 12) out.add(v);
|
|
2973
|
+
}
|
|
2974
|
+
return [...out];
|
|
2594
2975
|
}
|
|
2595
2976
|
var NEGATION_WORDS = new Set(
|
|
2596
|
-
"not no never none nobody nothing nowhere neither nor cannot without dont doesnt didnt isnt arent wasnt werent wont wouldnt shouldnt couldnt cant hasnt havent hadnt mustnt neednt".split(" ")
|
|
2977
|
+
"not no never none nobody nothing nowhere neither nor cannot without dont doesnt didnt isnt arent wasnt werent wont wouldnt shouldnt couldnt cant hasnt havent hadnt mustnt neednt nicht kein keine keinen keinem keiner keines nie niemals nunca jam\xE1s jamas nadie nada tampoco ni pas jamais rien aucun aucune".split(" ")
|
|
2597
2978
|
);
|
|
2979
|
+
var CLAUSE_BOUNDARY_WORDS = new Set(
|
|
2980
|
+
"but whereas although though while however yet and or except unlike".split(
|
|
2981
|
+
" "
|
|
2982
|
+
)
|
|
2983
|
+
);
|
|
2984
|
+
var ELLIPTICAL_START = new Set(
|
|
2985
|
+
"not never was were is are be been did does do has have had can could will would should may might must shall".split(" ")
|
|
2986
|
+
);
|
|
2987
|
+
var FRAMING_WORDS = new Set(
|
|
2988
|
+
"according based in on given as from per under after before during".split(
|
|
2989
|
+
" "
|
|
2990
|
+
)
|
|
2991
|
+
);
|
|
2992
|
+
function clauseTokens(text) {
|
|
2993
|
+
const out = [];
|
|
2994
|
+
for (const part of String(text || "").split(/[,;:()[\]]+/)) {
|
|
2995
|
+
let current = [];
|
|
2996
|
+
for (const t of tokenize(part)) {
|
|
2997
|
+
if (CLAUSE_BOUNDARY_WORDS.has(t)) {
|
|
2998
|
+
if (current.length) out.push(current);
|
|
2999
|
+
current = [];
|
|
3000
|
+
} else {
|
|
3001
|
+
current.push(t);
|
|
3002
|
+
}
|
|
3003
|
+
}
|
|
3004
|
+
if (current.length) out.push(current);
|
|
3005
|
+
}
|
|
3006
|
+
return out;
|
|
3007
|
+
}
|
|
3008
|
+
var stem = (t) => t.length > 4 ? t.replace(/(ing|ed|es|s|e)$/, "") : t;
|
|
3009
|
+
var CONTINUING_NEGATIONS = /* @__PURE__ */ new Set(["nor", "ni"]);
|
|
2598
3010
|
var NON_NEGATING_NEXT = {
|
|
2599
3011
|
not: /* @__PURE__ */ new Set(["only", "just", "merely"]),
|
|
2600
3012
|
no: /* @__PURE__ */ new Set(["doubt", "matter"]),
|
|
@@ -2605,6 +3017,7 @@ var require_scoring = __commonJS({
|
|
|
2605
3017
|
for (let i = 0; i < tokens.length; i++) {
|
|
2606
3018
|
const t = tokens[i];
|
|
2607
3019
|
if (!NEGATION_WORDS.has(t)) continue;
|
|
3020
|
+
if (CONTINUING_NEGATIONS.has(t) && count > 0) continue;
|
|
2608
3021
|
if (NON_NEGATING_NEXT[t] && NON_NEGATING_NEXT[t].has(tokens[i + 1])) {
|
|
2609
3022
|
continue;
|
|
2610
3023
|
}
|
|
@@ -2613,51 +3026,95 @@ var require_scoring = __commonJS({
|
|
|
2613
3026
|
return count % 2 === 1;
|
|
2614
3027
|
}
|
|
2615
3028
|
function checkPolarityConsistency(claimText, docText) {
|
|
2616
|
-
const
|
|
2617
|
-
const claimNegated = isNegated(claimTokens);
|
|
2618
|
-
const claimWords = [
|
|
3029
|
+
const words = (tokens) => [
|
|
2619
3030
|
...new Set(
|
|
2620
|
-
contentTokens(
|
|
3031
|
+
contentTokens(tokens.filter((t) => !NEGATION_WORDS.has(t))).map(stem)
|
|
2621
3032
|
)
|
|
2622
3033
|
];
|
|
2623
|
-
|
|
2624
|
-
|
|
3034
|
+
let claimClauses = clauseTokens(claimText).map((tokens) => ({ tokens, words: words(tokens) })).filter((c) => c.words.length > 0);
|
|
3035
|
+
if (claimClauses.length > 1 && FRAMING_WORDS.has(claimClauses[0].tokens[0]) && !isNegated(claimClauses[0].tokens)) {
|
|
3036
|
+
claimClauses = claimClauses.slice(1);
|
|
2625
3037
|
}
|
|
2626
|
-
const
|
|
2627
|
-
|
|
2628
|
-
const set = new Set(tokens);
|
|
2629
|
-
const support = claimWords.filter((t) => set.has(t)).length / claimWords.length;
|
|
2630
|
-
return { sentence, tokens, support };
|
|
2631
|
-
}).filter((c) => c.support > 0).sort((a, b) => b.support - a.support);
|
|
2632
|
-
if (candidates.length === 0) {
|
|
3038
|
+
const claimNegated = claimClauses.some((c) => isNegated(c.tokens));
|
|
3039
|
+
if (claimClauses.length === 0) {
|
|
2633
3040
|
return { consistent: true, claimNegated, sourceSentence: null };
|
|
2634
3041
|
}
|
|
2635
|
-
const
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
3042
|
+
const sentences = splitSentences(docText).map((sentence) => {
|
|
3043
|
+
const clauses = clauseTokens(sentence).map((tokens, i) => ({
|
|
3044
|
+
negated: isNegated(tokens),
|
|
3045
|
+
stems: new Set(tokens.map(stem)),
|
|
3046
|
+
// "... but was not better than Y", "... but did not reduce Y": no
|
|
3047
|
+
// subject of its own, so it continues the first clause's (3.7)
|
|
3048
|
+
continues: i > 0 && ELLIPTICAL_START.has(tokens[0])
|
|
3049
|
+
}));
|
|
3050
|
+
const stems = new Set(clauses.flatMap((c) => [...c.stems]));
|
|
3051
|
+
return { sentence, clauses, stems };
|
|
3052
|
+
});
|
|
3053
|
+
const agrees = (clause, sentence) => {
|
|
3054
|
+
const negated = isNegated(clause.tokens);
|
|
3055
|
+
let distinctive = clause.words.map((w) => sentence.clauses.filter((c) => c.stems.has(w))).filter((owners) => owners.length === 1).map((owners) => owners[0]);
|
|
3056
|
+
const first = sentence.clauses[0];
|
|
3057
|
+
if (distinctive.some((c) => c.continues)) {
|
|
3058
|
+
distinctive = distinctive.filter((c) => c !== first);
|
|
3059
|
+
}
|
|
3060
|
+
if (distinctive.length > 0) {
|
|
3061
|
+
return distinctive.every((c) => c.negated === negated);
|
|
3062
|
+
}
|
|
3063
|
+
const ranked = sentence.clauses.map((c) => ({
|
|
3064
|
+
c,
|
|
3065
|
+
support: clause.words.filter((w) => c.stems.has(w)).length
|
|
3066
|
+
})).filter((r) => r.support > 0).sort((a, b) => b.support - a.support);
|
|
3067
|
+
if (ranked.length === 0) return true;
|
|
3068
|
+
const best = ranked[0].support;
|
|
3069
|
+
return ranked.filter((r) => r.support === best).some((r) => r.c.negated === negated);
|
|
2642
3070
|
};
|
|
3071
|
+
let consistent = true;
|
|
3072
|
+
let sourceSentence = null;
|
|
3073
|
+
for (const clause of claimClauses) {
|
|
3074
|
+
const candidates = sentences.map((s) => ({
|
|
3075
|
+
...s,
|
|
3076
|
+
support: clause.words.filter((w) => s.stems.has(w)).length / clause.words.length
|
|
3077
|
+
})).filter((s) => s.support > 0).sort((a, b) => b.support - a.support);
|
|
3078
|
+
if (candidates.length === 0) continue;
|
|
3079
|
+
const best = candidates[0].support;
|
|
3080
|
+
const compared = candidates.filter((s) => s.support === best);
|
|
3081
|
+
const match = compared.find((s) => agrees(clause, s));
|
|
3082
|
+
if (!match) consistent = false;
|
|
3083
|
+
if (sourceSentence === null || !match) {
|
|
3084
|
+
sourceSentence = (match || compared[0]).sentence;
|
|
3085
|
+
}
|
|
3086
|
+
}
|
|
3087
|
+
return { consistent, claimNegated, sourceSentence };
|
|
2643
3088
|
}
|
|
2644
3089
|
var POLARITY_MISMATCH_FACTOR = 0.4;
|
|
2645
3090
|
var NUMERIC_MISMATCH_FACTOR = 0.4;
|
|
2646
3091
|
function computeSupportScore(claimTokens, docTokens, idfWeights = null, options = {}) {
|
|
2647
|
-
const
|
|
3092
|
+
const rawClaim = options && options.claimText;
|
|
3093
|
+
const fromText = typeof rawClaim === "string" && rawClaim !== withoutReferences(rawClaim) ? tokenize(withoutReferences(rawClaim)) : claimTokens;
|
|
3094
|
+
const claim = [...new Set(contentTokens(fromText))];
|
|
2648
3095
|
if (claim.length === 0) return 0;
|
|
2649
3096
|
const docSet = docTokens instanceof Set ? docTokens : new Set(docTokens);
|
|
2650
3097
|
let matched = 0;
|
|
2651
3098
|
let total = 0;
|
|
3099
|
+
const unmatched = [];
|
|
2652
3100
|
for (const token of claim) {
|
|
2653
3101
|
const weight = idfWeights ? idfWeights.get(token) ?? 1 : 1;
|
|
2654
3102
|
total += weight;
|
|
2655
3103
|
if (docSet.has(token)) matched += weight;
|
|
3104
|
+
else unmatched.push([token, weight]);
|
|
2656
3105
|
}
|
|
2657
|
-
|
|
3106
|
+
if (matched === 0) return 0;
|
|
2658
3107
|
const { claimText, docText } = options || {};
|
|
2659
|
-
|
|
2660
|
-
|
|
3108
|
+
const texts = typeof claimText === "string" && typeof docText === "string";
|
|
3109
|
+
const numeric = texts ? checkNumericConsistency(claimText, docText) : null;
|
|
3110
|
+
if (numeric && numeric.matchedWords.length) {
|
|
3111
|
+
const restated = new Set(numeric.matchedWords);
|
|
3112
|
+
for (const [token, weight] of unmatched) {
|
|
3113
|
+
if (restated.has(token)) matched += weight;
|
|
3114
|
+
}
|
|
3115
|
+
}
|
|
3116
|
+
let score = total > 0 ? matched / total : 0;
|
|
3117
|
+
if (texts) {
|
|
2661
3118
|
if (!numeric.consistent) {
|
|
2662
3119
|
const share = (numeric.claimNumbers.length - numeric.missing.length) / numeric.claimNumbers.length;
|
|
2663
3120
|
score *= NUMERIC_MISMATCH_FACTOR * share;
|
|
@@ -5217,6 +5674,8 @@ var require_connector_utils = __commonJS({
|
|
|
5217
5674
|
`${provider} API key required. Set ${envVar} or pass the apiKey option.`
|
|
5218
5675
|
);
|
|
5219
5676
|
err.code = "MISSING_API_KEY";
|
|
5677
|
+
err.provider = provider;
|
|
5678
|
+
err.envVar = envVar;
|
|
5220
5679
|
return err;
|
|
5221
5680
|
}
|
|
5222
5681
|
function signalOptions(signal) {
|
|
@@ -5276,6 +5735,7 @@ var require_connector_utils = __commonJS({
|
|
|
5276
5735
|
message = `${provider} request failed: ${err.message}`;
|
|
5277
5736
|
}
|
|
5278
5737
|
const error = new Error(message);
|
|
5738
|
+
error.provider = provider;
|
|
5279
5739
|
if (err.code) error.code = err.code;
|
|
5280
5740
|
if (response) {
|
|
5281
5741
|
error.status = response.status;
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @param {Object} components - { embedder, retriever, reranker, llm }
|
|
3
|
-
* @returns {{ components: Object, timings: Array<{stage: string, ms: number}>, captured: { results: Array|null } }}
|
|
3
|
+
* @returns {{ components: Object, timings: Array<{stage: string, ms: number}>, captured: { results: Array|null, error: * } }}
|
|
4
|
+
* `captured.error` is the last error a stage threw: the pipeline reports
|
|
5
|
+
* only its message, and the CLI words a refused key from the error itself.
|
|
4
6
|
*/
|
|
5
7
|
export function instrumentComponents(components: any): {
|
|
6
8
|
components: any;
|
|
@@ -10,5 +12,6 @@ export function instrumentComponents(components: any): {
|
|
|
10
12
|
}>;
|
|
11
13
|
captured: {
|
|
12
14
|
results: any[] | null;
|
|
15
|
+
error: any;
|
|
13
16
|
};
|
|
14
17
|
};
|
|
@@ -94,7 +94,8 @@ export function query({ config, configDir, question, topK, components, timeout,
|
|
|
94
94
|
* @param {object} params
|
|
95
95
|
* @param {object} params.config
|
|
96
96
|
* @param {string} params.configDir
|
|
97
|
-
* @returns {{ pipeline: object, hasLlm: boolean }}
|
|
97
|
+
* @returns {{ pipeline: object, hasLlm: boolean, captured: { error: * } }}
|
|
98
|
+
* `captured.error` is the last error a stage threw (see instrument.js)
|
|
98
99
|
*/
|
|
99
100
|
export function createPipeline({ config, configDir }: {
|
|
100
101
|
config: object;
|
|
@@ -102,6 +103,9 @@ export function createPipeline({ config, configDir }: {
|
|
|
102
103
|
}): {
|
|
103
104
|
pipeline: object;
|
|
104
105
|
hasLlm: boolean;
|
|
106
|
+
captured: {
|
|
107
|
+
error: any;
|
|
108
|
+
};
|
|
105
109
|
};
|
|
106
110
|
/**
|
|
107
111
|
* Retrieval only (no llm): embed the question, retrieve and, if configured,
|
|
@@ -42,6 +42,8 @@ export namespace BUILT_INS {
|
|
|
42
42
|
memory: (cfg: any) => import("../connectors/memory-retriever").MemoryRetriever;
|
|
43
43
|
pgvector: (cfg: any) => import("../retrieval/pgvector-retriever").PgVectorRetriever;
|
|
44
44
|
qdrant: (cfg: any) => import("../retrieval/qdrant-retriever").QdrantRetriever;
|
|
45
|
+
weaviate: (cfg: any) => import("../retrieval/weaviate-retriever").WeaviateRetriever;
|
|
46
|
+
pinecone: (cfg: any) => import("../retrieval/pinecone-retriever").PineconeRetriever;
|
|
45
47
|
};
|
|
46
48
|
namespace llm {
|
|
47
49
|
export function openai_1(cfg: any): import("../connectors/openai-connector").OpenAIConnector;
|
|
@@ -62,6 +62,13 @@ export class OpenAIConnector extends BaseConnector {
|
|
|
62
62
|
* @private
|
|
63
63
|
*/
|
|
64
64
|
private _request;
|
|
65
|
+
/**
|
|
66
|
+
* Name the provider on an HTTP error from the SDK, as the axios-based
|
|
67
|
+
* connectors do, so a caller such as the CLI can say which key was refused.
|
|
68
|
+
* A custom baseURL never reads OPENAI_API_KEY, so it gets no envVar.
|
|
69
|
+
* @private
|
|
70
|
+
*/
|
|
71
|
+
private _tagError;
|
|
65
72
|
/**
|
|
66
73
|
* Generate a chat completion using the provided query and optional RAG context.
|
|
67
74
|
*
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retriever on a Pinecone serverless index over its REST API, with no
|
|
3
|
+
* client package. The index must exist: with a dense vector of your
|
|
4
|
+
* embedding model's dimension, or with a text field Pinecone embeds itself
|
|
5
|
+
* (then documents need no vector and retrieve() takes the query text). Use
|
|
6
|
+
* the cosine metric for the scores the other retrievers return.
|
|
7
|
+
*/
|
|
8
|
+
export class PineconeRetriever {
|
|
9
|
+
/**
|
|
10
|
+
* @param {Object} options
|
|
11
|
+
* @param {string} [options.apiKey] - Defaults to PINECONE_API_KEY
|
|
12
|
+
* @param {string} [options.index] - The index name, described on first use
|
|
13
|
+
* @param {string} [options.host] - The index host (from the console), instead of or with `index`
|
|
14
|
+
* @param {string} [options.namespace='rag_documents'] - A namespace this retriever owns
|
|
15
|
+
* @param {'auto'|'documents'|'vectors'} [options.api='auto'] - Document index (Documents API) or vector index; auto reads it from the index description
|
|
16
|
+
* @param {string} [options.vectorField] - The document index's dense_vector field (default: the schema's)
|
|
17
|
+
* @param {string} [options.textField] - The document index's text field that Pinecone embeds (integrated embedding; default: the schema's, when the index has no dense_vector field)
|
|
18
|
+
* @param {number} [options.freshnessTimeoutMs=30000] - How long a read waits for this retriever's writes to be indexed; 0 to not wait
|
|
19
|
+
* @param {string} [options.cacheKey] - Cache identity; change it when the index changes
|
|
20
|
+
* @param {Function} [options.fetch] - fetch implementation (defaults to the global)
|
|
21
|
+
*/
|
|
22
|
+
constructor(options?: {
|
|
23
|
+
apiKey?: string;
|
|
24
|
+
index?: string;
|
|
25
|
+
host?: string;
|
|
26
|
+
namespace?: string;
|
|
27
|
+
api?: "auto" | "documents" | "vectors";
|
|
28
|
+
vectorField?: string;
|
|
29
|
+
textField?: string;
|
|
30
|
+
freshnessTimeoutMs?: number;
|
|
31
|
+
cacheKey?: string;
|
|
32
|
+
fetch?: Function;
|
|
33
|
+
});
|
|
34
|
+
index: string;
|
|
35
|
+
namespace: string;
|
|
36
|
+
freshnessTimeoutMs: number;
|
|
37
|
+
cacheKey: string;
|
|
38
|
+
_apiKey: string;
|
|
39
|
+
_host: string;
|
|
40
|
+
_api: "auto" | "documents" | "vectors";
|
|
41
|
+
_vectorField: string;
|
|
42
|
+
_textField: string;
|
|
43
|
+
_fetch: any;
|
|
44
|
+
_resolving: any;
|
|
45
|
+
_writeLsn: number;
|
|
46
|
+
_dimension: any;
|
|
47
|
+
/**
|
|
48
|
+
* Cache identity for createRagPipeline({ cache }): none unless you set
|
|
49
|
+
* `cacheKey`, since other clients can change the index.
|
|
50
|
+
* @param {'retriever'} role
|
|
51
|
+
* @returns {string|null}
|
|
52
|
+
*/
|
|
53
|
+
getCacheKey(role: "retriever"): string | null;
|
|
54
|
+
/**
|
|
55
|
+
* Whether the index embeds text itself (integrated embedding): then
|
|
56
|
+
* store() needs no vectors and retrieve() takes the query text, so no
|
|
57
|
+
* embedder is needed. The CLI checks this before requiring an embedder.
|
|
58
|
+
* Reads the index description on first use.
|
|
59
|
+
* @param {{ signal?: AbortSignal }} [options]
|
|
60
|
+
* @returns {Promise<boolean>}
|
|
61
|
+
*/
|
|
62
|
+
embedsText({ signal }?: {
|
|
63
|
+
signal?: AbortSignal;
|
|
64
|
+
}): Promise<boolean>;
|
|
65
|
+
/**
|
|
66
|
+
* One REST call; resolves the parsed body and the response headers.
|
|
67
|
+
* @private
|
|
68
|
+
*/
|
|
69
|
+
private _call;
|
|
70
|
+
/** True on a document index that embeds text itself. @private */
|
|
71
|
+
private get _integrated();
|
|
72
|
+
/**
|
|
73
|
+
* The index's host and kind, from its description (once): a document
|
|
74
|
+
* index's dense_vector field and dimension or the text field it embeds,
|
|
75
|
+
* or a vector index's dimension.
|
|
76
|
+
* @private
|
|
77
|
+
*/
|
|
78
|
+
private _resolve;
|
|
79
|
+
/** @private */
|
|
80
|
+
private _describe;
|
|
81
|
+
_indexDimension: any;
|
|
82
|
+
/** A data-plane call; records the log sequence number of writes. @private */
|
|
83
|
+
private _data;
|
|
84
|
+
/** The Documents API path for this namespace. @private */
|
|
85
|
+
private _docs;
|
|
86
|
+
/**
|
|
87
|
+
* A search, repeated until it reflects this retriever's writes (when the
|
|
88
|
+
* index reports log sequence numbers) or `freshnessTimeoutMs` passes.
|
|
89
|
+
* `query` is a vector, or the query text on an index that embeds text
|
|
90
|
+
* itself. Resolves `[{ id, score, fields }]`.
|
|
91
|
+
* @private
|
|
92
|
+
*/
|
|
93
|
+
private _search;
|
|
94
|
+
/** Waits until this retriever's writes are indexed (for list and size). @private */
|
|
95
|
+
private _waitForWrites;
|
|
96
|
+
/**
|
|
97
|
+
* What list() and size() search with: a unit vector of the index's
|
|
98
|
+
* dimension, or a fixed text on an index that embeds text itself.
|
|
99
|
+
* @private
|
|
100
|
+
*/
|
|
101
|
+
private _probe;
|
|
102
|
+
/**
|
|
103
|
+
* Inserts or replaces documents. On an index that embeds text itself,
|
|
104
|
+
* Pinecone embeds `content` (required there) and `vector` is ignored.
|
|
105
|
+
* @param {Array<{ id: string, vector?: number[], content?: string, metadata?: Object }>} docs
|
|
106
|
+
* `vector` is required unless the index embeds text itself
|
|
107
|
+
* @returns {Promise<void>}
|
|
108
|
+
*/
|
|
109
|
+
store(docs: Array<{
|
|
110
|
+
id: string;
|
|
111
|
+
vector?: number[];
|
|
112
|
+
content?: string;
|
|
113
|
+
metadata?: any;
|
|
114
|
+
}>): Promise<void>;
|
|
115
|
+
/** store() on an index that embeds text itself. @private */
|
|
116
|
+
private _storeText;
|
|
117
|
+
/**
|
|
118
|
+
* The documents most similar to a query vector, or to the query text on
|
|
119
|
+
* an index that embeds text itself (Pinecone embeds the query; a
|
|
120
|
+
* `queryVector` is then ignored).
|
|
121
|
+
* @param {number[]|string|{ query?: string, queryVector?: number[], topK?: number, signal?: AbortSignal }} request
|
|
122
|
+
* A vector, the query text, or an object with them: `queryVector`, or
|
|
123
|
+
* `query` on an index that embeds text itself (required; optional in
|
|
124
|
+
* the type to match the retriever contract)
|
|
125
|
+
* @param {number} [k=10] - Results when `request` is a vector or text
|
|
126
|
+
* @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
|
|
127
|
+
*/
|
|
128
|
+
retrieve(request: number[] | string | {
|
|
129
|
+
query?: string;
|
|
130
|
+
queryVector?: number[];
|
|
131
|
+
topK?: number;
|
|
132
|
+
signal?: AbortSignal;
|
|
133
|
+
}, k?: number): Promise<Array<{
|
|
134
|
+
id: string;
|
|
135
|
+
score: number;
|
|
136
|
+
content: string;
|
|
137
|
+
metadata: any;
|
|
138
|
+
}>>;
|
|
139
|
+
/** @private */
|
|
140
|
+
private _delete;
|
|
141
|
+
/**
|
|
142
|
+
* Deletes the chunks of one source (`metadata.source`), for re-ingesting.
|
|
143
|
+
* @param {string} source
|
|
144
|
+
*/
|
|
145
|
+
removeBySource(source: string): Promise<void>;
|
|
146
|
+
/**
|
|
147
|
+
* Deletes documents by id.
|
|
148
|
+
* @param {string[]} ids
|
|
149
|
+
*/
|
|
150
|
+
delete(ids: string[]): Promise<void>;
|
|
151
|
+
/** Deletes every document this retriever stored in its namespace. */
|
|
152
|
+
clear(): Promise<void>;
|
|
153
|
+
/**
|
|
154
|
+
* Every document, without vectors.
|
|
155
|
+
* @param {{ limit?: number }} [options]
|
|
156
|
+
* @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
|
|
157
|
+
*/
|
|
158
|
+
list({ limit }?: {
|
|
159
|
+
limit?: number;
|
|
160
|
+
}): Promise<Array<{
|
|
161
|
+
id: string;
|
|
162
|
+
content: string;
|
|
163
|
+
metadata: any;
|
|
164
|
+
}>>;
|
|
165
|
+
/** @returns {Promise<number>} The documents this retriever stored */
|
|
166
|
+
size(): Promise<number>;
|
|
167
|
+
/** Documents API: fetch by filter, page by page. @private */
|
|
168
|
+
private _listDocuments;
|
|
169
|
+
/** Vectors API: list ids, then fetch them. @private */
|
|
170
|
+
private _listVectors;
|
|
171
|
+
}
|
|
172
|
+
/** The record id for a document id: itself when Pinecone accepts it. */
|
|
173
|
+
export function recordId(id: any): string;
|
|
174
|
+
export const API_VERSION: "2026-07";
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retriever on Weaviate over its REST and GraphQL APIs, with no client
|
|
3
|
+
* package. Creates the collection on first store; similarity is cosine.
|
|
4
|
+
*/
|
|
5
|
+
export class WeaviateRetriever {
|
|
6
|
+
/**
|
|
7
|
+
* @param {Object} [options]
|
|
8
|
+
* @param {string} [options.url='http://localhost:8080'] - Weaviate's REST endpoint (without /v1)
|
|
9
|
+
* @param {string} [options.apiKey] - Sent as `Authorization: Bearer <key>`
|
|
10
|
+
* @param {string} [options.collection='RagDocuments'] - Class name; the first letter is upper-cased, as Weaviate does
|
|
11
|
+
* @param {number} [options.dimensions] - Expected vector size; defaults to the collection's or the first stored vector's
|
|
12
|
+
* @param {boolean} [options.createCollection=true] - Create the collection (and any missing properties) on first store
|
|
13
|
+
* @param {string} [options.cacheKey] - Cache identity; change it when the collection changes
|
|
14
|
+
* @param {Function} [options.fetch] - fetch implementation (defaults to the global)
|
|
15
|
+
*/
|
|
16
|
+
constructor(options?: {
|
|
17
|
+
url?: string;
|
|
18
|
+
apiKey?: string;
|
|
19
|
+
collection?: string;
|
|
20
|
+
dimensions?: number;
|
|
21
|
+
createCollection?: boolean;
|
|
22
|
+
cacheKey?: string;
|
|
23
|
+
fetch?: Function;
|
|
24
|
+
});
|
|
25
|
+
url: any;
|
|
26
|
+
collection: string;
|
|
27
|
+
dimensions: number;
|
|
28
|
+
createCollection: boolean;
|
|
29
|
+
cacheKey: string;
|
|
30
|
+
_apiKey: string;
|
|
31
|
+
_fetch: any;
|
|
32
|
+
_ready: any;
|
|
33
|
+
_exists: boolean;
|
|
34
|
+
/**
|
|
35
|
+
* Cache identity for createRagPipeline({ cache }): none unless you set
|
|
36
|
+
* `cacheKey`, since other clients can change the collection.
|
|
37
|
+
* @param {'retriever'} role
|
|
38
|
+
* @returns {string|null}
|
|
39
|
+
*/
|
|
40
|
+
getCacheKey(role: "retriever"): string | null;
|
|
41
|
+
/** The message without the API key, should a server ever echo it. */
|
|
42
|
+
_redact(message: any): string;
|
|
43
|
+
/**
|
|
44
|
+
* One REST call; resolves the parsed body.
|
|
45
|
+
* @private
|
|
46
|
+
* @param {string} method
|
|
47
|
+
* @param {string} path - After /v1
|
|
48
|
+
* @param {Object} [body]
|
|
49
|
+
* @param {{ signal?: AbortSignal, allow404?: boolean }} [options] - allow404 resolves null on 404
|
|
50
|
+
* @returns {Promise<any>}
|
|
51
|
+
*/
|
|
52
|
+
private _request;
|
|
53
|
+
/** A GraphQL query; resolves `data`, throws on GraphQL errors. */
|
|
54
|
+
_graphql(query: any, signal: any): Promise<any>;
|
|
55
|
+
/** The collection's schema, or null when it doesn't exist. */
|
|
56
|
+
_schema(signal: any): Promise<any>;
|
|
57
|
+
/** Whether the collection exists; only a yes is remembered. */
|
|
58
|
+
_collectionExists(signal: any): Promise<boolean>;
|
|
59
|
+
_ensureCollection(): any;
|
|
60
|
+
/**
|
|
61
|
+
* Inserts or replaces documents.
|
|
62
|
+
* @param {Array<{ id: string, vector: number[], content?: string, metadata?: Object }>} docs
|
|
63
|
+
* @returns {Promise<void>}
|
|
64
|
+
*/
|
|
65
|
+
store(docs: Array<{
|
|
66
|
+
id: string;
|
|
67
|
+
vector: number[];
|
|
68
|
+
content?: string;
|
|
69
|
+
metadata?: any;
|
|
70
|
+
}>): Promise<void>;
|
|
71
|
+
/**
|
|
72
|
+
* The documents most similar to a query vector.
|
|
73
|
+
* @param {number[]|{ queryVector?: number[], topK?: number, signal?: AbortSignal }} request
|
|
74
|
+
* A vector, or an object with one (required; optional in the type to
|
|
75
|
+
* match the retriever contract)
|
|
76
|
+
* @param {number} [k=10] - Results when `request` is a vector
|
|
77
|
+
* @returns {Promise<Array<{ id: string, score: number, content: string, metadata: Object }>>}
|
|
78
|
+
*/
|
|
79
|
+
retrieve(request: number[] | {
|
|
80
|
+
queryVector?: number[];
|
|
81
|
+
topK?: number;
|
|
82
|
+
signal?: AbortSignal;
|
|
83
|
+
}, k?: number): Promise<Array<{
|
|
84
|
+
id: string;
|
|
85
|
+
score: number;
|
|
86
|
+
content: string;
|
|
87
|
+
metadata: any;
|
|
88
|
+
}>>;
|
|
89
|
+
/**
|
|
90
|
+
* Deletes the matching objects, a server-limited round at a time.
|
|
91
|
+
* @private
|
|
92
|
+
*/
|
|
93
|
+
private _deleteWhere;
|
|
94
|
+
/**
|
|
95
|
+
* Deletes the chunks of one source (`metadata.source`), for re-ingesting.
|
|
96
|
+
* @param {string} source
|
|
97
|
+
*/
|
|
98
|
+
removeBySource(source: string): Promise<void>;
|
|
99
|
+
/**
|
|
100
|
+
* Deletes documents by id.
|
|
101
|
+
* @param {string[]} ids
|
|
102
|
+
*/
|
|
103
|
+
delete(ids: string[]): Promise<void>;
|
|
104
|
+
/** Deletes every document this class stored; the collection stays. */
|
|
105
|
+
clear(): Promise<void>;
|
|
106
|
+
/** @returns {Promise<number>} */
|
|
107
|
+
size(): Promise<number>;
|
|
108
|
+
/**
|
|
109
|
+
* Every document, without vectors.
|
|
110
|
+
* @param {{ limit?: number }} [options]
|
|
111
|
+
* @returns {Promise<Array<{ id: string, content: string, metadata: Object }>>}
|
|
112
|
+
*/
|
|
113
|
+
list({ limit }?: {
|
|
114
|
+
limit?: number;
|
|
115
|
+
}): Promise<Array<{
|
|
116
|
+
id: string;
|
|
117
|
+
content: string;
|
|
118
|
+
metadata: any;
|
|
119
|
+
}>>;
|
|
120
|
+
}
|
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
* Load an optional peer dependency. `load` must be a literal
|
|
3
3
|
* `() => require('name')`, so bundlers (esbuild, Wrangler, webpack) can see
|
|
4
4
|
* and include the package; probing with require.resolve() instead fails in
|
|
5
|
-
* bundles even when the package is there.
|
|
5
|
+
* bundles even when the package is there. A bundle of a project without the
|
|
6
|
+
* package fails to build, though, because the require isn't inside a
|
|
7
|
+
* `try`: loaders on the main entry's path use `missingPeer` with the
|
|
8
|
+
* require in their own `try` instead.
|
|
6
9
|
*
|
|
7
10
|
* Only the package itself being absent becomes a `MISSING_DEPENDENCY` error
|
|
8
11
|
* carrying `hint`; any other failure, including a module missing inside the
|
|
@@ -14,3 +17,20 @@
|
|
|
14
17
|
* @returns {any} The module
|
|
15
18
|
*/
|
|
16
19
|
export function loadOptionalPeer(name: string, load: () => any, hint: string): any;
|
|
20
|
+
/**
|
|
21
|
+
* For a loader that requires an optional peer inside its own `try`, which
|
|
22
|
+
* bundlers (esbuild, Wrangler) then include when the package is installed
|
|
23
|
+
* and leave out, without failing the build, when it isn't:
|
|
24
|
+
*
|
|
25
|
+
* try {
|
|
26
|
+
* return require('openai');
|
|
27
|
+
* } catch (error) {
|
|
28
|
+
* throw missingPeer(error, 'openai', hint);
|
|
29
|
+
* }
|
|
30
|
+
*
|
|
31
|
+
* @param {Error} error - What require() threw
|
|
32
|
+
* @param {string} name - The package name
|
|
33
|
+
* @param {string} hint - The error message when the package is missing
|
|
34
|
+
* @returns {Error} A `MISSING_DEPENDENCY` error carrying `hint` when the package itself is absent; otherwise `error`
|
|
35
|
+
*/
|
|
36
|
+
export function missingPeer(error: Error, name: string, hint: string): Error;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devilsdev/rag-core",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.7.0",
|
|
4
4
|
"description": "createRagPipeline and its building blocks: retrieval, reranking, caching, cost tracking and model routing.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"rag",
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
"homepage": "https://devilsdev.github.io/rag-pipeline-utils/",
|
|
14
14
|
"repository": {
|
|
15
15
|
"type": "git",
|
|
16
|
-
"url": "https://github.com/DevilsDev/rag-pipeline-utils.git"
|
|
16
|
+
"url": "git+https://github.com/DevilsDev/rag-pipeline-utils.git"
|
|
17
17
|
},
|
|
18
18
|
"bugs": {
|
|
19
19
|
"url": "https://github.com/DevilsDev/rag-pipeline-utils/issues"
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
"exports": {
|
|
27
27
|
".": {
|
|
28
28
|
"types": "./dist/index.d.ts",
|
|
29
|
+
"workerd": "./dist/index.cjs",
|
|
29
30
|
"require": "./dist/index.cjs",
|
|
30
31
|
"import": "./dist/index.mjs"
|
|
31
32
|
},
|