@absolutejs/rag 0.0.9 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai/client/index.js +49 -29
- package/dist/ai/client/index.js.map +3 -3
- package/dist/ai/client/ui.js +49 -29
- package/dist/ai/client/ui.js.map +3 -3
- package/dist/ai/rag/index.js +161 -47
- package/dist/ai/rag/index.js.map +7 -6
- package/dist/ai/rag/quality.js +49 -29
- package/dist/ai/rag/quality.js.map +3 -3
- package/dist/ai/rag/ui.js +49 -29
- package/dist/ai/rag/ui.js.map +3 -3
- package/dist/angular/ai/index.js +2 -2
- package/dist/angular/ai/index.js.map +2 -2
- package/dist/react/ai/index.js +2 -2
- package/dist/react/ai/index.js.map +2 -2
- package/dist/src/ai/rag/index.d.ts +2 -0
- package/dist/src/ai/rag/rerankerProviders.d.ts +14 -0
- package/dist/svelte/ai/index.js +2 -2
- package/dist/svelte/ai/index.js.map +2 -2
- package/dist/vue/ai/index.js +2 -2
- package/dist/vue/ai/index.js.map +2 -2
- package/package.json +1 -1
package/dist/ai/rag/index.js
CHANGED
|
@@ -3,22 +3,42 @@
|
|
|
3
3
|
import { mkdir, readFile } from "fs/promises";
|
|
4
4
|
|
|
5
5
|
// src/ai/rag/atomicWrite.ts
|
|
6
|
-
import { rename, unlink
|
|
6
|
+
import { open, rename, unlink } from "fs/promises";
|
|
7
|
+
import { dirname } from "path";
|
|
7
8
|
var writeFileAtomic = async (path, data, encoding = "utf8") => {
|
|
8
9
|
const tmpPath = `${path}.tmp.${process.pid}.${Math.random().toString(36).slice(2, 10)}`;
|
|
10
|
+
const buffer = Buffer.from(data, encoding);
|
|
11
|
+
let renamed = false;
|
|
9
12
|
try {
|
|
10
|
-
await
|
|
13
|
+
const fileHandle = await open(tmpPath, "w");
|
|
14
|
+
try {
|
|
15
|
+
await fileHandle.writeFile(buffer);
|
|
16
|
+
await fileHandle.sync();
|
|
17
|
+
} finally {
|
|
18
|
+
await fileHandle.close();
|
|
19
|
+
}
|
|
11
20
|
await rename(tmpPath, path);
|
|
12
|
-
|
|
21
|
+
renamed = true;
|
|
13
22
|
try {
|
|
14
|
-
await
|
|
23
|
+
const dirHandle = await open(dirname(path), "r");
|
|
24
|
+
try {
|
|
25
|
+
await dirHandle.sync();
|
|
26
|
+
} finally {
|
|
27
|
+
await dirHandle.close();
|
|
28
|
+
}
|
|
15
29
|
} catch {}
|
|
30
|
+
} catch (error) {
|
|
31
|
+
if (!renamed) {
|
|
32
|
+
try {
|
|
33
|
+
await unlink(tmpPath);
|
|
34
|
+
} catch {}
|
|
35
|
+
}
|
|
16
36
|
throw error;
|
|
17
37
|
}
|
|
18
38
|
};
|
|
19
39
|
|
|
20
40
|
// src/ai/rag/quality.ts
|
|
21
|
-
import { dirname } from "path";
|
|
41
|
+
import { dirname as dirname2 } from "path";
|
|
22
42
|
import { generateId } from "@absolutejs/ai";
|
|
23
43
|
|
|
24
44
|
// src/ai/rag/grounding.ts
|
|
@@ -6521,7 +6541,7 @@ var createRAGFileEvaluationHistoryStore = (path) => ({
|
|
|
6521
6541
|
run,
|
|
6522
6542
|
...existing.filter((entry) => entry.id !== run.id)
|
|
6523
6543
|
]);
|
|
6524
|
-
await mkdir(
|
|
6544
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
6525
6545
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
6526
6546
|
`, "utf8");
|
|
6527
6547
|
},
|
|
@@ -6541,7 +6561,7 @@ var createRAGFileEvaluationHistoryStore = (path) => ({
|
|
|
6541
6561
|
runs: existing,
|
|
6542
6562
|
sort: normalizeHistoryRuns
|
|
6543
6563
|
});
|
|
6544
|
-
await mkdir(
|
|
6564
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
6545
6565
|
await writeFileAtomic(path, JSON.stringify(pruned.next, null, "\t") + `
|
|
6546
6566
|
`, "utf8");
|
|
6547
6567
|
return {
|
|
@@ -6592,7 +6612,7 @@ var createRAGFileEvaluationSuiteSnapshotHistoryStore = (path) => ({
|
|
|
6592
6612
|
snapshot,
|
|
6593
6613
|
...existing.filter((entry) => entry.id !== snapshot.id)
|
|
6594
6614
|
]);
|
|
6595
|
-
await mkdir(
|
|
6615
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
6596
6616
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
6597
6617
|
`, "utf8");
|
|
6598
6618
|
},
|
|
@@ -6612,7 +6632,7 @@ var createRAGFileEvaluationSuiteSnapshotHistoryStore = (path) => ({
|
|
|
6612
6632
|
runs: existing,
|
|
6613
6633
|
sort: normalizeEvaluationSuiteSnapshots
|
|
6614
6634
|
});
|
|
6615
|
-
await mkdir(
|
|
6635
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
6616
6636
|
await writeFileAtomic(path, JSON.stringify(pruned.next, null, "\t") + `
|
|
6617
6637
|
`, "utf8");
|
|
6618
6638
|
return {
|
|
@@ -6655,7 +6675,7 @@ var createRAGFileRetrievalComparisonHistoryStore = (path) => ({
|
|
|
6655
6675
|
run,
|
|
6656
6676
|
...existing.filter((entry) => entry.id !== run.id)
|
|
6657
6677
|
]);
|
|
6658
|
-
await mkdir(
|
|
6678
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
6659
6679
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
6660
6680
|
`, "utf8");
|
|
6661
6681
|
}
|
|
@@ -6709,7 +6729,7 @@ var createRAGFileRetrievalBaselineStore = (path) => {
|
|
|
6709
6729
|
},
|
|
6710
6730
|
...existing.map((entry) => entry.groupKey === record.groupKey && (entry.rolloutLabel ?? undefined) === (record.rolloutLabel ?? undefined) ? { ...entry, status: "superseded" } : entry)
|
|
6711
6731
|
]);
|
|
6712
|
-
await mkdir(
|
|
6732
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
6713
6733
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
6714
6734
|
`, "utf8");
|
|
6715
6735
|
}
|
|
@@ -6748,7 +6768,7 @@ var createRAGFileRetrievalReleaseDecisionStore = (path) => ({
|
|
|
6748
6768
|
record,
|
|
6749
6769
|
...existing.filter((entry) => entry.id !== record.id)
|
|
6750
6770
|
]);
|
|
6751
|
-
await mkdir(
|
|
6771
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
6752
6772
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
6753
6773
|
`, "utf8");
|
|
6754
6774
|
}
|
|
@@ -6792,7 +6812,7 @@ var createRAGFileRetrievalLaneHandoffDecisionStore = (path) => ({
|
|
|
6792
6812
|
record,
|
|
6793
6813
|
...existing.filter((entry) => entry.id !== record.id)
|
|
6794
6814
|
]);
|
|
6795
|
-
await mkdir(
|
|
6815
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
6796
6816
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
6797
6817
|
`, "utf8");
|
|
6798
6818
|
}
|
|
@@ -6837,7 +6857,7 @@ var createRAGFileRetrievalReleaseIncidentStore = (path) => ({
|
|
|
6837
6857
|
record,
|
|
6838
6858
|
...existing.filter((entry) => entry.id !== record.id)
|
|
6839
6859
|
]);
|
|
6840
|
-
await mkdir(
|
|
6860
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
6841
6861
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
6842
6862
|
`, "utf8");
|
|
6843
6863
|
}
|
|
@@ -6881,7 +6901,7 @@ var createRAGFileRetrievalLaneHandoffIncidentStore = (path) => ({
|
|
|
6881
6901
|
record,
|
|
6882
6902
|
...existing.filter((entry) => entry.id !== record.id)
|
|
6883
6903
|
]);
|
|
6884
|
-
await mkdir(
|
|
6904
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
6885
6905
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
6886
6906
|
`, "utf8");
|
|
6887
6907
|
}
|
|
@@ -6921,7 +6941,7 @@ var createRAGFileRetrievalLaneHandoffIncidentHistoryStore = (path) => ({
|
|
|
6921
6941
|
}
|
|
6922
6942
|
})();
|
|
6923
6943
|
const next = [record, ...existing].sort((left, right) => right.recordedAt - left.recordedAt);
|
|
6924
|
-
await mkdir(
|
|
6944
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
6925
6945
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
6926
6946
|
`, "utf8");
|
|
6927
6947
|
}
|
|
@@ -6965,7 +6985,7 @@ var createRAGFileRetrievalIncidentRemediationDecisionStore = (path) => ({
|
|
|
6965
6985
|
record,
|
|
6966
6986
|
...existing
|
|
6967
6987
|
]);
|
|
6968
|
-
await mkdir(
|
|
6988
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
6969
6989
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
6970
6990
|
`, "utf8");
|
|
6971
6991
|
}
|
|
@@ -7011,7 +7031,7 @@ var createRAGFileRetrievalIncidentRemediationExecutionHistoryStore = (path) => (
|
|
|
7011
7031
|
record,
|
|
7012
7032
|
...existing
|
|
7013
7033
|
]);
|
|
7014
|
-
await mkdir(
|
|
7034
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
7015
7035
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
7016
7036
|
`, "utf8");
|
|
7017
7037
|
}
|
|
@@ -7045,7 +7065,7 @@ var createRAGFileRetrievalLaneHandoffAutoCompletePolicyHistoryStore = (path) =>
|
|
|
7045
7065
|
}
|
|
7046
7066
|
})();
|
|
7047
7067
|
const next = [record, ...existing].sort((left, right) => right.recordedAt - left.recordedAt);
|
|
7048
|
-
await mkdir(
|
|
7068
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
7049
7069
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
7050
7070
|
`, "utf8");
|
|
7051
7071
|
}
|
|
@@ -7079,7 +7099,7 @@ var createRAGFileRetrievalReleaseLanePolicyHistoryStore = (path) => ({
|
|
|
7079
7099
|
}
|
|
7080
7100
|
})();
|
|
7081
7101
|
const next = [record, ...existing].sort((left, right) => right.recordedAt - left.recordedAt);
|
|
7082
|
-
await mkdir(
|
|
7102
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
7083
7103
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
7084
7104
|
`, "utf8");
|
|
7085
7105
|
}
|
|
@@ -7113,7 +7133,7 @@ var createRAGFileRetrievalBaselineGatePolicyHistoryStore = (path) => ({
|
|
|
7113
7133
|
}
|
|
7114
7134
|
})();
|
|
7115
7135
|
const next = [record, ...existing].sort((left, right) => right.recordedAt - left.recordedAt);
|
|
7116
|
-
await mkdir(
|
|
7136
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
7117
7137
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
7118
7138
|
`, "utf8");
|
|
7119
7139
|
}
|
|
@@ -7147,7 +7167,7 @@ var createRAGFileRetrievalReleaseLaneEscalationPolicyHistoryStore = (path) => ({
|
|
|
7147
7167
|
}
|
|
7148
7168
|
})();
|
|
7149
7169
|
const next = [record, ...existing].sort((left, right) => right.recordedAt - left.recordedAt);
|
|
7150
|
-
await mkdir(
|
|
7170
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
7151
7171
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
7152
7172
|
`, "utf8");
|
|
7153
7173
|
}
|
|
@@ -7193,7 +7213,7 @@ var createRAGFileSearchTraceStore = (path) => ({
|
|
|
7193
7213
|
trace,
|
|
7194
7214
|
...traces.filter((entry) => entry.id !== trace.id)
|
|
7195
7215
|
]);
|
|
7196
|
-
await mkdir(
|
|
7216
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
7197
7217
|
await writeFileAtomic(path, JSON.stringify({
|
|
7198
7218
|
traces: nextTraces
|
|
7199
7219
|
}, null, 2));
|
|
@@ -7213,7 +7233,7 @@ var createRAGFileSearchTraceStore = (path) => ({
|
|
|
7213
7233
|
input,
|
|
7214
7234
|
traces
|
|
7215
7235
|
});
|
|
7216
|
-
await mkdir(
|
|
7236
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
7217
7237
|
await writeFileAtomic(path, JSON.stringify({
|
|
7218
7238
|
traces: pruned.next
|
|
7219
7239
|
}, null, 2));
|
|
@@ -7256,7 +7276,7 @@ var createRAGFileSearchTracePruneHistoryStore = (path) => ({
|
|
|
7256
7276
|
run,
|
|
7257
7277
|
...existing.filter((entry) => entry.id !== run.id)
|
|
7258
7278
|
]);
|
|
7259
|
-
await mkdir(
|
|
7279
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
7260
7280
|
await writeFileAtomic(path, JSON.stringify(next, null, "\t") + `
|
|
7261
7281
|
`, "utf8");
|
|
7262
7282
|
}
|
|
@@ -8885,7 +8905,7 @@ var createRAGFileAnswerGroundingEvaluationHistoryStore = (path) => ({
|
|
|
8885
8905
|
run,
|
|
8886
8906
|
...runs.filter((entry) => entry.id !== run.id)
|
|
8887
8907
|
]);
|
|
8888
|
-
await mkdir(
|
|
8908
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
8889
8909
|
await writeFileAtomic(path, JSON.stringify({
|
|
8890
8910
|
runs: nextRuns
|
|
8891
8911
|
}, null, 2));
|
|
@@ -8906,7 +8926,7 @@ var createRAGFileAnswerGroundingEvaluationHistoryStore = (path) => ({
|
|
|
8906
8926
|
runs,
|
|
8907
8927
|
sort: normalizeGroundingHistoryRuns
|
|
8908
8928
|
});
|
|
8909
|
-
await mkdir(
|
|
8929
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
8910
8930
|
await writeFileAtomic(path, JSON.stringify({
|
|
8911
8931
|
runs: pruned.next
|
|
8912
8932
|
}, null, 2));
|
|
@@ -8946,7 +8966,7 @@ var createRAGFileAnswerGroundingCaseDifficultyHistoryStore = (path) => ({
|
|
|
8946
8966
|
run,
|
|
8947
8967
|
...runs.filter((entry) => entry.id !== run.id)
|
|
8948
8968
|
]);
|
|
8949
|
-
await mkdir(
|
|
8969
|
+
await mkdir(dirname2(path), { recursive: true });
|
|
8950
8970
|
await writeFileAtomic(path, JSON.stringify({
|
|
8951
8971
|
runs: nextRuns
|
|
8952
8972
|
}, null, 2));
|
|
@@ -13242,6 +13262,7 @@ var validateRAGEmbeddingDimensions = (vector, expectedDimensions, context) => {
|
|
|
13242
13262
|
import { readdir, readFile as readFile2 } from "fs/promises";
|
|
13243
13263
|
import { basename, extname, join, relative, resolve } from "path";
|
|
13244
13264
|
import { gunzipSync, inflateRawSync } from "zlib";
|
|
13265
|
+
var h2IfHttps = (url) => url.startsWith("https://") ? { protocol: "http2" } : {};
|
|
13245
13266
|
var DEFAULT_MAX_CHUNK_LENGTH = 900;
|
|
13246
13267
|
var DEFAULT_CHUNK_OVERLAP = 120;
|
|
13247
13268
|
var DEFAULT_MIN_CHUNK_LENGTH = 80;
|
|
@@ -19420,7 +19441,7 @@ var loadRAGDocumentFromURL = async (input) => {
|
|
|
19420
19441
|
if (!url) {
|
|
19421
19442
|
throw new Error("RAG URL is required");
|
|
19422
19443
|
}
|
|
19423
|
-
const response = await fetch(url);
|
|
19444
|
+
const response = await fetch(url, h2IfHttps(url));
|
|
19424
19445
|
if (!response.ok) {
|
|
19425
19446
|
throw new Error(`Failed to fetch RAG URL ${url}: ${response.status} ${response.statusText}`);
|
|
19426
19447
|
}
|
|
@@ -19468,7 +19489,7 @@ var loadRAGDocumentsFromURLs = async (input) => {
|
|
|
19468
19489
|
if (!url) {
|
|
19469
19490
|
throw new Error("RAG URL is required");
|
|
19470
19491
|
}
|
|
19471
|
-
const response = await fetch(url);
|
|
19492
|
+
const response = await fetch(url, h2IfHttps(url));
|
|
19472
19493
|
if (!response.ok) {
|
|
19473
19494
|
throw new Error(`Failed to fetch RAG URL ${url}: ${response.status} ${response.statusText}`);
|
|
19474
19495
|
}
|
|
@@ -30845,6 +30866,94 @@ var ragChat = (config) => {
|
|
|
30845
30866
|
// src/ai/rag/htmxConfig.ts
|
|
30846
30867
|
var createRAGHTMXConfig = (config) => config;
|
|
30847
30868
|
var createRAGHTMXWorkflowRenderConfig = (config) => config;
|
|
30869
|
+
// src/ai/rag/rerankerProviders.ts
|
|
30870
|
+
var rowScore = (row) => row.relevance_score ?? row.relevanceScore ?? row.score ?? 0;
|
|
30871
|
+
var applyRanking = (candidates, rows, topK) => {
|
|
30872
|
+
const ranked = [];
|
|
30873
|
+
for (const row of rows) {
|
|
30874
|
+
const candidate = candidates[row.index];
|
|
30875
|
+
if (!candidate)
|
|
30876
|
+
continue;
|
|
30877
|
+
ranked.push({ ...candidate, score: rowScore(row) });
|
|
30878
|
+
}
|
|
30879
|
+
ranked.sort((left, right) => right.score - left.score);
|
|
30880
|
+
return ranked.slice(0, topK);
|
|
30881
|
+
};
|
|
30882
|
+
var limitCandidates = (input) => {
|
|
30883
|
+
const cap = input.candidateTopK ?? input.results.length;
|
|
30884
|
+
return input.results.slice(0, Math.max(0, cap));
|
|
30885
|
+
};
|
|
30886
|
+
var createHttpCrossEncoderReranker = (options) => {
|
|
30887
|
+
const fetchImpl = options.config.fetch ?? fetch;
|
|
30888
|
+
const defaultModel = options.config.defaultModel ?? options.fallbackModel;
|
|
30889
|
+
return {
|
|
30890
|
+
defaultModel,
|
|
30891
|
+
providerName: options.providerName,
|
|
30892
|
+
rerank: async (input) => {
|
|
30893
|
+
const candidates = limitCandidates(input);
|
|
30894
|
+
if (candidates.length === 0)
|
|
30895
|
+
return [];
|
|
30896
|
+
const model = input.model ?? defaultModel;
|
|
30897
|
+
const documents = candidates.map((candidate) => candidate.chunkText);
|
|
30898
|
+
const topN = Math.min(input.topK, candidates.length);
|
|
30899
|
+
const response = await fetchImpl(options.endpoint, {
|
|
30900
|
+
body: JSON.stringify(options.buildBody({ documents, model, query: input.query, topN })),
|
|
30901
|
+
headers: {
|
|
30902
|
+
Authorization: `Bearer ${options.config.apiKey}`,
|
|
30903
|
+
"Content-Type": "application/json",
|
|
30904
|
+
...options.config.headers
|
|
30905
|
+
},
|
|
30906
|
+
method: "POST"
|
|
30907
|
+
});
|
|
30908
|
+
if (!response.ok) {
|
|
30909
|
+
throw new Error(`${options.providerName} rerank failed: HTTP ${response.status}`);
|
|
30910
|
+
}
|
|
30911
|
+
const payload = await response.json();
|
|
30912
|
+
const rows = payload.results ?? payload.data ?? [];
|
|
30913
|
+
const ranked = applyRanking(candidates, rows, input.topK);
|
|
30914
|
+
if (input.scoreThreshold !== undefined) {
|
|
30915
|
+
return ranked.filter((result) => result.score >= input.scoreThreshold);
|
|
30916
|
+
}
|
|
30917
|
+
return ranked;
|
|
30918
|
+
}
|
|
30919
|
+
};
|
|
30920
|
+
};
|
|
30921
|
+
var createCohereRAGReranker = (config) => createHttpCrossEncoderReranker({
|
|
30922
|
+
buildBody: ({ model, query, documents, topN }) => ({
|
|
30923
|
+
documents,
|
|
30924
|
+
model,
|
|
30925
|
+
query,
|
|
30926
|
+
top_n: topN
|
|
30927
|
+
}),
|
|
30928
|
+
config,
|
|
30929
|
+
endpoint: `${config.baseUrl ?? "https://api.cohere.com"}/v2/rerank`,
|
|
30930
|
+
fallbackModel: "rerank-v3.5",
|
|
30931
|
+
providerName: "cohere"
|
|
30932
|
+
});
|
|
30933
|
+
var createVoyageRAGReranker = (config) => createHttpCrossEncoderReranker({
|
|
30934
|
+
buildBody: ({ model, query, documents, topN }) => ({
|
|
30935
|
+
documents,
|
|
30936
|
+
model,
|
|
30937
|
+
query,
|
|
30938
|
+
top_k: topN
|
|
30939
|
+
}),
|
|
30940
|
+
config,
|
|
30941
|
+
endpoint: `${config.baseUrl ?? "https://api.voyageai.com"}/v1/rerank`,
|
|
30942
|
+
fallbackModel: "rerank-2",
|
|
30943
|
+
providerName: "voyage"
|
|
30944
|
+
});
|
|
30945
|
+
var createJinaRAGReranker = (config) => createHttpCrossEncoderReranker({
|
|
30946
|
+
buildBody: ({ model, query, documents, topN }) => ({
|
|
30947
|
+
documents,
|
|
30948
|
+
model,
|
|
30949
|
+
query,
|
|
30950
|
+
top_n: topN
|
|
30951
|
+
}),
|
|
30952
|
+
config,
|
|
30953
|
+
endpoint: `${config.baseUrl ?? "https://api.jina.ai"}/v1/rerank`,
|
|
30954
|
+
fallbackModel: "jina-reranker-v2-base-multilingual",
|
|
30955
|
+
providerName: "jina"
|
|
30956
|
+
});
|
|
30848
30957
|
// src/ai/rag/retrievalStrategies.ts
|
|
30849
30958
|
var tokenize4 = (value) => value.toLowerCase().split(/[^a-z0-9]+/i).map((token) => token.trim()).filter((token) => token.length > 0);
|
|
30850
30959
|
var hasAnyToken2 = (tokens, values) => values.some((value) => tokens.includes(value));
|
|
@@ -32155,7 +32264,8 @@ var createRAGInstagramBusinessConnector = (input) => ({
|
|
|
32155
32264
|
var {S3Client } = globalThis.Bun;
|
|
32156
32265
|
import { createHash } from "crypto";
|
|
32157
32266
|
import { mkdir as mkdir2, readFile as readFile3, readdir as readdir2 } from "fs/promises";
|
|
32158
|
-
import { basename as basename2, dirname as
|
|
32267
|
+
import { basename as basename2, dirname as dirname3, extname as extname2, join as join2, relative as relative2, resolve as resolve2 } from "path";
|
|
32268
|
+
var h2IfHttps2 = (url) => url.startsWith("https://") ? { protocol: "http2" } : {};
|
|
32159
32269
|
var toSyncError = (caught) => caught instanceof Error ? caught.message : String(caught);
|
|
32160
32270
|
var wait = async (delayMs) => {
|
|
32161
32271
|
if (!(delayMs > 0)) {
|
|
@@ -32968,7 +33078,7 @@ var isFeedDocument = (value) => {
|
|
|
32968
33078
|
return trimmed.includes("<rss") || trimmed.includes("<channel") || trimmed.includes("<feed") || trimmed.includes("<entry");
|
|
32969
33079
|
};
|
|
32970
33080
|
var discoverFeedsFromHTML = async (feed) => {
|
|
32971
|
-
const response = await fetch(feed.url);
|
|
33081
|
+
const response = await fetch(feed.url, h2IfHttps2(feed.url));
|
|
32972
33082
|
if (!response.ok) {
|
|
32973
33083
|
return [];
|
|
32974
33084
|
}
|
|
@@ -33014,7 +33124,7 @@ var discoverFeedsFromHTML = async (feed) => {
|
|
|
33014
33124
|
}
|
|
33015
33125
|
const validated = [];
|
|
33016
33126
|
for (const candidate of discovered.values()) {
|
|
33017
|
-
const candidateResponse = await fetch(candidate.url);
|
|
33127
|
+
const candidateResponse = await fetch(candidate.url, h2IfHttps2(candidate.url));
|
|
33018
33128
|
if (!candidateResponse.ok) {
|
|
33019
33129
|
continue;
|
|
33020
33130
|
}
|
|
@@ -33078,7 +33188,7 @@ var parseSitemapEntries = (sitemap, value) => {
|
|
|
33078
33188
|
};
|
|
33079
33189
|
var discoverSitemapsFromRobots = async (sitemap) => {
|
|
33080
33190
|
const robotsURL = resolveSiblingURL(sitemap.url, "/robots.txt");
|
|
33081
|
-
const response = await fetch(robotsURL);
|
|
33191
|
+
const response = await fetch(robotsURL, h2IfHttps2(robotsURL));
|
|
33082
33192
|
if (!response.ok) {
|
|
33083
33193
|
return [];
|
|
33084
33194
|
}
|
|
@@ -33099,7 +33209,7 @@ var discoverSitemapsFromRobots = async (sitemap) => {
|
|
|
33099
33209
|
};
|
|
33100
33210
|
var loadRobotsDisallowRules = async (siteURL) => {
|
|
33101
33211
|
const robotsURL = resolveSiblingURL(siteURL, "/robots.txt");
|
|
33102
|
-
const response = await fetch(robotsURL);
|
|
33212
|
+
const response = await fetch(robotsURL, h2IfHttps2(robotsURL));
|
|
33103
33213
|
if (!response.ok) {
|
|
33104
33214
|
return [];
|
|
33105
33215
|
}
|
|
@@ -33140,7 +33250,7 @@ var discoverRecursiveSitemapURLs = async (input) => {
|
|
|
33140
33250
|
}
|
|
33141
33251
|
seen.add(current.sitemap.url);
|
|
33142
33252
|
resolved.push(current.sitemap);
|
|
33143
|
-
const response = await fetch(current.sitemap.url);
|
|
33253
|
+
const response = await fetch(current.sitemap.url, h2IfHttps2(current.sitemap.url));
|
|
33144
33254
|
if (!response.ok) {
|
|
33145
33255
|
throw new Error(`Failed to load sitemap ${current.sitemap.url}: ${response.status} ${response.statusText}`);
|
|
33146
33256
|
}
|
|
@@ -33291,7 +33401,7 @@ var discoverLinkedPagesFromHTML = async (input) => {
|
|
|
33291
33401
|
pruneCounts.robotsBlockedCount += 1;
|
|
33292
33402
|
continue;
|
|
33293
33403
|
}
|
|
33294
|
-
const response = await fetch(current.url);
|
|
33404
|
+
const response = await fetch(current.url, h2IfHttps2(current.url));
|
|
33295
33405
|
if (!response.ok) {
|
|
33296
33406
|
continue;
|
|
33297
33407
|
}
|
|
@@ -33546,6 +33656,7 @@ var loadDiscoveredGitHubRepositoryFiles = async (input) => {
|
|
|
33546
33656
|
repo: input.repo
|
|
33547
33657
|
});
|
|
33548
33658
|
const response = await fetch(requestURL, {
|
|
33659
|
+
...h2IfHttps2(requestURL),
|
|
33549
33660
|
headers: input.requestHeaders
|
|
33550
33661
|
});
|
|
33551
33662
|
if (!response.ok) {
|
|
@@ -33709,7 +33820,7 @@ var createRAGFeedSyncSource = (options) => ({
|
|
|
33709
33820
|
}
|
|
33710
33821
|
}
|
|
33711
33822
|
const discoveredEntries = (await Promise.all([...feedMap.values()].map(async (feed) => {
|
|
33712
|
-
const response = await fetch(feed.url);
|
|
33823
|
+
const response = await fetch(feed.url, h2IfHttps2(feed.url));
|
|
33713
33824
|
if (!response.ok) {
|
|
33714
33825
|
throw new Error(`Failed to load feed ${feed.url}: ${response.status} ${response.statusText}`);
|
|
33715
33826
|
}
|
|
@@ -33783,7 +33894,7 @@ var createRAGSitemapSyncSource = (options) => ({
|
|
|
33783
33894
|
}
|
|
33784
33895
|
}
|
|
33785
33896
|
const discoveredEntries = (await Promise.all([...resolvedSitemapMap.values()].map(async (sitemap) => {
|
|
33786
|
-
const response = await fetch(sitemap.url);
|
|
33897
|
+
const response = await fetch(sitemap.url, h2IfHttps2(sitemap.url));
|
|
33787
33898
|
if (!response.ok) {
|
|
33788
33899
|
throw new Error(`Failed to load sitemap ${sitemap.url}: ${response.status} ${response.statusText}`);
|
|
33789
33900
|
}
|
|
@@ -33848,7 +33959,7 @@ var createRAGSiteDiscoverySyncSource = (options) => ({
|
|
|
33848
33959
|
}
|
|
33849
33960
|
}
|
|
33850
33961
|
const feedEntries = (await Promise.all([...feedMap.values()].map(async (feed) => {
|
|
33851
|
-
const response = await fetch(feed.url);
|
|
33962
|
+
const response = await fetch(feed.url, h2IfHttps2(feed.url));
|
|
33852
33963
|
if (!response.ok) {
|
|
33853
33964
|
throw new Error(`Failed to load feed ${feed.url}: ${response.status} ${response.statusText}`);
|
|
33854
33965
|
}
|
|
@@ -33898,7 +34009,7 @@ var createRAGSiteDiscoverySyncSource = (options) => ({
|
|
|
33898
34009
|
sitemap
|
|
33899
34010
|
})))).flat();
|
|
33900
34011
|
const sitemapEntries = (await Promise.all(resolvedSitemaps.map(async (sitemap) => {
|
|
33901
|
-
const response = await fetch(sitemap.url);
|
|
34012
|
+
const response = await fetch(sitemap.url, h2IfHttps2(sitemap.url));
|
|
33902
34013
|
if (!response.ok) {
|
|
33903
34014
|
throw new Error(`Failed to load sitemap ${sitemap.url}: ${response.status} ${response.statusText}`);
|
|
33904
34015
|
}
|
|
@@ -34652,7 +34763,7 @@ var createRAGFileSyncStateStore = (path) => {
|
|
|
34652
34763
|
}
|
|
34653
34764
|
},
|
|
34654
34765
|
save: async (records) => {
|
|
34655
|
-
await mkdir2(
|
|
34766
|
+
await mkdir2(dirname3(resolvedPath), { recursive: true });
|
|
34656
34767
|
await writeFileAtomic(resolvedPath, JSON.stringify(records, null, 2), "utf8");
|
|
34657
34768
|
}
|
|
34658
34769
|
};
|
|
@@ -34701,7 +34812,7 @@ var createRAGSyncScheduler = (input) => {
|
|
|
34701
34812
|
};
|
|
34702
34813
|
// src/ai/rag/jobState.ts
|
|
34703
34814
|
import { mkdir as mkdir3, readFile as readFile4 } from "fs/promises";
|
|
34704
|
-
import { dirname as
|
|
34815
|
+
import { dirname as dirname4, resolve as resolve3 } from "path";
|
|
34705
34816
|
var parseJobState = (content) => {
|
|
34706
34817
|
try {
|
|
34707
34818
|
const parsed = JSON.parse(content);
|
|
@@ -34736,7 +34847,7 @@ var createRAGFileJobStateStore = (path) => {
|
|
|
34736
34847
|
}
|
|
34737
34848
|
},
|
|
34738
34849
|
save: async (state) => {
|
|
34739
|
-
await mkdir3(
|
|
34850
|
+
await mkdir3(dirname4(resolvedPath), { recursive: true });
|
|
34740
34851
|
await writeFileAtomic(resolvedPath, JSON.stringify(state, null, 2), "utf8");
|
|
34741
34852
|
}
|
|
34742
34853
|
};
|
|
@@ -35969,7 +36080,7 @@ import { existsSync as existsSync2 } from "fs";
|
|
|
35969
36080
|
// src/ai/rag/resolveAbsoluteSQLiteVec.ts
|
|
35970
36081
|
import { existsSync, readFileSync } from "fs";
|
|
35971
36082
|
import { arch, platform } from "os";
|
|
35972
|
-
import { dirname as
|
|
36083
|
+
import { dirname as dirname5, join as join3 } from "path";
|
|
35973
36084
|
var PLATFORM_PACKAGE_MAP = {
|
|
35974
36085
|
"darwin-arm64": {
|
|
35975
36086
|
libraryFile: "vec0.dylib",
|
|
@@ -36023,7 +36134,7 @@ var resolveAbsoluteSQLiteVec = () => {
|
|
|
36023
36134
|
throw new Error("AbsoluteJS sqlite-vec package resolution requires import.meta.resolve support.");
|
|
36024
36135
|
}
|
|
36025
36136
|
const packageJsonPath = new URL(resolve4(`${packageInfo.packageName}/package.json`)).pathname;
|
|
36026
|
-
const packageRoot =
|
|
36137
|
+
const packageRoot = dirname5(packageJsonPath);
|
|
36027
36138
|
const libraryPath = join3(packageRoot, packageInfo.libraryFile);
|
|
36028
36139
|
const packageVersion = readPackageVersion(packageJsonPath);
|
|
36029
36140
|
if (!existsSync(libraryPath)) {
|
|
@@ -37305,6 +37416,7 @@ export {
|
|
|
37305
37416
|
evaluateRAGAnswerGroundingCase,
|
|
37306
37417
|
evaluateRAGAnswerGrounding,
|
|
37307
37418
|
deepseekEmbeddings,
|
|
37419
|
+
createVoyageRAGReranker,
|
|
37308
37420
|
createTextFileExtractor,
|
|
37309
37421
|
createSQLiteRAGStore,
|
|
37310
37422
|
createRAGVector,
|
|
@@ -37402,12 +37514,14 @@ export {
|
|
|
37402
37514
|
createPDFFileExtractor,
|
|
37403
37515
|
createOfficeDocumentExtractor,
|
|
37404
37516
|
createLegacyDocumentExtractor,
|
|
37517
|
+
createJinaRAGReranker,
|
|
37405
37518
|
createInMemoryRAGStore,
|
|
37406
37519
|
createHeuristicRAGRetrievalStrategy,
|
|
37407
37520
|
createHeuristicRAGReranker,
|
|
37408
37521
|
createHeuristicRAGQueryTransform,
|
|
37409
37522
|
createEmailExtractor,
|
|
37410
37523
|
createEPUBExtractor,
|
|
37524
|
+
createCohereRAGReranker,
|
|
37411
37525
|
createBuiltinArchiveExpander,
|
|
37412
37526
|
compareRAGRetrievalTraceSummaries,
|
|
37413
37527
|
compareRAGRetrievalStrategies,
|
|
@@ -37491,5 +37605,5 @@ export {
|
|
|
37491
37605
|
addRAGEvaluationSuiteCase
|
|
37492
37606
|
};
|
|
37493
37607
|
|
|
37494
|
-
//# debugId=
|
|
37608
|
+
//# debugId=20E465BEF0B96FD164756E2164756E21
|
|
37495
37609
|
//# sourceMappingURL=index.js.map
|