@arnilo/prism-rag 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/dist/chunk.js +32 -3
- package/dist/fusion.d.ts +18 -0
- package/dist/fusion.js +0 -0
- package/dist/hash.d.ts +6 -0
- package/dist/hash.js +9 -0
- package/dist/index.d.ts +8 -2
- package/dist/index.js +4 -1
- package/dist/indexing.js +33 -3
- package/dist/limits.d.ts +4 -0
- package/dist/limits.js +4 -0
- package/dist/parsers.js +156 -16
- package/dist/retrieve.js +262 -92
- package/dist/sources.d.ts +2 -0
- package/dist/sources.js +65 -3
- package/dist/tei-reranker.d.ts +35 -0
- package/dist/tei-reranker.js +138 -0
- package/dist/telemetry.d.ts +16 -0
- package/dist/telemetry.js +2 -0
- package/dist/types.d.ts +35 -2
- package/package.json +2 -2
package/dist/retrieve.js
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { resolveRedactor } from "@arnilo/prism";
|
|
2
|
-
import { RagScopeError, RagValidationError } from "./errors.js";
|
|
3
|
-
import {
|
|
2
|
+
import { RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
|
|
3
|
+
import { fuseReciprocalRankLists } from "./fusion.js";
|
|
4
|
+
import { HARD_CHUNK_SIZE_CAP, HARD_RETRIEVE_SCOPE_CAP, resolveRagLimits } from "./limits.js";
|
|
4
5
|
import { rerankHits } from "./rerank.js";
|
|
5
|
-
import { assertBytes, assertNotAborted,
|
|
6
|
+
import { assertBytes, assertNotAborted, byteLength, isJsonObject, matchesFilter, nonEmpty, requireScope, requireSourceId, truncateUtf8, } from "./util.js";
|
|
6
7
|
const RETRIEVED_CONTENT_TRUST = Object.freeze({ untrusted: true, inert: true, injectionCapable: true });
|
|
7
8
|
export async function retrieveContext(query, options) {
|
|
8
9
|
nonEmpty(query, "query");
|
|
9
10
|
if (query.length > HARD_CHUNK_SIZE_CAP)
|
|
10
11
|
throw new RagValidationError(`query exceeds ${HARD_CHUNK_SIZE_CAP} characters`);
|
|
11
|
-
const
|
|
12
|
+
const scopes = resolveRetrieveScopes(options);
|
|
12
13
|
const limits = resolveRagLimits({
|
|
13
14
|
topK: options.topK,
|
|
14
15
|
queryCandidates: options.queryCandidates,
|
|
@@ -19,6 +20,7 @@ export async function retrieveContext(query, options) {
|
|
|
19
20
|
maxRerankBytes: options.maxRerankBytes,
|
|
20
21
|
maxRerankMs: options.maxRerankMs,
|
|
21
22
|
rerankConcurrency: options.rerankConcurrency,
|
|
23
|
+
rrfK: options.rrfK,
|
|
22
24
|
});
|
|
23
25
|
if (!Number.isInteger(options.embedder.dimensions) ||
|
|
24
26
|
options.embedder.dimensions <= 0 ||
|
|
@@ -27,100 +29,220 @@ export async function retrieveContext(query, options) {
|
|
|
27
29
|
}
|
|
28
30
|
if (options.filter)
|
|
29
31
|
assertBytes(options.filter, limits.maxMetadataBytes, "metadata filter");
|
|
30
|
-
const
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
const vectors = await options.embedder.embed([safeQuery], { signal: options.signal });
|
|
34
|
-
const embedding = vectors[0];
|
|
35
|
-
if (vectors.length !== 1 ||
|
|
36
|
-
!embedding ||
|
|
37
|
-
embedding.length !== options.embedder.dimensions ||
|
|
38
|
-
embedding.some((value) => !Number.isFinite(value))) {
|
|
39
|
-
throw new RagValidationError("embedder returned invalid query vector");
|
|
32
|
+
const lexical = options.lexical ?? (options.store.lexicalQuery ? "fts" : "off");
|
|
33
|
+
if (lexical !== "fts" && lexical !== "bm25" && lexical !== "off") {
|
|
34
|
+
throw new RagValidationError('lexical must be "fts", "bm25", or "off"');
|
|
40
35
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
assertNotAborted(options.signal);
|
|
50
|
-
const retrievedAt = new Date().toISOString();
|
|
51
|
-
const retrieved = [];
|
|
52
|
-
for (const candidate of candidates.slice(0, limits.queryCandidates)) {
|
|
53
|
-
assertScope(scope, candidate);
|
|
54
|
-
const parsed = parseHit(candidate, retrieved.length, retrievedAt);
|
|
55
|
-
if (!matchesFilter(parsed.metadata, options.filter))
|
|
56
|
-
continue;
|
|
57
|
-
retrieved.push(Object.freeze(redactor?.redact(parsed) ?? parsed));
|
|
36
|
+
if (options.fusion !== undefined && options.fusion !== "rrf") {
|
|
37
|
+
throw new RagValidationError('fusion must be "rrf"');
|
|
38
|
+
}
|
|
39
|
+
if (lexical !== "off" && !options.store.lexicalQuery) {
|
|
40
|
+
throw new RagValidationError(`lexical "${lexical}" requested but the store has no lexicalQuery capability`);
|
|
41
|
+
}
|
|
42
|
+
if (lexical === "bm25" && !options.store.lexicalModes?.includes("bm25")) {
|
|
43
|
+
throw new RagValidationError('lexical "bm25" requested but the store does not declare BM25 support');
|
|
58
44
|
}
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
45
|
+
const useLexical = lexical !== "off";
|
|
46
|
+
const redactor = resolveRedactor(options.redactor, options.secrets);
|
|
47
|
+
const telemetry = options.telemetry;
|
|
48
|
+
const root = telemetry?.startSpan("rag_request", {
|
|
49
|
+
...(scopes[0] ? { "rag.scope.tenant_id": scopes[0].tenantId } : {}),
|
|
50
|
+
"rag.scope_count": scopes.length,
|
|
51
|
+
"rag.embedder_id": options.embedder.id,
|
|
52
|
+
"rag.top_k": limits.topK,
|
|
53
|
+
"rag.lexical_mode": lexical,
|
|
54
|
+
});
|
|
55
|
+
try {
|
|
56
|
+
const safeQuery = redactor?.redact(query) ?? query;
|
|
57
|
+
assertNotAborted(options.signal);
|
|
58
|
+
if (scopes.length === 0) {
|
|
59
|
+
return emptyResult(safeQuery);
|
|
60
|
+
}
|
|
61
|
+
if (scopes.length === 1) {
|
|
62
|
+
const currentGeneration = await options.store.getCurrentGeneration?.({
|
|
63
|
+
tenantId: scopes[0].tenantId,
|
|
64
|
+
resourceId: scopes[0].resourceId,
|
|
65
|
+
threadId: scopes[0].corpusId,
|
|
66
|
+
});
|
|
67
|
+
if (currentGeneration !== undefined)
|
|
68
|
+
root?.setAttribute("rag.index_generation", Number(currentGeneration));
|
|
69
|
+
}
|
|
70
|
+
const vectors = await span(telemetry, "embedding.query", undefined, root, () => options.embedder.embed([safeQuery], { signal: options.signal }));
|
|
71
|
+
const embedding = vectors[0];
|
|
72
|
+
if (vectors.length !== 1 ||
|
|
73
|
+
!embedding ||
|
|
74
|
+
embedding.length !== options.embedder.dimensions ||
|
|
75
|
+
embedding.some((value) => !Number.isFinite(value))) {
|
|
76
|
+
throw new RagValidationError("embedder returned invalid query vector");
|
|
87
77
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
78
|
+
const vectorLists = [];
|
|
79
|
+
await span(telemetry, "retrieval.vector_search", undefined, root, async (leg) => {
|
|
80
|
+
let total = 0;
|
|
81
|
+
for (const scope of scopes) {
|
|
82
|
+
assertNotAborted(options.signal);
|
|
83
|
+
const found = await options.store.query({
|
|
84
|
+
tenantId: scope.tenantId,
|
|
85
|
+
resourceId: scope.resourceId,
|
|
86
|
+
threadId: scope.corpusId,
|
|
87
|
+
embedding,
|
|
88
|
+
topK: limits.queryCandidates,
|
|
89
|
+
signal: options.signal,
|
|
90
|
+
});
|
|
91
|
+
const sliced = found.slice(0, limits.queryCandidates);
|
|
92
|
+
vectorLists.push(sliced);
|
|
93
|
+
total += sliced.length;
|
|
94
|
+
}
|
|
95
|
+
leg?.setAttribute("rag.vector_candidates", total);
|
|
96
|
+
});
|
|
97
|
+
const lexicalLists = [];
|
|
98
|
+
if (useLexical && options.store.lexicalQuery) {
|
|
99
|
+
await span(telemetry, "retrieval.lexical", undefined, root, async (leg) => {
|
|
100
|
+
let total = 0;
|
|
101
|
+
for (const scope of scopes) {
|
|
102
|
+
assertNotAborted(options.signal);
|
|
103
|
+
const found = await options.store.lexicalQuery({
|
|
104
|
+
tenantId: scope.tenantId,
|
|
105
|
+
resourceId: scope.resourceId,
|
|
106
|
+
threadId: scope.corpusId,
|
|
107
|
+
text: safeQuery,
|
|
108
|
+
topK: limits.queryCandidates,
|
|
109
|
+
signal: options.signal,
|
|
110
|
+
});
|
|
111
|
+
const sliced = found.slice(0, limits.queryCandidates);
|
|
112
|
+
lexicalLists.push(sliced);
|
|
113
|
+
total += sliced.length;
|
|
114
|
+
}
|
|
115
|
+
leg?.setAttribute("rag.lexical_candidates", total);
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
const retrievedAt = new Date().toISOString();
|
|
119
|
+
const retrieved = [];
|
|
120
|
+
const fused = await span(telemetry, "retrieval.fusion", undefined, root, (fusion) => {
|
|
121
|
+
const lists = [
|
|
122
|
+
...vectorLists.map((hits) => ({ hits, leg: "vector" })),
|
|
123
|
+
...lexicalLists.map((hits) => ({ hits, leg: "lexical" })),
|
|
124
|
+
];
|
|
125
|
+
const fusedCandidates = fuseReciprocalRankLists(lists, limits.rrfK);
|
|
126
|
+
fusion?.setAttribute("rag.fused_candidates", fusedCandidates.length);
|
|
127
|
+
return fusedCandidates;
|
|
128
|
+
});
|
|
129
|
+
for (const { hit: candidate, retrieval } of fused) {
|
|
130
|
+
assertRequestedScope(scopes, candidate);
|
|
131
|
+
if (candidate.embedderId === undefined) {
|
|
132
|
+
throw new RagError(`stored record ${candidate.id} has no embedderId; re-index the source to stamp embedder identity`, "ERR_PRISM_RAG_EMBEDDER_MISMATCH");
|
|
133
|
+
}
|
|
134
|
+
if (candidate.embedderId !== options.embedder.id || candidate.embedding.length !== options.embedder.dimensions) {
|
|
135
|
+
throw new RagError(`embedder mismatch: record ${candidate.id} was embedded by "${candidate.embedderId}" (${candidate.embedding.length} dims) but the query embedder is "${options.embedder.id}" (${options.embedder.dimensions} dims)`, "ERR_PRISM_RAG_EMBEDDER_MISMATCH");
|
|
136
|
+
}
|
|
137
|
+
const parsed = parseHit(candidate, retrieved.length, retrievedAt, retrieval);
|
|
138
|
+
if (!matchesFilter(parsed.metadata, options.filter))
|
|
139
|
+
continue;
|
|
140
|
+
retrieved.push(Object.freeze(redactor?.redact(parsed) ?? parsed));
|
|
141
|
+
}
|
|
142
|
+
const reranker = options.reranker;
|
|
143
|
+
const ranked = reranker
|
|
144
|
+
? await span(telemetry, "retrieval.rerank", undefined, root, () => rerankHits(safeQuery, retrieved, {
|
|
145
|
+
reranker,
|
|
146
|
+
maxBytes: limits.maxRerankBytes,
|
|
147
|
+
maxMs: limits.maxRerankMs,
|
|
148
|
+
concurrency: limits.rerankConcurrency,
|
|
149
|
+
signal: options.signal,
|
|
150
|
+
redactor: options.redactor,
|
|
151
|
+
secrets: options.secrets,
|
|
152
|
+
}))
|
|
153
|
+
: retrieved;
|
|
154
|
+
const hits = [];
|
|
155
|
+
const citations = [];
|
|
156
|
+
const rendered = [];
|
|
157
|
+
const maxChars = limits.maxContextTokens * 4;
|
|
158
|
+
let usedBytes = 0;
|
|
159
|
+
let usedChars = 0;
|
|
160
|
+
let truncated = false;
|
|
161
|
+
const assemblySpan = telemetry?.startSpan("prompt.assembly", undefined, root);
|
|
162
|
+
for (const hit of ranked) {
|
|
163
|
+
if (hits.length >= limits.topK)
|
|
164
|
+
break;
|
|
165
|
+
const prefix = `[${hit.citationId}] `;
|
|
166
|
+
const separator = rendered.length ? "\n\n" : "";
|
|
167
|
+
const availableBytes = limits.maxResultBytes - usedBytes - byteLength(separator + prefix);
|
|
168
|
+
const availableChars = maxChars - usedChars - separator.length - prefix.length;
|
|
169
|
+
if (availableBytes <= 0 || availableChars <= 0) {
|
|
170
|
+
truncated = true;
|
|
171
|
+
break;
|
|
172
|
+
}
|
|
173
|
+
let text = hit.text.slice(0, availableChars);
|
|
174
|
+
text = truncateUtf8(text, availableBytes);
|
|
175
|
+
if (!text) {
|
|
176
|
+
truncated = true;
|
|
177
|
+
break;
|
|
178
|
+
}
|
|
179
|
+
if (text.length < hit.text.length)
|
|
180
|
+
truncated = true;
|
|
181
|
+
const renderedHit = Object.freeze({ ...hit, text });
|
|
182
|
+
const citation = Object.freeze({
|
|
183
|
+
id: renderedHit.citationId,
|
|
184
|
+
sourceId: renderedHit.sourceId,
|
|
185
|
+
chunkId: renderedHit.id,
|
|
186
|
+
provenance: renderedHit.provenance,
|
|
187
|
+
trust: renderedHit.trust,
|
|
188
|
+
...(renderedHit.metadata ? { metadata: renderedHit.metadata } : {}),
|
|
189
|
+
});
|
|
190
|
+
const block = `${separator}${prefix}${text}`;
|
|
191
|
+
rendered.push(block);
|
|
192
|
+
usedBytes += byteLength(block);
|
|
193
|
+
usedChars += block.length;
|
|
194
|
+
hits.push(renderedHit);
|
|
195
|
+
citations.push(citation);
|
|
196
|
+
if (truncated)
|
|
197
|
+
break;
|
|
198
|
+
}
|
|
199
|
+
assemblySpan?.setAttribute("rag.result_count", hits.length);
|
|
200
|
+
assemblySpan?.end();
|
|
201
|
+
for (const hit of hits) {
|
|
202
|
+
root?.addEvent("chunk_retrieved", {
|
|
203
|
+
"rag.chunk.source_id": hit.sourceId,
|
|
204
|
+
"rag.chunk.id": hit.id,
|
|
205
|
+
"rag.chunk.rank": hit.retrievalRank,
|
|
206
|
+
"rag.chunk.score": hit.score,
|
|
207
|
+
"rag.chunk.embedder_id": options.embedder.id,
|
|
208
|
+
"rag.chunk.tenant_id": hit.provenance.tenantId,
|
|
209
|
+
"rag.chunk.corpus_id": hit.provenance.corpusId,
|
|
210
|
+
});
|
|
93
211
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
provenance: renderedHit.provenance,
|
|
102
|
-
trust: renderedHit.trust,
|
|
103
|
-
...(renderedHit.metadata ? { metadata: renderedHit.metadata } : {}),
|
|
212
|
+
return Object.freeze({
|
|
213
|
+
query: safeQuery,
|
|
214
|
+
trust: RETRIEVED_CONTENT_TRUST,
|
|
215
|
+
text: rendered.join(""),
|
|
216
|
+
hits: Object.freeze(hits),
|
|
217
|
+
citations: Object.freeze(citations),
|
|
218
|
+
truncated,
|
|
104
219
|
});
|
|
105
|
-
const block = `${separator}${prefix}${text}`;
|
|
106
|
-
rendered.push(block);
|
|
107
|
-
usedBytes += byteLength(block);
|
|
108
|
-
usedChars += block.length;
|
|
109
|
-
hits.push(renderedHit);
|
|
110
|
-
citations.push(citation);
|
|
111
|
-
if (truncated)
|
|
112
|
-
break;
|
|
113
220
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
221
|
+
catch (error) {
|
|
222
|
+
root?.recordError();
|
|
223
|
+
throw error;
|
|
224
|
+
}
|
|
225
|
+
finally {
|
|
226
|
+
root?.end();
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
/** Opens a child span only when telemetry is present; otherwise runs the section untouched. */
|
|
230
|
+
async function span(telemetry, name, attributes, parent, fn) {
|
|
231
|
+
if (!telemetry)
|
|
232
|
+
return await fn(undefined);
|
|
233
|
+
const child = telemetry.startSpan(name, attributes, parent);
|
|
234
|
+
try {
|
|
235
|
+
return await fn(child);
|
|
236
|
+
}
|
|
237
|
+
catch (error) {
|
|
238
|
+
child.recordError();
|
|
239
|
+
throw error;
|
|
240
|
+
}
|
|
241
|
+
finally {
|
|
242
|
+
child.end();
|
|
243
|
+
}
|
|
122
244
|
}
|
|
123
|
-
function parseHit(hit, retrievalRank, retrievedAt) {
|
|
245
|
+
function parseHit(hit, retrievalRank, retrievedAt, retrieval) {
|
|
124
246
|
const metadata = hit.metadata;
|
|
125
247
|
const rag = metadata?._rag;
|
|
126
248
|
if (!isJsonObject(rag))
|
|
@@ -144,7 +266,17 @@ function parseHit(hit, retrievalRank, retrievedAt) {
|
|
|
144
266
|
userMetadata[key] = value;
|
|
145
267
|
const web = isJsonObject(userMetadata.web) ? userMetadata.web : undefined;
|
|
146
268
|
const provider = typeof web?.provider === "string" && web.provider.trim() ? web.provider : "host";
|
|
147
|
-
const provenance = Object.freeze({
|
|
269
|
+
const provenance = Object.freeze({
|
|
270
|
+
sourceId,
|
|
271
|
+
chunkId: hit.id,
|
|
272
|
+
citationId,
|
|
273
|
+
provider,
|
|
274
|
+
tenantId: hit.tenantId,
|
|
275
|
+
resourceId: hit.resourceId,
|
|
276
|
+
corpusId: hit.threadId,
|
|
277
|
+
retrieval,
|
|
278
|
+
retrievedAt,
|
|
279
|
+
});
|
|
148
280
|
return {
|
|
149
281
|
id: hit.id,
|
|
150
282
|
citationId,
|
|
@@ -160,4 +292,42 @@ function parseHit(hit, retrievalRank, retrievedAt) {
|
|
|
160
292
|
...(Object.keys(userMetadata).length ? { metadata: userMetadata } : {}),
|
|
161
293
|
};
|
|
162
294
|
}
|
|
295
|
+
function resolveRetrieveScopes(options) {
|
|
296
|
+
const hasScope = options.scope !== undefined;
|
|
297
|
+
const hasScopes = options.scopes !== undefined;
|
|
298
|
+
if (hasScope && hasScopes)
|
|
299
|
+
throw new RagValidationError("provide either scope or scopes, not both");
|
|
300
|
+
if (!hasScope && !hasScopes)
|
|
301
|
+
throw new RagValidationError("scope or scopes is required");
|
|
302
|
+
const raw = hasScopes ? options.scopes : [options.scope];
|
|
303
|
+
if (raw.length > HARD_RETRIEVE_SCOPE_CAP)
|
|
304
|
+
throw new RagLimitError(`scopes exceeds hard cap ${HARD_RETRIEVE_SCOPE_CAP}`);
|
|
305
|
+
const seen = new Set();
|
|
306
|
+
const resolved = [];
|
|
307
|
+
for (const item of raw) {
|
|
308
|
+
const scope = requireScope(item);
|
|
309
|
+
const key = `${scope.tenantId}${scope.resourceId}${scope.corpusId}`;
|
|
310
|
+
if (seen.has(key))
|
|
311
|
+
continue;
|
|
312
|
+
seen.add(key);
|
|
313
|
+
resolved.push(scope);
|
|
314
|
+
}
|
|
315
|
+
return resolved;
|
|
316
|
+
}
|
|
317
|
+
function assertRequestedScope(scopes, actual) {
|
|
318
|
+
if (scopes.some((scope) => scope.tenantId === actual.tenantId && scope.resourceId === actual.resourceId && scope.corpusId === actual.threadId)) {
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
throw new RagScopeError("vector hit crossed tenant/resource/corpus boundary");
|
|
322
|
+
}
|
|
323
|
+
function emptyResult(query) {
|
|
324
|
+
return Object.freeze({
|
|
325
|
+
query,
|
|
326
|
+
trust: RETRIEVED_CONTENT_TRUST,
|
|
327
|
+
text: "",
|
|
328
|
+
hits: Object.freeze([]),
|
|
329
|
+
citations: Object.freeze([]),
|
|
330
|
+
truncated: false,
|
|
331
|
+
});
|
|
332
|
+
}
|
|
163
333
|
//# sourceMappingURL=retrieve.js.map
|
package/dist/sources.d.ts
CHANGED
|
@@ -3,6 +3,8 @@ export interface SourceMutationResult {
|
|
|
3
3
|
readonly sourceId: string;
|
|
4
4
|
readonly deleted: number;
|
|
5
5
|
readonly indexed: number;
|
|
6
|
+
/** Set when an unchanged document hash short-circuited the replace. */
|
|
7
|
+
readonly skipped?: true;
|
|
6
8
|
}
|
|
7
9
|
export declare function replaceSource(options: ReplaceSourceOptions): Promise<SourceMutationResult>;
|
|
8
10
|
export declare function deleteSource(options: DeleteSourceOptions): Promise<SourceMutationResult>;
|
package/dist/sources.js
CHANGED
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
import { resolveRedactor } from "@arnilo/prism";
|
|
2
2
|
import { chunkText } from "./chunk.js";
|
|
3
3
|
import { RagScopeError, RagValidationError } from "./errors.js";
|
|
4
|
+
import { isValidContentHash } from "./hash.js";
|
|
4
5
|
import { indexChunkBatches } from "./indexing.js";
|
|
5
6
|
import { ingestionStatus } from "./ingestion-status.js";
|
|
6
7
|
import { assertNotAborted, byteLength, requireScope, requireSourceId } from "./util.js";
|
|
8
|
+
function storedDocHash(record) {
|
|
9
|
+
const value = record.metadata?._rag?.contentHash;
|
|
10
|
+
return isValidContentHash(value) ? value : undefined;
|
|
11
|
+
}
|
|
7
12
|
export async function replaceSource(options) {
|
|
8
13
|
const sourceId = requireSourceId(options.sourceId);
|
|
9
14
|
const scope = requireScope(options.scope);
|
|
@@ -12,6 +17,10 @@ export async function replaceSource(options) {
|
|
|
12
17
|
throw new RagValidationError("replaceSource chunks must all belong to sourceId");
|
|
13
18
|
}
|
|
14
19
|
const redactor = resolveRedactor(options.redactor, options.secrets);
|
|
20
|
+
if (options.contentHash !== undefined && !isValidContentHash(options.contentHash)) {
|
|
21
|
+
throw new RagValidationError("contentHash must be a hex digest of 32..128 characters");
|
|
22
|
+
}
|
|
23
|
+
const contentHash = options.contentHash?.toLowerCase();
|
|
15
24
|
const totalBytes = options.chunks.reduce((total, chunk) => total + byteLength(redactor?.redact(chunk.text) ?? chunk.text), 0);
|
|
16
25
|
const setStatus = async (state, error) => {
|
|
17
26
|
if (!options.statusStore)
|
|
@@ -20,29 +29,82 @@ export async function replaceSource(options) {
|
|
|
20
29
|
await options.statusStore.set(ingestionStatus(scope, sourceId, state, state === "indexed" ? totalBytes : 0, state === "indexed" ? options.chunks.length : 0, message ? (redactor?.redact(message) ?? message) : undefined));
|
|
21
30
|
};
|
|
22
31
|
await setStatus("pending");
|
|
32
|
+
const telemetry = options.telemetry;
|
|
33
|
+
const root = telemetry?.startSpan("rag_index", {
|
|
34
|
+
"rag.scope.tenant_id": scope.tenantId,
|
|
35
|
+
"rag.source_id": sourceId,
|
|
36
|
+
"rag.embedder_id": options.embedder.id,
|
|
37
|
+
"rag.chunk_count": options.chunks.length,
|
|
38
|
+
});
|
|
23
39
|
try {
|
|
40
|
+
// One read decides the skip; unchanged sources cost zero embeds and zero writes.
|
|
41
|
+
const previous = await sourceRecords(options.store, sourceId, scope, options.signal);
|
|
42
|
+
if (contentHash &&
|
|
43
|
+
options.skipIfUnchanged !== false &&
|
|
44
|
+
previous.length > 0 &&
|
|
45
|
+
previous.every((record) => storedDocHash(record) === contentHash)) {
|
|
46
|
+
// Incoming stats describe the now-live content even though nothing was rewritten.
|
|
47
|
+
await setStatus("indexed", undefined);
|
|
48
|
+
return Object.freeze({ sourceId, deleted: 0, indexed: 0, skipped: true });
|
|
49
|
+
}
|
|
50
|
+
const reuseEmbeddings = new Map();
|
|
51
|
+
if (options.skipIfUnchanged !== false) {
|
|
52
|
+
// skipIfUnchanged: false means rebuild everything — no embedding reuse either.
|
|
53
|
+
for (const record of previous) {
|
|
54
|
+
if (record.embedderId === options.embedder.id) {
|
|
55
|
+
reuseEmbeddings.set(record.id, { text: record.text, embedding: record.embedding });
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
24
59
|
const staged = [];
|
|
25
|
-
const indexed = await indexChunkBatches({ ...options, statusStore: undefined }, async (records) => {
|
|
60
|
+
const indexed = await indexChunkBatches({ ...options, statusStore: undefined, contentHash, reuseEmbeddings, telemetry, telemetryParent: root }, async (records) => {
|
|
26
61
|
staged.push(...records);
|
|
27
62
|
});
|
|
28
63
|
assertNotAborted(options.signal);
|
|
29
64
|
const result = await options.store.transaction(async (store) => {
|
|
65
|
+
// Generation visibility: stamp chunks at N+1 and advance the scope pointer in the
|
|
66
|
+
// same transaction as the swap. Stores without generation tracking keep legacy behavior.
|
|
67
|
+
const getCurrent = store.getCurrentGeneration?.bind(store);
|
|
68
|
+
const setCurrent = store.setCurrentGeneration?.bind(store);
|
|
69
|
+
let nextGeneration;
|
|
70
|
+
if (getCurrent && setCurrent) {
|
|
71
|
+
const current = await getCurrent({
|
|
72
|
+
tenantId: scope.tenantId,
|
|
73
|
+
resourceId: scope.resourceId,
|
|
74
|
+
threadId: scope.corpusId,
|
|
75
|
+
});
|
|
76
|
+
nextGeneration = (current === undefined ? 0 : Number(current)) + 1;
|
|
77
|
+
root?.setAttribute("rag.index_generation", nextGeneration);
|
|
78
|
+
}
|
|
30
79
|
const previous = await sourceRecords(store, sourceId, scope, options.signal);
|
|
31
80
|
assertNotAborted(options.signal);
|
|
32
81
|
if (previous.length) {
|
|
33
82
|
await store.delete({ tenantId: scope.tenantId, resourceId: scope.resourceId, threadId: scope.corpusId, ids: previous.map((record) => record.id) }, { signal: options.signal });
|
|
34
83
|
}
|
|
35
|
-
if (staged.length)
|
|
36
|
-
|
|
84
|
+
if (staged.length) {
|
|
85
|
+
const stamped = nextGeneration === undefined ? staged : staged.map((record) => ({ ...record, generation: nextGeneration }));
|
|
86
|
+
await store.upsert(stamped, { signal: options.signal });
|
|
87
|
+
}
|
|
88
|
+
if (setCurrent && nextGeneration !== undefined) {
|
|
89
|
+
await setCurrent({
|
|
90
|
+
tenantId: scope.tenantId,
|
|
91
|
+
resourceId: scope.resourceId,
|
|
92
|
+
threadId: scope.corpusId,
|
|
93
|
+
}, nextGeneration);
|
|
94
|
+
}
|
|
37
95
|
return Object.freeze({ sourceId, deleted: previous.length, indexed: indexed.indexed });
|
|
38
96
|
}, { signal: options.signal });
|
|
39
97
|
await setStatus("indexed");
|
|
40
98
|
return result;
|
|
41
99
|
}
|
|
42
100
|
catch (error) {
|
|
101
|
+
root?.recordError();
|
|
43
102
|
await setStatus("failed", error);
|
|
44
103
|
throw error;
|
|
45
104
|
}
|
|
105
|
+
finally {
|
|
106
|
+
root?.end();
|
|
107
|
+
}
|
|
46
108
|
}
|
|
47
109
|
export async function deleteSource(options) {
|
|
48
110
|
const sourceId = requireSourceId(options.sourceId);
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { type SsrfPolicy } from "@arnilo/prism";
|
|
2
|
+
import type { Reranker } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* Hugging Face TEI rerank adapter (plan 034 Task 8 / request P8):
|
|
5
|
+
* `POST <baseUrl>/rerank` with `{query, texts, raw_scores:false}` →
|
|
6
|
+
* `{results: [{index, score}]}` mapped to a permutation-only reorder of the
|
|
7
|
+
* provided `RagHit[]` (same object references — provenance/trust untouched).
|
|
8
|
+
*
|
|
9
|
+
* - URL shape validated at construction (absolute, http/https, no embedded
|
|
10
|
+
* credentials or fragment); SSRF/enforcement is host-side via `ssrf`,
|
|
11
|
+
* `allowLoopback`, or an injected `fetch`. Default transport is the core
|
|
12
|
+
* `pinnedFetch` primitive (DNS-pinned, redirect-free, byte-bounded).
|
|
13
|
+
* - Out-of-range/duplicate/missing indices, non-finite scores, HTTP errors,
|
|
14
|
+
* timeouts, and oversized bodies all fail closed in the rerank error
|
|
15
|
+
* family. Seam caps (`maxRerankBytes`, `maxRerankMs`, `rerankConcurrency`)
|
|
16
|
+
* stay enforced by `rerankHits` around this adapter.
|
|
17
|
+
* - No credentials, no SaaS default URL.
|
|
18
|
+
*/
|
|
19
|
+
export interface CreateTeiRerankerOptions {
|
|
20
|
+
/** Base URL of the TEI service, e.g. `http://tei.svc:8080`. `/rerank` is appended. */
|
|
21
|
+
readonly baseUrl: string;
|
|
22
|
+
/** Optional model name sent in the rerank body. */
|
|
23
|
+
readonly model?: string;
|
|
24
|
+
/** Per-call timeout combined with the caller signal; aborts fail closed. */
|
|
25
|
+
readonly timeoutMs?: number;
|
|
26
|
+
/** SSRF policy applied on resolved hosts (default: core default). */
|
|
27
|
+
readonly ssrf?: SsrfPolicy;
|
|
28
|
+
/** Allow loopback destinations (local/dev TEI). Default `false`. */
|
|
29
|
+
readonly allowLoopback?: boolean;
|
|
30
|
+
/** Maximum response body bytes. Default 65,536 (plan 021 ceiling precedent). */
|
|
31
|
+
readonly maxResponseBytes?: number;
|
|
32
|
+
/** Trusted custom transport; host owns DNS/Bonding protection (OPA precedent). */
|
|
33
|
+
readonly fetch?: typeof globalThis.fetch;
|
|
34
|
+
}
|
|
35
|
+
export declare function createTeiReranker(options: CreateTeiRerankerOptions): Reranker;
|