@arnilo/prism-rag 0.0.15 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/dist/chunk.d.ts +1 -1
- package/dist/chunk.js +1 -1
- package/dist/context.js +5 -3
- package/dist/index.d.ts +9 -9
- package/dist/index.js +6 -6
- package/dist/indexing.js +18 -7
- package/dist/ingestion-status.js +3 -1
- package/dist/parsers.js +3 -1
- package/dist/rerank.d.ts +1 -1
- package/dist/rerank.js +5 -2
- package/dist/retrieve.js +17 -5
- package/dist/sources.js +10 -5
- package/dist/util.d.ts +1 -2
- package/dist/util.js +1 -7
- package/package.json +3 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.0.17] - 2026-07-29
|
|
4
|
+
|
|
5
|
+
### Changed
|
|
6
|
+
- Released with exact 0.0.17 graph.
|
|
7
|
+
|
|
8
|
+
## [0.0.16] - 2026-07-26
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
- Sourced `resolveRedactor` from `@arnilo/prism` core and deleted the private duplicate; no public API change.
|
|
12
|
+
|
|
3
13
|
## [0.0.15] - 2026-07-26
|
|
4
14
|
|
|
5
15
|
### Added
|
package/dist/chunk.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ChunkOptions, RagChunk } from "./types.js";
|
|
2
2
|
export declare function chunkText(text: string, options: ChunkOptions): readonly RagChunk[];
|
|
3
3
|
export declare function chunkMarkdown(markdown: string, options: ChunkOptions): readonly RagChunk[];
|
package/dist/chunk.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { resolveRagLimits } from "./limits.js";
|
|
2
1
|
import { RagLimitError } from "./errors.js";
|
|
2
|
+
import { resolveRagLimits } from "./limits.js";
|
|
3
3
|
import { assertBytes, requireSourceId } from "./util.js";
|
|
4
4
|
export function chunkText(text, options) {
|
|
5
5
|
return chunkDocument(text, options, false);
|
package/dist/context.js
CHANGED
|
@@ -7,18 +7,20 @@ export function createRagContextProvider(options) {
|
|
|
7
7
|
context.signal?.throwIfAborted();
|
|
8
8
|
const query = typeof options.query === "function"
|
|
9
9
|
? options.query({ messages: context.messages })
|
|
10
|
-
: options.query ?? latestUserText(context.messages);
|
|
10
|
+
: (options.query ?? latestUserText(context.messages));
|
|
11
11
|
if (!query?.trim())
|
|
12
12
|
return [];
|
|
13
13
|
const result = await retrieveContext(query, { ...options, signal: context.signal });
|
|
14
14
|
if (!result.text)
|
|
15
15
|
return [];
|
|
16
|
-
return [
|
|
16
|
+
return [
|
|
17
|
+
{
|
|
17
18
|
id: `${options.name ?? "rag"}:context`,
|
|
18
19
|
title: options.title ?? "Retrieved context",
|
|
19
20
|
content: result.text,
|
|
20
21
|
metadata: { citations: result.citations, trust: result.trust, inert: true, untrusted: true, injectionCapable: true },
|
|
21
|
-
}
|
|
22
|
+
},
|
|
23
|
+
];
|
|
22
24
|
},
|
|
23
25
|
};
|
|
24
26
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
export { DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, DEFAULT_MAX_DOCUMENT_BYTES, HARD_MAX_DOCUMENT_BYTES_CAP, DEFAULT_MAX_PARSE_MS, HARD_MAX_PARSE_MS_CAP, DEFAULT_MAX_PDF_PAGES, HARD_MAX_PDF_PAGES_CAP, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, DEFAULT_TOP_K, HARD_TOP_K_CAP, DEFAULT_QUERY_CANDIDATES, HARD_QUERY_CANDIDATES_CAP, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_MAX_RERANK_BYTES, HARD_MAX_RERANK_BYTES_CAP, DEFAULT_MAX_RERANK_MS, HARD_MAX_RERANK_MS_CAP, DEFAULT_RERANK_CONCURRENCY, HARD_RERANK_CONCURRENCY_CAP, DEFAULT_INGESTION_STATUS_PAGE_SIZE, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, resolveRagLimits, } from "./limits.js";
|
|
2
|
-
export type { RagLimits, RagLimitsInput } from "./limits.js";
|
|
3
|
-
export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
|
|
4
1
|
export { chunkMarkdown, chunkText } from "./chunk.js";
|
|
5
|
-
export {
|
|
6
|
-
export {
|
|
2
|
+
export { createRagContextProvider } from "./context.js";
|
|
3
|
+
export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
|
|
7
4
|
export { indexChunks } from "./indexing.js";
|
|
8
|
-
export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
|
|
9
|
-
export type { SourceMutationResult } from "./sources.js";
|
|
10
5
|
export { createMemoryIngestionStatusStore, listIngestionStatus } from "./ingestion-status.js";
|
|
6
|
+
export type { RagLimits, RagLimitsInput } from "./limits.js";
|
|
7
|
+
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
|
|
8
|
+
export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
|
|
9
|
+
export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
|
|
11
10
|
export { retrieveContext } from "./retrieve.js";
|
|
12
|
-
export {
|
|
13
|
-
export
|
|
11
|
+
export type { SourceMutationResult } from "./sources.js";
|
|
12
|
+
export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
|
|
13
|
+
export type { Chunker, ChunkOptions, DeleteSourceOptions, DocumentLoader, DocumentLoadOptions, DocumentParseOptions, IndexChunksOptions, IndexChunksResult, IngestionState, IngestionStatus, IngestionStatusQuery, IngestionStatusStore, LoadedDocument, ParsedDocument, Parser, RagChunk, RagCitation, RagContentTrust, RagContextProvider, RagContextProviderOptions, RagContextResult, RagHit, RagProvenance, RagScope, ReplaceDocumentOptions, ReplaceSourceOptions, Reranker, RetrieveContextOptions, SourceVectorStore, TransactionalVectorStore, } from "./types.js";
|
|
14
14
|
export declare const packageName = "@arnilo/prism-rag";
|
package/dist/index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
export { DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, DEFAULT_MAX_DOCUMENT_BYTES, HARD_MAX_DOCUMENT_BYTES_CAP, DEFAULT_MAX_PARSE_MS, HARD_MAX_PARSE_MS_CAP, DEFAULT_MAX_PDF_PAGES, HARD_MAX_PDF_PAGES_CAP, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, DEFAULT_TOP_K, HARD_TOP_K_CAP, DEFAULT_QUERY_CANDIDATES, HARD_QUERY_CANDIDATES_CAP, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_MAX_RERANK_BYTES, HARD_MAX_RERANK_BYTES_CAP, DEFAULT_MAX_RERANK_MS, HARD_MAX_RERANK_MS_CAP, DEFAULT_RERANK_CONCURRENCY, HARD_RERANK_CONCURRENCY_CAP, DEFAULT_INGESTION_STATUS_PAGE_SIZE, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, resolveRagLimits, } from "./limits.js";
|
|
2
|
-
export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
|
|
3
1
|
export { chunkMarkdown, chunkText } from "./chunk.js";
|
|
4
|
-
export {
|
|
5
|
-
export {
|
|
2
|
+
export { createRagContextProvider } from "./context.js";
|
|
3
|
+
export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
|
|
6
4
|
export { indexChunks } from "./indexing.js";
|
|
7
|
-
export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
|
|
8
5
|
export { createMemoryIngestionStatusStore, listIngestionStatus } from "./ingestion-status.js";
|
|
6
|
+
export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
|
|
7
|
+
export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
|
|
8
|
+
export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
|
|
9
9
|
export { retrieveContext } from "./retrieve.js";
|
|
10
|
-
export {
|
|
10
|
+
export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
|
|
11
11
|
export const packageName = "@arnilo/prism-rag";
|
|
12
12
|
//# sourceMappingURL=index.js.map
|
package/dist/indexing.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { resolveRedactor } from "@arnilo/prism";
|
|
1
2
|
import { RagValidationError } from "./errors.js";
|
|
2
3
|
import { ingestionStatus } from "./ingestion-status.js";
|
|
3
4
|
import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
|
|
4
|
-
import { assertBytes, assertNotAborted, byteLength, nonEmpty, requireScope, requireSourceId
|
|
5
|
+
import { assertBytes, assertNotAborted, byteLength, nonEmpty, requireScope, requireSourceId } from "./util.js";
|
|
5
6
|
export async function indexChunks(options) {
|
|
6
7
|
return indexChunkBatches(options, async (records) => options.store.upsert(records, { signal: options.signal }));
|
|
7
8
|
}
|
|
@@ -14,7 +15,9 @@ export async function indexChunkBatches(options, write) {
|
|
|
14
15
|
maxVectorDimensions: options.maxVectorDimensions,
|
|
15
16
|
maxMetadataBytes: options.maxMetadataBytes,
|
|
16
17
|
});
|
|
17
|
-
if (!Number.isInteger(options.embedder.dimensions) ||
|
|
18
|
+
if (!Number.isInteger(options.embedder.dimensions) ||
|
|
19
|
+
options.embedder.dimensions <= 0 ||
|
|
20
|
+
options.embedder.dimensions > limits.maxVectorDimensions) {
|
|
18
21
|
throw new RagValidationError(`embedder dimensions must be an integer in 1..${limits.maxVectorDimensions}`);
|
|
19
22
|
}
|
|
20
23
|
if (options.chunks.length > limits.maxChunks)
|
|
@@ -31,7 +34,12 @@ export async function indexChunkBatches(options, write) {
|
|
|
31
34
|
throw new RagValidationError("chunk has inconsistent citation identity");
|
|
32
35
|
if (chunkIds.has(chunk.id))
|
|
33
36
|
throw new RagValidationError(`duplicate chunk id: ${chunk.id}`);
|
|
34
|
-
if (!Number.isInteger(chunk.index) ||
|
|
37
|
+
if (!Number.isInteger(chunk.index) ||
|
|
38
|
+
chunk.index < 0 ||
|
|
39
|
+
!Number.isInteger(chunk.start) ||
|
|
40
|
+
chunk.start < 0 ||
|
|
41
|
+
!Number.isInteger(chunk.end) ||
|
|
42
|
+
chunk.end < chunk.start) {
|
|
35
43
|
throw new RagValidationError("chunk has invalid index or offsets");
|
|
36
44
|
}
|
|
37
45
|
if (chunk.text.length > limits.chunkSize)
|
|
@@ -44,9 +52,9 @@ export async function indexChunkBatches(options, write) {
|
|
|
44
52
|
if (!options.statusStore)
|
|
45
53
|
return;
|
|
46
54
|
const message = error instanceof Error ? error.message : error === undefined ? undefined : "indexing failed";
|
|
47
|
-
const safeError = message ? redactor?.redact(message) ?? message : undefined;
|
|
55
|
+
const safeError = message ? (redactor?.redact(message) ?? message) : undefined;
|
|
48
56
|
for (const sourceId of sourceIds) {
|
|
49
|
-
const progress = state === "indexed" ? total.get(sourceId) : written.get(sourceId) ?? { bytes: 0, chunks: 0 };
|
|
57
|
+
const progress = state === "indexed" ? total.get(sourceId) : (written.get(sourceId) ?? { bytes: 0, chunks: 0 });
|
|
50
58
|
await options.statusStore.set(ingestionStatus(scope, sourceId, state, progress.bytes, progress.chunks, safeError));
|
|
51
59
|
}
|
|
52
60
|
};
|
|
@@ -64,8 +72,11 @@ export async function indexChunkBatches(options, write) {
|
|
|
64
72
|
if (embedding.length !== options.embedder.dimensions || embedding.some((value) => !Number.isFinite(value))) {
|
|
65
73
|
throw new RagValidationError(`embedder returned invalid vector; expected ${options.embedder.dimensions} finite values`);
|
|
66
74
|
}
|
|
67
|
-
const safeMetadata = redactor?.redact(chunk.metadata ?? {}) ??
|
|
68
|
-
const metadata = {
|
|
75
|
+
const safeMetadata = redactor?.redact(chunk.metadata ?? {}) ?? chunk.metadata ?? {};
|
|
76
|
+
const metadata = {
|
|
77
|
+
...safeMetadata,
|
|
78
|
+
_rag: { sourceId: chunk.sourceId, citationId: chunk.citationId, chunkIndex: chunk.index, start: chunk.start, end: chunk.end },
|
|
79
|
+
};
|
|
69
80
|
assertBytes(metadata, limits.maxMetadataBytes, "chunk metadata");
|
|
70
81
|
return {
|
|
71
82
|
id: chunk.id,
|
package/dist/ingestion-status.js
CHANGED
|
@@ -65,7 +65,9 @@ export function ingestionStatus(scope, sourceId, state, bytes, chunks, error) {
|
|
|
65
65
|
}
|
|
66
66
|
function assertStatus(status, scope) {
|
|
67
67
|
requireSourceId(status.sourceId);
|
|
68
|
-
if (status.scope.tenantId !== scope.tenantId ||
|
|
68
|
+
if (status.scope.tenantId !== scope.tenantId ||
|
|
69
|
+
status.scope.resourceId !== scope.resourceId ||
|
|
70
|
+
status.scope.corpusId !== scope.corpusId) {
|
|
69
71
|
throw new RagScopeError("ingestion status crossed tenant/resource/corpus boundary");
|
|
70
72
|
}
|
|
71
73
|
if (!["pending", "indexed", "failed", "partial"].includes(status.state))
|
package/dist/parsers.js
CHANGED
|
@@ -3,7 +3,9 @@ import { resolveRagLimits } from "./limits.js";
|
|
|
3
3
|
import { assertNotAborted } from "./util.js";
|
|
4
4
|
export const textParser = { parse: (document, options) => parseText(document, options, ["text/plain"]) };
|
|
5
5
|
export const markdownParser = { parse: (document, options) => parseText(document, options, ["text/markdown", "text/x-markdown"]) };
|
|
6
|
-
export const htmlParser = {
|
|
6
|
+
export const htmlParser = {
|
|
7
|
+
parse: (document, options) => parseText(document, options, ["text/html", "application/xhtml+xml"], htmlToText),
|
|
8
|
+
};
|
|
7
9
|
export const pdfParser = { parse: parsePdf };
|
|
8
10
|
async function parseText(document, options = {}, mediaTypes, transform = (text) => text) {
|
|
9
11
|
assertMediaType(document, mediaTypes);
|
package/dist/rerank.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { resolveRedactor } from "@arnilo/prism";
|
|
1
2
|
import type { RagHit, Reranker } from "./types.js";
|
|
2
|
-
import { resolveRedactor } from "./util.js";
|
|
3
3
|
export declare function rerankHits(query: string, hits: readonly RagHit[], options: {
|
|
4
4
|
readonly reranker: Reranker;
|
|
5
5
|
readonly maxBytes: number;
|
package/dist/rerank.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { resolveRedactor } from "@arnilo/prism";
|
|
1
2
|
import { RagAbortError, RagLimitError, RagValidationError } from "./errors.js";
|
|
2
|
-
import { assertNotAborted, byteLength
|
|
3
|
+
import { assertNotAborted, byteLength } from "./util.js";
|
|
3
4
|
const active = new WeakMap();
|
|
4
5
|
export async function rerankHits(query, hits, options) {
|
|
5
6
|
assertNotAborted(options.signal);
|
|
@@ -21,7 +22,9 @@ export async function rerankHits(query, hits, options) {
|
|
|
21
22
|
active.delete(options.reranker);
|
|
22
23
|
};
|
|
23
24
|
try {
|
|
24
|
-
const ordered = await boundedRerank((signal) => options.reranker.rerank({ query: safeQuery, hits: safeHits, signal }), (operation) => {
|
|
25
|
+
const ordered = await boundedRerank((signal) => options.reranker.rerank({ query: safeQuery, hits: safeHits, signal }), (operation) => {
|
|
26
|
+
void operation.then(release, release);
|
|
27
|
+
}, options.maxMs, options.signal);
|
|
25
28
|
const originals = new Map(hits.map((hit) => [hit.id, hit]));
|
|
26
29
|
if (ordered.length !== hits.length || new Set(ordered.map((hit) => hit.id)).size !== hits.length) {
|
|
27
30
|
throw new RagValidationError("reranker must return each retrieved hit exactly once");
|
package/dist/retrieve.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { resolveRedactor } from "@arnilo/prism";
|
|
2
2
|
import { RagScopeError, RagValidationError } from "./errors.js";
|
|
3
|
+
import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
|
|
3
4
|
import { rerankHits } from "./rerank.js";
|
|
4
|
-
import { assertBytes, assertNotAborted, assertScope, byteLength, isJsonObject, matchesFilter, nonEmpty, requireScope, requireSourceId,
|
|
5
|
+
import { assertBytes, assertNotAborted, assertScope, byteLength, isJsonObject, matchesFilter, nonEmpty, requireScope, requireSourceId, truncateUtf8, } from "./util.js";
|
|
5
6
|
const RETRIEVED_CONTENT_TRUST = Object.freeze({ untrusted: true, inert: true, injectionCapable: true });
|
|
6
7
|
export async function retrieveContext(query, options) {
|
|
7
8
|
nonEmpty(query, "query");
|
|
@@ -19,7 +20,9 @@ export async function retrieveContext(query, options) {
|
|
|
19
20
|
maxRerankMs: options.maxRerankMs,
|
|
20
21
|
rerankConcurrency: options.rerankConcurrency,
|
|
21
22
|
});
|
|
22
|
-
if (!Number.isInteger(options.embedder.dimensions) ||
|
|
23
|
+
if (!Number.isInteger(options.embedder.dimensions) ||
|
|
24
|
+
options.embedder.dimensions <= 0 ||
|
|
25
|
+
options.embedder.dimensions > limits.maxVectorDimensions) {
|
|
23
26
|
throw new RagValidationError(`embedder dimensions must be an integer in 1..${limits.maxVectorDimensions}`);
|
|
24
27
|
}
|
|
25
28
|
if (options.filter)
|
|
@@ -29,7 +32,10 @@ export async function retrieveContext(query, options) {
|
|
|
29
32
|
assertNotAborted(options.signal);
|
|
30
33
|
const vectors = await options.embedder.embed([safeQuery], { signal: options.signal });
|
|
31
34
|
const embedding = vectors[0];
|
|
32
|
-
if (vectors.length !== 1 ||
|
|
35
|
+
if (vectors.length !== 1 ||
|
|
36
|
+
!embedding ||
|
|
37
|
+
embedding.length !== options.embedder.dimensions ||
|
|
38
|
+
embedding.some((value) => !Number.isFinite(value))) {
|
|
33
39
|
throw new RagValidationError("embedder returned invalid query vector");
|
|
34
40
|
}
|
|
35
41
|
const candidates = await options.store.query({
|
|
@@ -121,7 +127,13 @@ function parseHit(hit, retrievalRank, retrievedAt) {
|
|
|
121
127
|
throw new RagScopeError("vector hit is missing RAG source metadata");
|
|
122
128
|
const sourceId = requireSourceId(rag.sourceId);
|
|
123
129
|
const citationId = nonEmpty(rag.citationId, "metadata._rag.citationId");
|
|
124
|
-
if (!Number.isInteger(rag.chunkIndex) ||
|
|
130
|
+
if (!Number.isInteger(rag.chunkIndex) ||
|
|
131
|
+
Number(rag.chunkIndex) < 0 ||
|
|
132
|
+
!Number.isInteger(rag.start) ||
|
|
133
|
+
Number(rag.start) < 0 ||
|
|
134
|
+
!Number.isInteger(rag.end) ||
|
|
135
|
+
Number(rag.end) < Number(rag.start) ||
|
|
136
|
+
!Number.isFinite(hit.score)) {
|
|
125
137
|
throw new RagValidationError("vector hit has invalid RAG offsets");
|
|
126
138
|
}
|
|
127
139
|
if (hit.id !== citationId || !citationId.startsWith(`${sourceId}#`))
|
package/dist/sources.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import { resolveRedactor } from "@arnilo/prism";
|
|
1
2
|
import { chunkText } from "./chunk.js";
|
|
2
3
|
import { RagScopeError, RagValidationError } from "./errors.js";
|
|
3
|
-
import { ingestionStatus } from "./ingestion-status.js";
|
|
4
4
|
import { indexChunkBatches } from "./indexing.js";
|
|
5
|
-
import {
|
|
5
|
+
import { ingestionStatus } from "./ingestion-status.js";
|
|
6
|
+
import { assertNotAborted, byteLength, requireScope, requireSourceId } from "./util.js";
|
|
6
7
|
export async function replaceSource(options) {
|
|
7
8
|
const sourceId = requireSourceId(options.sourceId);
|
|
8
9
|
const scope = requireScope(options.scope);
|
|
@@ -16,12 +17,14 @@ export async function replaceSource(options) {
|
|
|
16
17
|
if (!options.statusStore)
|
|
17
18
|
return;
|
|
18
19
|
const message = error instanceof Error ? error.message : error === undefined ? undefined : "source replacement failed";
|
|
19
|
-
await options.statusStore.set(ingestionStatus(scope, sourceId, state, state === "indexed" ? totalBytes : 0, state === "indexed" ? options.chunks.length : 0, message ? redactor?.redact(message) ?? message : undefined));
|
|
20
|
+
await options.statusStore.set(ingestionStatus(scope, sourceId, state, state === "indexed" ? totalBytes : 0, state === "indexed" ? options.chunks.length : 0, message ? (redactor?.redact(message) ?? message) : undefined));
|
|
20
21
|
};
|
|
21
22
|
await setStatus("pending");
|
|
22
23
|
try {
|
|
23
24
|
const staged = [];
|
|
24
|
-
const indexed = await indexChunkBatches({ ...options, statusStore: undefined }, async (records) => {
|
|
25
|
+
const indexed = await indexChunkBatches({ ...options, statusStore: undefined }, async (records) => {
|
|
26
|
+
staged.push(...records);
|
|
27
|
+
});
|
|
25
28
|
assertNotAborted(options.signal);
|
|
26
29
|
const result = await options.store.transaction(async (store) => {
|
|
27
30
|
const previous = await sourceRecords(store, sourceId, scope, options.signal);
|
|
@@ -87,7 +90,9 @@ function mergeMetadata(parser, supplied) {
|
|
|
87
90
|
}
|
|
88
91
|
async function sourceRecords(store, sourceId, scope, signal) {
|
|
89
92
|
assertNotAborted(signal);
|
|
90
|
-
const records = await store.getBySource({ tenantId: scope.tenantId, resourceId: scope.resourceId, threadId: scope.corpusId }, sourceId, {
|
|
93
|
+
const records = await store.getBySource({ tenantId: scope.tenantId, resourceId: scope.resourceId, threadId: scope.corpusId }, sourceId, {
|
|
94
|
+
signal,
|
|
95
|
+
});
|
|
91
96
|
for (const record of records) {
|
|
92
97
|
if (record.tenantId !== scope.tenantId || record.resourceId !== scope.resourceId || record.threadId !== scope.corpusId) {
|
|
93
98
|
throw new RagScopeError("source lookup crossed tenant/resource/corpus boundary");
|
package/dist/util.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type JsonObject, type JsonValue, type Message
|
|
1
|
+
import { type JsonObject, type JsonValue, type Message } from "@arnilo/prism";
|
|
2
2
|
import type { RagScope } from "./types.js";
|
|
3
3
|
export declare function assertNotAborted(signal?: AbortSignal): void;
|
|
4
4
|
export declare function nonEmpty(value: unknown, label: string): string;
|
|
@@ -9,7 +9,6 @@ export declare function assertScope(expected: RagScope, actual: {
|
|
|
9
9
|
resourceId: string;
|
|
10
10
|
threadId: string;
|
|
11
11
|
}): void;
|
|
12
|
-
export declare function resolveRedactor(redactor?: SecretRedactor, secrets?: readonly (string | undefined)[]): SecretRedactor | undefined;
|
|
13
12
|
export declare function byteLength(value: unknown): number;
|
|
14
13
|
export declare function assertBytes(value: unknown, limit: number, label: string): void;
|
|
15
14
|
export declare function latestUserText(messages: readonly Message[]): string | undefined;
|
package/dist/util.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { createSecretRedactor } from "@arnilo/prism";
|
|
2
1
|
import { RagAbortError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
|
|
3
2
|
export function assertNotAborted(signal) {
|
|
4
3
|
if (signal?.aborted)
|
|
@@ -24,15 +23,10 @@ export function requireScope(scope) {
|
|
|
24
23
|
};
|
|
25
24
|
}
|
|
26
25
|
export function assertScope(expected, actual) {
|
|
27
|
-
if (actual.tenantId !== expected.tenantId
|
|
28
|
-
|| actual.resourceId !== expected.resourceId
|
|
29
|
-
|| actual.threadId !== expected.corpusId) {
|
|
26
|
+
if (actual.tenantId !== expected.tenantId || actual.resourceId !== expected.resourceId || actual.threadId !== expected.corpusId) {
|
|
30
27
|
throw new RagScopeError("vector hit crossed tenant/resource/corpus boundary");
|
|
31
28
|
}
|
|
32
29
|
}
|
|
33
|
-
export function resolveRedactor(redactor, secrets) {
|
|
34
|
-
return redactor ?? (secrets?.length ? createSecretRedactor(secrets) : undefined);
|
|
35
|
-
}
|
|
36
30
|
export function byteLength(value) {
|
|
37
31
|
return Buffer.byteLength(typeof value === "string" ? value : JSON.stringify(value), "utf8");
|
|
38
32
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@arnilo/prism-rag",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.17",
|
|
4
4
|
"description": "Optional bounded text and Markdown RAG primitives for Prism.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -33,8 +33,8 @@
|
|
|
33
33
|
"pack:dry-run": "npm pack --dry-run"
|
|
34
34
|
},
|
|
35
35
|
"peerDependencies": {
|
|
36
|
-
"@arnilo/prism": "0.0.
|
|
37
|
-
"@arnilo/prism-memory": "0.0.
|
|
36
|
+
"@arnilo/prism": "0.0.17",
|
|
37
|
+
"@arnilo/prism-memory": "0.0.17"
|
|
38
38
|
},
|
|
39
39
|
"devDependencies": {
|
|
40
40
|
"@arnilo/prism": "file:../..",
|