@arnilo/prism-rag 0.0.15 → 0.0.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,15 @@
1
1
  # Changelog
2
2
 
3
+ ## [0.0.17] - 2026-07-29
4
+
5
+ ### Changed
6
+ - Released with exact 0.0.17 graph.
7
+
8
+ ## [0.0.16] - 2026-07-26
9
+
10
+ ### Changed
11
+ - Sourced `resolveRedactor` from `@arnilo/prism` core and deleted the private duplicate; no public API change.
12
+
3
13
  ## [0.0.15] - 2026-07-26
4
14
 
5
15
  ### Added
package/dist/chunk.d.ts CHANGED
@@ -1,3 +1,3 @@
1
- import type { RagChunk, ChunkOptions } from "./types.js";
1
+ import type { ChunkOptions, RagChunk } from "./types.js";
2
2
  export declare function chunkText(text: string, options: ChunkOptions): readonly RagChunk[];
3
3
  export declare function chunkMarkdown(markdown: string, options: ChunkOptions): readonly RagChunk[];
package/dist/chunk.js CHANGED
@@ -1,5 +1,5 @@
1
- import { resolveRagLimits } from "./limits.js";
2
1
  import { RagLimitError } from "./errors.js";
2
+ import { resolveRagLimits } from "./limits.js";
3
3
  import { assertBytes, requireSourceId } from "./util.js";
4
4
  export function chunkText(text, options) {
5
5
  return chunkDocument(text, options, false);
package/dist/context.js CHANGED
@@ -7,18 +7,20 @@ export function createRagContextProvider(options) {
7
7
  context.signal?.throwIfAborted();
8
8
  const query = typeof options.query === "function"
9
9
  ? options.query({ messages: context.messages })
10
- : options.query ?? latestUserText(context.messages);
10
+ : (options.query ?? latestUserText(context.messages));
11
11
  if (!query?.trim())
12
12
  return [];
13
13
  const result = await retrieveContext(query, { ...options, signal: context.signal });
14
14
  if (!result.text)
15
15
  return [];
16
- return [{
16
+ return [
17
+ {
17
18
  id: `${options.name ?? "rag"}:context`,
18
19
  title: options.title ?? "Retrieved context",
19
20
  content: result.text,
20
21
  metadata: { citations: result.citations, trust: result.trust, inert: true, untrusted: true, injectionCapable: true },
21
- }];
22
+ },
23
+ ];
22
24
  },
23
25
  };
24
26
  }
package/dist/index.d.ts CHANGED
@@ -1,14 +1,14 @@
1
- export { DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, DEFAULT_MAX_DOCUMENT_BYTES, HARD_MAX_DOCUMENT_BYTES_CAP, DEFAULT_MAX_PARSE_MS, HARD_MAX_PARSE_MS_CAP, DEFAULT_MAX_PDF_PAGES, HARD_MAX_PDF_PAGES_CAP, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, DEFAULT_TOP_K, HARD_TOP_K_CAP, DEFAULT_QUERY_CANDIDATES, HARD_QUERY_CANDIDATES_CAP, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_MAX_RERANK_BYTES, HARD_MAX_RERANK_BYTES_CAP, DEFAULT_MAX_RERANK_MS, HARD_MAX_RERANK_MS_CAP, DEFAULT_RERANK_CONCURRENCY, HARD_RERANK_CONCURRENCY_CAP, DEFAULT_INGESTION_STATUS_PAGE_SIZE, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, resolveRagLimits, } from "./limits.js";
2
- export type { RagLimits, RagLimitsInput } from "./limits.js";
3
- export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
4
1
  export { chunkMarkdown, chunkText } from "./chunk.js";
5
- export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
6
- export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
2
+ export { createRagContextProvider } from "./context.js";
3
+ export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
7
4
  export { indexChunks } from "./indexing.js";
8
- export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
9
- export type { SourceMutationResult } from "./sources.js";
10
5
  export { createMemoryIngestionStatusStore, listIngestionStatus } from "./ingestion-status.js";
6
+ export type { RagLimits, RagLimitsInput } from "./limits.js";
7
+ export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
8
+ export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
9
+ export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
11
10
  export { retrieveContext } from "./retrieve.js";
12
- export { createRagContextProvider } from "./context.js";
13
- export type { ChunkOptions, Chunker, DeleteSourceOptions, DocumentLoader, DocumentLoadOptions, DocumentParseOptions, IndexChunksOptions, IndexChunksResult, IngestionState, IngestionStatus, IngestionStatusQuery, IngestionStatusStore, RagChunk, LoadedDocument, ParsedDocument, Parser, RagCitation, RagContextProvider, RagContextProviderOptions, RagContextResult, RagContentTrust, RagHit, RagProvenance, Reranker, RagScope, ReplaceDocumentOptions, ReplaceSourceOptions, RetrieveContextOptions, SourceVectorStore, TransactionalVectorStore, } from "./types.js";
11
+ export type { SourceMutationResult } from "./sources.js";
12
+ export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
13
+ export type { Chunker, ChunkOptions, DeleteSourceOptions, DocumentLoader, DocumentLoadOptions, DocumentParseOptions, IndexChunksOptions, IndexChunksResult, IngestionState, IngestionStatus, IngestionStatusQuery, IngestionStatusStore, LoadedDocument, ParsedDocument, Parser, RagChunk, RagCitation, RagContentTrust, RagContextProvider, RagContextProviderOptions, RagContextResult, RagHit, RagProvenance, RagScope, ReplaceDocumentOptions, ReplaceSourceOptions, Reranker, RetrieveContextOptions, SourceVectorStore, TransactionalVectorStore, } from "./types.js";
14
14
  export declare const packageName = "@arnilo/prism-rag";
package/dist/index.js CHANGED
@@ -1,12 +1,12 @@
1
- export { DEFAULT_CHUNK_SIZE, HARD_CHUNK_SIZE_CAP, DEFAULT_CHUNK_OVERLAP, HARD_CHUNK_OVERLAP_CAP, DEFAULT_MAX_DOCUMENT_CHARS, HARD_MAX_DOCUMENT_CHARS_CAP, DEFAULT_MAX_DOCUMENT_BYTES, HARD_MAX_DOCUMENT_BYTES_CAP, DEFAULT_MAX_PARSE_MS, HARD_MAX_PARSE_MS_CAP, DEFAULT_MAX_PDF_PAGES, HARD_MAX_PDF_PAGES_CAP, DEFAULT_MAX_CHUNKS, HARD_MAX_CHUNKS_CAP, DEFAULT_EMBED_BATCH_SIZE, HARD_EMBED_BATCH_SIZE_CAP, DEFAULT_TOP_K, HARD_TOP_K_CAP, DEFAULT_QUERY_CANDIDATES, HARD_QUERY_CANDIDATES_CAP, DEFAULT_MAX_RESULT_BYTES, HARD_MAX_RESULT_BYTES_CAP, DEFAULT_MAX_CONTEXT_TOKENS, HARD_MAX_CONTEXT_TOKENS_CAP, DEFAULT_MAX_METADATA_BYTES, HARD_MAX_METADATA_BYTES_CAP, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_MAX_RERANK_BYTES, HARD_MAX_RERANK_BYTES_CAP, DEFAULT_MAX_RERANK_MS, HARD_MAX_RERANK_MS_CAP, DEFAULT_RERANK_CONCURRENCY, HARD_RERANK_CONCURRENCY_CAP, DEFAULT_INGESTION_STATUS_PAGE_SIZE, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, resolveRagLimits, } from "./limits.js";
2
- export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
3
1
  export { chunkMarkdown, chunkText } from "./chunk.js";
4
- export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
5
- export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
2
+ export { createRagContextProvider } from "./context.js";
3
+ export { RagAbortError, RagError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
6
4
  export { indexChunks } from "./indexing.js";
7
- export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
8
5
  export { createMemoryIngestionStatusStore, listIngestionStatus } from "./ingestion-status.js";
6
+ export { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_INGESTION_STATUS_PAGE_SIZE, DEFAULT_MAX_CHUNKS, DEFAULT_MAX_CONTEXT_TOKENS, DEFAULT_MAX_DOCUMENT_BYTES, DEFAULT_MAX_DOCUMENT_CHARS, DEFAULT_MAX_METADATA_BYTES, DEFAULT_MAX_PARSE_MS, DEFAULT_MAX_PDF_PAGES, DEFAULT_MAX_RERANK_BYTES, DEFAULT_MAX_RERANK_MS, DEFAULT_MAX_RESULT_BYTES, DEFAULT_MAX_VECTOR_DIMENSIONS, DEFAULT_QUERY_CANDIDATES, DEFAULT_RERANK_CONCURRENCY, DEFAULT_TOP_K, HARD_CHUNK_OVERLAP_CAP, HARD_CHUNK_SIZE_CAP, HARD_EMBED_BATCH_SIZE_CAP, HARD_INGESTION_STATUS_PAGE_SIZE_CAP, HARD_MAX_CHUNKS_CAP, HARD_MAX_CONTEXT_TOKENS_CAP, HARD_MAX_DOCUMENT_BYTES_CAP, HARD_MAX_DOCUMENT_CHARS_CAP, HARD_MAX_METADATA_BYTES_CAP, HARD_MAX_PARSE_MS_CAP, HARD_MAX_PDF_PAGES_CAP, HARD_MAX_RERANK_BYTES_CAP, HARD_MAX_RERANK_MS_CAP, HARD_MAX_RESULT_BYTES_CAP, HARD_QUERY_CANDIDATES_CAP, HARD_RERANK_CONCURRENCY_CAP, HARD_TOP_K_CAP, resolveRagLimits, } from "./limits.js";
7
+ export { createResourceDocumentLoader, createWebFetchDocumentLoader } from "./loaders.js";
8
+ export { htmlParser, markdownParser, pdfParser, textParser } from "./parsers.js";
9
9
  export { retrieveContext } from "./retrieve.js";
10
- export { createRagContextProvider } from "./context.js";
10
+ export { deleteSource, replaceDocument, replaceSource } from "./sources.js";
11
11
  export const packageName = "@arnilo/prism-rag";
12
12
  //# sourceMappingURL=index.js.map
package/dist/indexing.js CHANGED
@@ -1,7 +1,8 @@
1
+ import { resolveRedactor } from "@arnilo/prism";
1
2
  import { RagValidationError } from "./errors.js";
2
3
  import { ingestionStatus } from "./ingestion-status.js";
3
4
  import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
4
- import { assertBytes, assertNotAborted, byteLength, nonEmpty, requireScope, requireSourceId, resolveRedactor } from "./util.js";
5
+ import { assertBytes, assertNotAborted, byteLength, nonEmpty, requireScope, requireSourceId } from "./util.js";
5
6
  export async function indexChunks(options) {
6
7
  return indexChunkBatches(options, async (records) => options.store.upsert(records, { signal: options.signal }));
7
8
  }
@@ -14,7 +15,9 @@ export async function indexChunkBatches(options, write) {
14
15
  maxVectorDimensions: options.maxVectorDimensions,
15
16
  maxMetadataBytes: options.maxMetadataBytes,
16
17
  });
17
- if (!Number.isInteger(options.embedder.dimensions) || options.embedder.dimensions <= 0 || options.embedder.dimensions > limits.maxVectorDimensions) {
18
+ if (!Number.isInteger(options.embedder.dimensions) ||
19
+ options.embedder.dimensions <= 0 ||
20
+ options.embedder.dimensions > limits.maxVectorDimensions) {
18
21
  throw new RagValidationError(`embedder dimensions must be an integer in 1..${limits.maxVectorDimensions}`);
19
22
  }
20
23
  if (options.chunks.length > limits.maxChunks)
@@ -31,7 +34,12 @@ export async function indexChunkBatches(options, write) {
31
34
  throw new RagValidationError("chunk has inconsistent citation identity");
32
35
  if (chunkIds.has(chunk.id))
33
36
  throw new RagValidationError(`duplicate chunk id: ${chunk.id}`);
34
- if (!Number.isInteger(chunk.index) || chunk.index < 0 || !Number.isInteger(chunk.start) || chunk.start < 0 || !Number.isInteger(chunk.end) || chunk.end < chunk.start) {
37
+ if (!Number.isInteger(chunk.index) ||
38
+ chunk.index < 0 ||
39
+ !Number.isInteger(chunk.start) ||
40
+ chunk.start < 0 ||
41
+ !Number.isInteger(chunk.end) ||
42
+ chunk.end < chunk.start) {
35
43
  throw new RagValidationError("chunk has invalid index or offsets");
36
44
  }
37
45
  if (chunk.text.length > limits.chunkSize)
@@ -44,9 +52,9 @@ export async function indexChunkBatches(options, write) {
44
52
  if (!options.statusStore)
45
53
  return;
46
54
  const message = error instanceof Error ? error.message : error === undefined ? undefined : "indexing failed";
47
- const safeError = message ? redactor?.redact(message) ?? message : undefined;
55
+ const safeError = message ? (redactor?.redact(message) ?? message) : undefined;
48
56
  for (const sourceId of sourceIds) {
49
- const progress = state === "indexed" ? total.get(sourceId) : written.get(sourceId) ?? { bytes: 0, chunks: 0 };
57
+ const progress = state === "indexed" ? total.get(sourceId) : (written.get(sourceId) ?? { bytes: 0, chunks: 0 });
50
58
  await options.statusStore.set(ingestionStatus(scope, sourceId, state, progress.bytes, progress.chunks, safeError));
51
59
  }
52
60
  };
@@ -64,8 +72,11 @@ export async function indexChunkBatches(options, write) {
64
72
  if (embedding.length !== options.embedder.dimensions || embedding.some((value) => !Number.isFinite(value))) {
65
73
  throw new RagValidationError(`embedder returned invalid vector; expected ${options.embedder.dimensions} finite values`);
66
74
  }
67
- const safeMetadata = redactor?.redact(chunk.metadata ?? {}) ?? (chunk.metadata ?? {});
68
- const metadata = { ...safeMetadata, _rag: { sourceId: chunk.sourceId, citationId: chunk.citationId, chunkIndex: chunk.index, start: chunk.start, end: chunk.end } };
75
+ const safeMetadata = redactor?.redact(chunk.metadata ?? {}) ?? chunk.metadata ?? {};
76
+ const metadata = {
77
+ ...safeMetadata,
78
+ _rag: { sourceId: chunk.sourceId, citationId: chunk.citationId, chunkIndex: chunk.index, start: chunk.start, end: chunk.end },
79
+ };
69
80
  assertBytes(metadata, limits.maxMetadataBytes, "chunk metadata");
70
81
  return {
71
82
  id: chunk.id,
@@ -65,7 +65,9 @@ export function ingestionStatus(scope, sourceId, state, bytes, chunks, error) {
65
65
  }
66
66
  function assertStatus(status, scope) {
67
67
  requireSourceId(status.sourceId);
68
- if (status.scope.tenantId !== scope.tenantId || status.scope.resourceId !== scope.resourceId || status.scope.corpusId !== scope.corpusId) {
68
+ if (status.scope.tenantId !== scope.tenantId ||
69
+ status.scope.resourceId !== scope.resourceId ||
70
+ status.scope.corpusId !== scope.corpusId) {
69
71
  throw new RagScopeError("ingestion status crossed tenant/resource/corpus boundary");
70
72
  }
71
73
  if (!["pending", "indexed", "failed", "partial"].includes(status.state))
package/dist/parsers.js CHANGED
@@ -3,7 +3,9 @@ import { resolveRagLimits } from "./limits.js";
3
3
  import { assertNotAborted } from "./util.js";
4
4
  export const textParser = { parse: (document, options) => parseText(document, options, ["text/plain"]) };
5
5
  export const markdownParser = { parse: (document, options) => parseText(document, options, ["text/markdown", "text/x-markdown"]) };
6
- export const htmlParser = { parse: (document, options) => parseText(document, options, ["text/html", "application/xhtml+xml"], htmlToText) };
6
+ export const htmlParser = {
7
+ parse: (document, options) => parseText(document, options, ["text/html", "application/xhtml+xml"], htmlToText),
8
+ };
7
9
  export const pdfParser = { parse: parsePdf };
8
10
  async function parseText(document, options = {}, mediaTypes, transform = (text) => text) {
9
11
  assertMediaType(document, mediaTypes);
package/dist/rerank.d.ts CHANGED
@@ -1,5 +1,5 @@
1
+ import { resolveRedactor } from "@arnilo/prism";
1
2
  import type { RagHit, Reranker } from "./types.js";
2
- import { resolveRedactor } from "./util.js";
3
3
  export declare function rerankHits(query: string, hits: readonly RagHit[], options: {
4
4
  readonly reranker: Reranker;
5
5
  readonly maxBytes: number;
package/dist/rerank.js CHANGED
@@ -1,5 +1,6 @@
1
+ import { resolveRedactor } from "@arnilo/prism";
1
2
  import { RagAbortError, RagLimitError, RagValidationError } from "./errors.js";
2
- import { assertNotAborted, byteLength, resolveRedactor } from "./util.js";
3
+ import { assertNotAborted, byteLength } from "./util.js";
3
4
  const active = new WeakMap();
4
5
  export async function rerankHits(query, hits, options) {
5
6
  assertNotAborted(options.signal);
@@ -21,7 +22,9 @@ export async function rerankHits(query, hits, options) {
21
22
  active.delete(options.reranker);
22
23
  };
23
24
  try {
24
- const ordered = await boundedRerank((signal) => options.reranker.rerank({ query: safeQuery, hits: safeHits, signal }), (operation) => { void operation.then(release, release); }, options.maxMs, options.signal);
25
+ const ordered = await boundedRerank((signal) => options.reranker.rerank({ query: safeQuery, hits: safeHits, signal }), (operation) => {
26
+ void operation.then(release, release);
27
+ }, options.maxMs, options.signal);
25
28
  const originals = new Map(hits.map((hit) => [hit.id, hit]));
26
29
  if (ordered.length !== hits.length || new Set(ordered.map((hit) => hit.id)).size !== hits.length) {
27
30
  throw new RagValidationError("reranker must return each retrieved hit exactly once");
package/dist/retrieve.js CHANGED
@@ -1,7 +1,8 @@
1
- import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
1
+ import { resolveRedactor } from "@arnilo/prism";
2
2
  import { RagScopeError, RagValidationError } from "./errors.js";
3
+ import { HARD_CHUNK_SIZE_CAP, resolveRagLimits } from "./limits.js";
3
4
  import { rerankHits } from "./rerank.js";
4
- import { assertBytes, assertNotAborted, assertScope, byteLength, isJsonObject, matchesFilter, nonEmpty, requireScope, requireSourceId, resolveRedactor, truncateUtf8, } from "./util.js";
5
+ import { assertBytes, assertNotAborted, assertScope, byteLength, isJsonObject, matchesFilter, nonEmpty, requireScope, requireSourceId, truncateUtf8, } from "./util.js";
5
6
  const RETRIEVED_CONTENT_TRUST = Object.freeze({ untrusted: true, inert: true, injectionCapable: true });
6
7
  export async function retrieveContext(query, options) {
7
8
  nonEmpty(query, "query");
@@ -19,7 +20,9 @@ export async function retrieveContext(query, options) {
19
20
  maxRerankMs: options.maxRerankMs,
20
21
  rerankConcurrency: options.rerankConcurrency,
21
22
  });
22
- if (!Number.isInteger(options.embedder.dimensions) || options.embedder.dimensions <= 0 || options.embedder.dimensions > limits.maxVectorDimensions) {
23
+ if (!Number.isInteger(options.embedder.dimensions) ||
24
+ options.embedder.dimensions <= 0 ||
25
+ options.embedder.dimensions > limits.maxVectorDimensions) {
23
26
  throw new RagValidationError(`embedder dimensions must be an integer in 1..${limits.maxVectorDimensions}`);
24
27
  }
25
28
  if (options.filter)
@@ -29,7 +32,10 @@ export async function retrieveContext(query, options) {
29
32
  assertNotAborted(options.signal);
30
33
  const vectors = await options.embedder.embed([safeQuery], { signal: options.signal });
31
34
  const embedding = vectors[0];
32
- if (vectors.length !== 1 || !embedding || embedding.length !== options.embedder.dimensions || embedding.some((value) => !Number.isFinite(value))) {
35
+ if (vectors.length !== 1 ||
36
+ !embedding ||
37
+ embedding.length !== options.embedder.dimensions ||
38
+ embedding.some((value) => !Number.isFinite(value))) {
33
39
  throw new RagValidationError("embedder returned invalid query vector");
34
40
  }
35
41
  const candidates = await options.store.query({
@@ -121,7 +127,13 @@ function parseHit(hit, retrievalRank, retrievedAt) {
121
127
  throw new RagScopeError("vector hit is missing RAG source metadata");
122
128
  const sourceId = requireSourceId(rag.sourceId);
123
129
  const citationId = nonEmpty(rag.citationId, "metadata._rag.citationId");
124
- if (!Number.isInteger(rag.chunkIndex) || Number(rag.chunkIndex) < 0 || !Number.isInteger(rag.start) || Number(rag.start) < 0 || !Number.isInteger(rag.end) || Number(rag.end) < Number(rag.start) || !Number.isFinite(hit.score)) {
130
+ if (!Number.isInteger(rag.chunkIndex) ||
131
+ Number(rag.chunkIndex) < 0 ||
132
+ !Number.isInteger(rag.start) ||
133
+ Number(rag.start) < 0 ||
134
+ !Number.isInteger(rag.end) ||
135
+ Number(rag.end) < Number(rag.start) ||
136
+ !Number.isFinite(hit.score)) {
125
137
  throw new RagValidationError("vector hit has invalid RAG offsets");
126
138
  }
127
139
  if (hit.id !== citationId || !citationId.startsWith(`${sourceId}#`))
package/dist/sources.js CHANGED
@@ -1,8 +1,9 @@
1
+ import { resolveRedactor } from "@arnilo/prism";
1
2
  import { chunkText } from "./chunk.js";
2
3
  import { RagScopeError, RagValidationError } from "./errors.js";
3
- import { ingestionStatus } from "./ingestion-status.js";
4
4
  import { indexChunkBatches } from "./indexing.js";
5
- import { assertNotAborted, byteLength, requireScope, requireSourceId, resolveRedactor } from "./util.js";
5
+ import { ingestionStatus } from "./ingestion-status.js";
6
+ import { assertNotAborted, byteLength, requireScope, requireSourceId } from "./util.js";
6
7
  export async function replaceSource(options) {
7
8
  const sourceId = requireSourceId(options.sourceId);
8
9
  const scope = requireScope(options.scope);
@@ -16,12 +17,14 @@ export async function replaceSource(options) {
16
17
  if (!options.statusStore)
17
18
  return;
18
19
  const message = error instanceof Error ? error.message : error === undefined ? undefined : "source replacement failed";
19
- await options.statusStore.set(ingestionStatus(scope, sourceId, state, state === "indexed" ? totalBytes : 0, state === "indexed" ? options.chunks.length : 0, message ? redactor?.redact(message) ?? message : undefined));
20
+ await options.statusStore.set(ingestionStatus(scope, sourceId, state, state === "indexed" ? totalBytes : 0, state === "indexed" ? options.chunks.length : 0, message ? (redactor?.redact(message) ?? message) : undefined));
20
21
  };
21
22
  await setStatus("pending");
22
23
  try {
23
24
  const staged = [];
24
- const indexed = await indexChunkBatches({ ...options, statusStore: undefined }, async (records) => { staged.push(...records); });
25
+ const indexed = await indexChunkBatches({ ...options, statusStore: undefined }, async (records) => {
26
+ staged.push(...records);
27
+ });
25
28
  assertNotAborted(options.signal);
26
29
  const result = await options.store.transaction(async (store) => {
27
30
  const previous = await sourceRecords(store, sourceId, scope, options.signal);
@@ -87,7 +90,9 @@ function mergeMetadata(parser, supplied) {
87
90
  }
88
91
  async function sourceRecords(store, sourceId, scope, signal) {
89
92
  assertNotAborted(signal);
90
- const records = await store.getBySource({ tenantId: scope.tenantId, resourceId: scope.resourceId, threadId: scope.corpusId }, sourceId, { signal });
93
+ const records = await store.getBySource({ tenantId: scope.tenantId, resourceId: scope.resourceId, threadId: scope.corpusId }, sourceId, {
94
+ signal,
95
+ });
91
96
  for (const record of records) {
92
97
  if (record.tenantId !== scope.tenantId || record.resourceId !== scope.resourceId || record.threadId !== scope.corpusId) {
93
98
  throw new RagScopeError("source lookup crossed tenant/resource/corpus boundary");
package/dist/util.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { type JsonObject, type JsonValue, type Message, type SecretRedactor } from "@arnilo/prism";
1
+ import { type JsonObject, type JsonValue, type Message } from "@arnilo/prism";
2
2
  import type { RagScope } from "./types.js";
3
3
  export declare function assertNotAborted(signal?: AbortSignal): void;
4
4
  export declare function nonEmpty(value: unknown, label: string): string;
@@ -9,7 +9,6 @@ export declare function assertScope(expected: RagScope, actual: {
9
9
  resourceId: string;
10
10
  threadId: string;
11
11
  }): void;
12
- export declare function resolveRedactor(redactor?: SecretRedactor, secrets?: readonly (string | undefined)[]): SecretRedactor | undefined;
13
12
  export declare function byteLength(value: unknown): number;
14
13
  export declare function assertBytes(value: unknown, limit: number, label: string): void;
15
14
  export declare function latestUserText(messages: readonly Message[]): string | undefined;
package/dist/util.js CHANGED
@@ -1,4 +1,3 @@
1
- import { createSecretRedactor } from "@arnilo/prism";
2
1
  import { RagAbortError, RagLimitError, RagScopeError, RagValidationError } from "./errors.js";
3
2
  export function assertNotAborted(signal) {
4
3
  if (signal?.aborted)
@@ -24,15 +23,10 @@ export function requireScope(scope) {
24
23
  };
25
24
  }
26
25
  export function assertScope(expected, actual) {
27
- if (actual.tenantId !== expected.tenantId
28
- || actual.resourceId !== expected.resourceId
29
- || actual.threadId !== expected.corpusId) {
26
+ if (actual.tenantId !== expected.tenantId || actual.resourceId !== expected.resourceId || actual.threadId !== expected.corpusId) {
30
27
  throw new RagScopeError("vector hit crossed tenant/resource/corpus boundary");
31
28
  }
32
29
  }
33
- export function resolveRedactor(redactor, secrets) {
34
- return redactor ?? (secrets?.length ? createSecretRedactor(secrets) : undefined);
35
- }
36
30
  export function byteLength(value) {
37
31
  return Buffer.byteLength(typeof value === "string" ? value : JSON.stringify(value), "utf8");
38
32
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@arnilo/prism-rag",
3
- "version": "0.0.15",
3
+ "version": "0.0.17",
4
4
  "description": "Optional bounded text and Markdown RAG primitives for Prism.",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -33,8 +33,8 @@
33
33
  "pack:dry-run": "npm pack --dry-run"
34
34
  },
35
35
  "peerDependencies": {
36
- "@arnilo/prism": "0.0.15",
37
- "@arnilo/prism-memory": "0.0.15"
36
+ "@arnilo/prism": "0.0.17",
37
+ "@arnilo/prism-memory": "0.0.17"
38
38
  },
39
39
  "devDependencies": {
40
40
  "@arnilo/prism": "file:../..",