@devilsdev/rag-core 3.3.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/all.d.ts +58 -0
- package/dist/index.cjs +275 -15
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +1 -1
- package/dist/types/adapters/ai-sdk-middleware.d.ts +10 -0
- package/dist/types/adapters/mastra.d.ts +6 -2
- package/dist/types/agentic/agentic-pipeline.d.ts +1 -1
- package/dist/types/agentic/iterative-retriever.d.ts +1 -1
- package/dist/types/agentic/query-planner.d.ts +1 -1
- package/dist/types/agentic/self-critique.d.ts +1 -1
- package/dist/types/ai/federation/federated-learning-coordinator.d.ts +1 -1
- package/dist/types/ai/training/model-training-orchestrator.d.ts +1 -1
- package/dist/types/chunking/chunking-engine.d.ts +1 -1
- package/dist/types/citation/citation-tracker.d.ts +1 -1
- package/dist/types/connectors/anthropic-connector.d.ts +35 -1
- package/dist/types/connectors/base-connector.d.ts +1 -1
- package/dist/types/connectors/connector-utils.d.ts +20 -0
- package/dist/types/cost/cost-calculator.d.ts +1 -1
- package/dist/types/cost/cost-tracker.d.ts +1 -1
- package/dist/types/cost/token-budget.d.ts +1 -1
- package/dist/types/dashboard/dashboard-generator.d.ts +1 -1
- package/dist/types/dashboard/metrics-aggregator.d.ts +1 -1
- package/dist/types/debug/execution-tracer.d.ts +1 -1
- package/dist/types/debug/trace-inspector.d.ts +1 -1
- package/dist/types/dev/dev-server.d.ts +1 -1
- package/dist/types/dev/hot-reload.d.ts +1 -1
- package/dist/types/ecosystem/plugin-analytics-dashboard.d.ts +1 -1
- package/dist/types/ecosystem/plugin-certification.d.ts +1 -1
- package/dist/types/ecosystem/plugin-hub.d.ts +1 -1
- package/dist/types/enterprise/audit-logging.d.ts +1 -1
- package/dist/types/enterprise/data-governance.d.ts +1 -1
- package/dist/types/enterprise/multi-tenancy.d.ts +1 -1
- package/dist/types/enterprise/sso-integration.d.ts +1 -1
- package/dist/types/evaluate/pipeline-evaluator.d.ts +1 -1
- package/dist/types/graph/entity-extractor.d.ts +1 -1
- package/dist/types/graph/graph-retriever.d.ts +1 -1
- package/dist/types/graph/knowledge-graph.d.ts +1 -1
- package/dist/types/grounding/check-answer.d.ts +109 -1
- package/dist/types/grounding/llm-judge.d.ts +93 -0
- package/dist/types/grounding/provider-citations.d.ts +244 -0
- package/dist/types/guardrails/detectors.d.ts +88 -0
- package/dist/types/guardrails/evaluate-detectors.d.ts +61 -0
- package/dist/types/guardrails/guardrails-pipeline.d.ts +2 -9
- package/dist/types/guardrails/post-generation-guard.d.ts +33 -3
- package/dist/types/guardrails/pre-retrieval-guard.d.ts +29 -1
- package/dist/types/guardrails/retrieval-guard.d.ts +30 -1
- package/dist/types/mcp/mcp-server.d.ts +1 -1
- package/dist/types/observability/otel.d.ts +9 -3
- package/dist/types/observability/scores.d.ts +57 -0
- package/dist/types/reranker/cascade-reranker.d.ts +1 -1
- package/dist/types/reranker/embedding-reranker.d.ts +1 -1
- package/dist/types/reranker/scoring-reranker.d.ts +1 -1
- package/dist/types/retrieval/bm25-search.d.ts +1 -1
- package/dist/types/retrieval/hybrid-retriever.d.ts +1 -1
- package/dist/types/security/jwt-validator.d.ts +1 -1
- package/dist/types/security/plugin-sandbox.d.ts +1 -1
- package/dist/types/streaming/sse-adapter.d.ts +1 -1
- package/dist/types/streaming/stream-router.d.ts +1 -1
- package/dist/types/streaming/streaming-embedder.d.ts +1 -1
- package/dist/types/streaming/websocket-adapter.d.ts +1 -1
- package/dist/types/templates/scaffold.d.ts +1 -1
- package/dist/types/testing/contract-tests.d.ts +8 -4
- package/dist/types/utils/batch-processor.d.ts +1 -1
- package/dist/types/utils/connection-pool.d.ts +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One citation of a claim, normalised.
|
|
3
|
+
*/
|
|
4
|
+
export type ProviderCitation = {
|
|
5
|
+
/**
|
|
6
|
+
* - Your document id, the document title or index, a search result's source, a URL or an OpenAI file id
|
|
7
|
+
*/
|
|
8
|
+
sourceId: string;
|
|
9
|
+
title: string | null;
|
|
10
|
+
/**
|
|
11
|
+
* - Where the passage is; `type` is the provider's citation type
|
|
12
|
+
*/
|
|
13
|
+
location: any;
|
|
14
|
+
/**
|
|
15
|
+
* - The cited text; empty when the response doesn't carry it
|
|
16
|
+
*/
|
|
17
|
+
passages: string[];
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* A claim of the answer and the citations attached to it.
|
|
21
|
+
*/
|
|
22
|
+
export type CitedClaim = {
|
|
23
|
+
text: string;
|
|
24
|
+
/**
|
|
25
|
+
* - Offset in `answer`
|
|
26
|
+
*/
|
|
27
|
+
start: number;
|
|
28
|
+
end: number;
|
|
29
|
+
citations: ProviderCitation[];
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* What fromAnthropicCitations() and fromOpenAIFileCitations() return.
|
|
33
|
+
*/
|
|
34
|
+
export type ProviderCitations = {
|
|
35
|
+
provider: "anthropic" | "openai";
|
|
36
|
+
/**
|
|
37
|
+
* - The answer text
|
|
38
|
+
*/
|
|
39
|
+
answer: string;
|
|
40
|
+
claims: CitedClaim[];
|
|
41
|
+
/**
|
|
42
|
+
* - Citations of other kinds, by type
|
|
43
|
+
*/
|
|
44
|
+
skipped: {
|
|
45
|
+
[x: string]: number;
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* - OpenAI: whether the response had file search results
|
|
49
|
+
*/
|
|
50
|
+
results?: boolean;
|
|
51
|
+
/**
|
|
52
|
+
* - Claude: the message's stop_reason ("refusal" when Claude declined)
|
|
53
|
+
*/
|
|
54
|
+
stopReason?: string | null;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* A claim's verdict from verifyCitations().
|
|
58
|
+
*/
|
|
59
|
+
export type VerifiedClaim = {
|
|
60
|
+
text: string;
|
|
61
|
+
start: number;
|
|
62
|
+
end: number;
|
|
63
|
+
verdict: "supported" | "unsupported" | "unverifiable";
|
|
64
|
+
lexicalReason: "supported" | "low_overlap" | "number_mismatch" | "polarity_mismatch" | "unverifiable";
|
|
65
|
+
/**
|
|
66
|
+
* - Why, in words
|
|
67
|
+
*/
|
|
68
|
+
reason: string;
|
|
69
|
+
decidedBy: "lexical" | "judge";
|
|
70
|
+
/**
|
|
71
|
+
* - Support of the claim by all its cited passages together, 0..1
|
|
72
|
+
*/
|
|
73
|
+
score: number | null;
|
|
74
|
+
/**
|
|
75
|
+
* - Each citation's own support; null when it had no cited text
|
|
76
|
+
*/
|
|
77
|
+
citations: Array<{
|
|
78
|
+
sourceId: string;
|
|
79
|
+
title: string | null;
|
|
80
|
+
location: any;
|
|
81
|
+
score: number | null;
|
|
82
|
+
supports: boolean | null;
|
|
83
|
+
}>;
|
|
84
|
+
};
|
|
85
|
+
/**
|
|
86
|
+
* What verifyCitations() resolves to.
|
|
87
|
+
*/
|
|
88
|
+
export type CitationVerification = {
|
|
89
|
+
/**
|
|
90
|
+
* - Not a refusal, at least one claim, every claim supported and every citation checkable; uncited sentences don't count against it (see `uncited`)
|
|
91
|
+
*/
|
|
92
|
+
verified: boolean;
|
|
93
|
+
provider: "anthropic" | "openai";
|
|
94
|
+
claims: VerifiedClaim[];
|
|
95
|
+
/**
|
|
96
|
+
* - Sentences with content that no citation covers
|
|
97
|
+
*/
|
|
98
|
+
uncited: Array<{
|
|
99
|
+
text: string;
|
|
100
|
+
start: number;
|
|
101
|
+
end: number;
|
|
102
|
+
}>;
|
|
103
|
+
summary: {
|
|
104
|
+
claims: number;
|
|
105
|
+
supported: number;
|
|
106
|
+
unsupported: number;
|
|
107
|
+
unverifiable: number;
|
|
108
|
+
citations: number;
|
|
109
|
+
unverifiableCitations: number;
|
|
110
|
+
uncitedSentences: number;
|
|
111
|
+
};
|
|
112
|
+
skipped: {
|
|
113
|
+
[x: string]: number;
|
|
114
|
+
};
|
|
115
|
+
/**
|
|
116
|
+
* - Claude declined to answer (stop_reason "refusal"); never verified
|
|
117
|
+
*/
|
|
118
|
+
refused: boolean;
|
|
119
|
+
/**
|
|
120
|
+
* - Only with the `judge` option
|
|
121
|
+
*/
|
|
122
|
+
judge?: {
|
|
123
|
+
calls: number;
|
|
124
|
+
usage?: any;
|
|
125
|
+
error?: string;
|
|
126
|
+
};
|
|
127
|
+
};
|
|
128
|
+
/**
|
|
129
|
+
* Normalise the citations of a Claude Messages API response.
|
|
130
|
+
*
|
|
131
|
+
* Each text block that carries citations is a claim. Pass a stream's
|
|
132
|
+
* final message (`stream.finalMessage()`).
|
|
133
|
+
*
|
|
134
|
+
* @param {object} message - The Message, as the SDK returns it or as JSON
|
|
135
|
+
* @param {object} [options]
|
|
136
|
+
* @param {Array<object>} [options.documents] - The request's document blocks
|
|
137
|
+
* in order (`document_index` counts them), as your own `{ id }` records,
|
|
138
|
+
* so citations name your ids; otherwise a citation is named by the
|
|
139
|
+
* document's title, or its index
|
|
140
|
+
* @returns {ProviderCitations}
|
|
141
|
+
*/
|
|
142
|
+
export function fromAnthropicCitations(message: object, { documents }?: {
|
|
143
|
+
documents?: Array<object>;
|
|
144
|
+
}): ProviderCitations;
|
|
145
|
+
/**
|
|
146
|
+
* Normalise the file citations of an OpenAI Responses API response.
|
|
147
|
+
*
|
|
148
|
+
* A `file_citation` carries only `file_id`, `filename` and an `index` into
|
|
149
|
+
* the text, not the passage: the passages come from the file search
|
|
150
|
+
* results, which the request must ask for with
|
|
151
|
+
* `include: ["file_search_call.results"]`. Without them every citation is
|
|
152
|
+
* unverifiable. OpenAI gives a position, not a span, and puts it after the
|
|
153
|
+
* text it cites, a sentence or a paragraph (several citations can share
|
|
154
|
+
* it): each sentence from the previous citation's position up to this one
|
|
155
|
+
* is a claim carrying its citations. The boundary is inferred.
|
|
156
|
+
* Web and container citations are counted in `skipped`. Pass a stream's
|
|
157
|
+
* completed response.
|
|
158
|
+
*
|
|
159
|
+
* @param {object} response - The Response, as the SDK returns it or as JSON
|
|
160
|
+
* @returns {ProviderCitations}
|
|
161
|
+
*/
|
|
162
|
+
export function fromOpenAIFileCitations(response: object): ProviderCitations;
|
|
163
|
+
/**
|
|
164
|
+
* Check every claim against the passages it cites: together, so a claim
|
|
165
|
+
* that takes one fact from each of two passages is supported. Each
|
|
166
|
+
* citation's own score is reported too.
|
|
167
|
+
*
|
|
168
|
+
* @param {ProviderCitations} input - From fromAnthropicCitations() or fromOpenAIFileCitations()
|
|
169
|
+
* @param {object} [options]
|
|
170
|
+
* @param {{generate: Function}} [options.judge] - Opt-in LLM judge for claims
|
|
171
|
+
* that failed for lack of word overlap (paraphrases), as in checkAnswer();
|
|
172
|
+
* it may only name a passage the claim cites, and never overturns a
|
|
173
|
+
* number or negation mismatch. One call at most. Errors from
|
|
174
|
+
* `judge.generate` are not caught.
|
|
175
|
+
* @returns {Promise<CitationVerification>}
|
|
176
|
+
*/
|
|
177
|
+
export function verifyCitations(input: ProviderCitations, { judge }?: {
|
|
178
|
+
judge?: {
|
|
179
|
+
generate: Function;
|
|
180
|
+
};
|
|
181
|
+
}): Promise<CitationVerification>;
|
|
182
|
+
/**
|
|
183
|
+
* One citation of a claim, normalised.
|
|
184
|
+
*
|
|
185
|
+
* @typedef {Object} ProviderCitation
|
|
186
|
+
* @property {string} sourceId - Your document id, the document title or index, a search result's source, a URL or an OpenAI file id
|
|
187
|
+
* @property {string|null} title
|
|
188
|
+
* @property {Object} location - Where the passage is; `type` is the provider's citation type
|
|
189
|
+
* @property {string[]} passages - The cited text; empty when the response doesn't carry it
|
|
190
|
+
*/
|
|
191
|
+
/**
|
|
192
|
+
* A claim of the answer and the citations attached to it.
|
|
193
|
+
*
|
|
194
|
+
* @typedef {Object} CitedClaim
|
|
195
|
+
* @property {string} text
|
|
196
|
+
* @property {number} start - Offset in `answer`
|
|
197
|
+
* @property {number} end
|
|
198
|
+
* @property {ProviderCitation[]} citations
|
|
199
|
+
*/
|
|
200
|
+
/**
|
|
201
|
+
* What fromAnthropicCitations() and fromOpenAIFileCitations() return.
|
|
202
|
+
*
|
|
203
|
+
* @typedef {Object} ProviderCitations
|
|
204
|
+
* @property {'anthropic'|'openai'} provider
|
|
205
|
+
* @property {string} answer - The answer text
|
|
206
|
+
* @property {CitedClaim[]} claims
|
|
207
|
+
* @property {Object<string, number>} skipped - Citations of other kinds, by type
|
|
208
|
+
* @property {boolean} [results] - OpenAI: whether the response had file search results
|
|
209
|
+
* @property {string|null} [stopReason] - Claude: the message's stop_reason ("refusal" when Claude declined)
|
|
210
|
+
*/
|
|
211
|
+
/**
|
|
212
|
+
* A claim's verdict from verifyCitations().
|
|
213
|
+
*
|
|
214
|
+
* @typedef {Object} VerifiedClaim
|
|
215
|
+
* @property {string} text
|
|
216
|
+
* @property {number} start
|
|
217
|
+
* @property {number} end
|
|
218
|
+
* @property {'supported'|'unsupported'|'unverifiable'} verdict
|
|
219
|
+
* @property {'supported'|'low_overlap'|'number_mismatch'|'polarity_mismatch'|'unverifiable'} lexicalReason
|
|
220
|
+
* @property {string} reason - Why, in words
|
|
221
|
+
* @property {'lexical'|'judge'} decidedBy
|
|
222
|
+
* @property {number|null} score - Support of the claim by all its cited passages together, 0..1
|
|
223
|
+
* @property {Array<{sourceId: string, title: string|null, location: Object, score: number|null, supports: boolean|null}>} citations
|
|
224
|
+
* - Each citation's own support; null when it had no cited text
|
|
225
|
+
*/
|
|
226
|
+
/**
|
|
227
|
+
* What verifyCitations() resolves to.
|
|
228
|
+
*
|
|
229
|
+
* @typedef {Object} CitationVerification
|
|
230
|
+
* @property {boolean} verified - Not a refusal, at least one claim, every claim supported and every citation checkable; uncited sentences don't count against it (see `uncited`)
|
|
231
|
+
* @property {'anthropic'|'openai'} provider
|
|
232
|
+
* @property {VerifiedClaim[]} claims
|
|
233
|
+
* @property {Array<{text: string, start: number, end: number}>} uncited - Sentences with content that no citation covers
|
|
234
|
+
* @property {{claims: number, supported: number, unsupported: number, unverifiable: number, citations: number, unverifiableCitations: number, uncitedSentences: number}} summary
|
|
235
|
+
* @property {Object<string, number>} skipped
|
|
236
|
+
* @property {boolean} refused - Claude declined to answer (stop_reason "refusal"); never verified
|
|
237
|
+
* @property {{calls: number, usage?: Object, error?: string}} [judge] - Only with the `judge` option
|
|
238
|
+
*/
|
|
239
|
+
/** Sentences of `text` with their offsets (the splitSentences rule). */
|
|
240
|
+
export function sentencesWithOffsets(text: any): {
|
|
241
|
+
text: string;
|
|
242
|
+
start: number;
|
|
243
|
+
end: number;
|
|
244
|
+
}[];
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
export namespace DEFAULT_PII_PATTERNS {
|
|
2
|
+
let email: RegExp;
|
|
3
|
+
let phone: RegExp;
|
|
4
|
+
let ssn: RegExp;
|
|
5
|
+
let creditCard: RegExp;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* The built-in prompt-injection check as a detector.
|
|
9
|
+
*
|
|
10
|
+
* @param {object} [options]
|
|
11
|
+
* @param {RegExp[]} [options.patterns] - Extra patterns, added to the built-in rules
|
|
12
|
+
* @returns {{name: string, kind: 'injection', detect: (text: string) => {flagged: boolean, labels: string[], matches: Array<{label: string, text: string}>}}}
|
|
13
|
+
*/
|
|
14
|
+
export function patternInjectionDetector({ patterns }?: {
|
|
15
|
+
patterns?: RegExp[];
|
|
16
|
+
}): {
|
|
17
|
+
name: string;
|
|
18
|
+
kind: "injection";
|
|
19
|
+
detect: (text: string) => {
|
|
20
|
+
flagged: boolean;
|
|
21
|
+
labels: string[];
|
|
22
|
+
matches: Array<{
|
|
23
|
+
label: string;
|
|
24
|
+
text: string;
|
|
25
|
+
}>;
|
|
26
|
+
};
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* The built-in PII check as a detector: emails, phone numbers, US social
|
|
30
|
+
* security numbers and card numbers, by pattern.
|
|
31
|
+
*
|
|
32
|
+
* @param {object} [options]
|
|
33
|
+
* @param {Object<string, RegExp>} [options.patterns] - Extra patterns by label, merged with the defaults
|
|
34
|
+
* @returns {{name: string, kind: 'pii', detect: (text: string) => {flagged: boolean, labels: string[], matches: Array<{label: string, text: string, start: number, end: number}>}}}
|
|
35
|
+
*/
|
|
36
|
+
export function patternPIIDetector({ patterns }?: {
|
|
37
|
+
patterns?: {
|
|
38
|
+
[x: string]: RegExp;
|
|
39
|
+
};
|
|
40
|
+
}): {
|
|
41
|
+
name: string;
|
|
42
|
+
kind: "pii";
|
|
43
|
+
detect: (text: string) => {
|
|
44
|
+
flagged: boolean;
|
|
45
|
+
labels: string[];
|
|
46
|
+
matches: Array<{
|
|
47
|
+
label: string;
|
|
48
|
+
text: string;
|
|
49
|
+
start: number;
|
|
50
|
+
end: number;
|
|
51
|
+
}>;
|
|
52
|
+
};
|
|
53
|
+
};
|
|
54
|
+
/**
|
|
55
|
+
* Check a guard's `detectors` and `builtIn` options.
|
|
56
|
+
*
|
|
57
|
+
* @param {string} guard - Guard name, for messages
|
|
58
|
+
* @param {'injection'|'pii'} kind - The kind of detector this guard runs
|
|
59
|
+
* @param {object} options - The guard's options
|
|
60
|
+
* @returns {object[]} The detectors
|
|
61
|
+
*/
|
|
62
|
+
export function resolveDetectors(guard: string, kind: "injection" | "pii", options?: object): object[];
|
|
63
|
+
/**
|
|
64
|
+
* Run detectors synchronously. A detector that returns a promise throws a
|
|
65
|
+
* TypeError naming the async method to use, rather than being ignored (an
|
|
66
|
+
* ignored promise is a check that never fires).
|
|
67
|
+
*
|
|
68
|
+
* @returns {object[]} One detection per detector
|
|
69
|
+
*/
|
|
70
|
+
export function runDetectorsSync(detectors: any, text: any, context: any, { guard, asyncMethod }: {
|
|
71
|
+
guard: any;
|
|
72
|
+
asyncMethod: any;
|
|
73
|
+
}): object[];
|
|
74
|
+
/**
|
|
75
|
+
* Run detectors concurrently, sync or async.
|
|
76
|
+
*
|
|
77
|
+
* @returns {Promise<object[]>} One detection per detector
|
|
78
|
+
*/
|
|
79
|
+
export function runDetectorsAsync(detectors: any, text: any, context: any): Promise<object[]>;
|
|
80
|
+
/**
|
|
81
|
+
* The strings PII detectors found in a PostGenerationGuard result, for
|
|
82
|
+
* `PostGenerationGuard.redact(text, matches)`. The result keeps them in a
|
|
83
|
+
* non-enumerable `matchedText`, so they don't appear in its JSON.
|
|
84
|
+
*
|
|
85
|
+
* @param {object} result - A PostGenerationGuard check() or checkAsync() result
|
|
86
|
+
* @returns {string[]}
|
|
87
|
+
*/
|
|
88
|
+
export function detectedText(result: object): string[];
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run detectors over labelled cases and score them.
|
|
3
|
+
*
|
|
4
|
+
* @param {Array<object>} cases - Labelled cases (see the module comment)
|
|
5
|
+
* @param {Array<{name: string, kind: 'injection'|'pii', detect: Function}>} detectors
|
|
6
|
+
* @param {object} [options]
|
|
7
|
+
* @param {number} [options.concurrency=4] - Cases in flight per detector
|
|
8
|
+
* @param {string[]|Object<string, string[]>} [options.coveredLabels] - Span labels each PII detector is meant to find, for the headline span recall; an array applies to every PII detector
|
|
9
|
+
* @param {AbortSignal} [options.signal] - Stops the run
|
|
10
|
+
* @returns {Promise<object>} The report: `{ createdAt, cases, detectors: { [name]: metrics } }`
|
|
11
|
+
*/
|
|
12
|
+
export function evaluateDetectors(cases: Array<object>, detectors: Array<{
|
|
13
|
+
name: string;
|
|
14
|
+
kind: "injection" | "pii";
|
|
15
|
+
detect: Function;
|
|
16
|
+
}>, options?: {
|
|
17
|
+
concurrency?: number;
|
|
18
|
+
coveredLabels?: string[] | {
|
|
19
|
+
[x: string]: string[];
|
|
20
|
+
};
|
|
21
|
+
signal?: AbortSignal;
|
|
22
|
+
}): Promise<object>;
|
|
23
|
+
/**
|
|
24
|
+
* Check labelled cases and fill in their ids.
|
|
25
|
+
*
|
|
26
|
+
* @param {Array<object>} cases
|
|
27
|
+
* @returns {Array<object>} The cases, each with an `id`
|
|
28
|
+
*/
|
|
29
|
+
export function validateDetectorCases(cases: Array<object>): Array<object>;
|
|
30
|
+
/**
|
|
31
|
+
* Check minimums such as `{ 'pattern-injection.recall': 0.5 }` against a
|
|
32
|
+
* report. A key is `<detector>.<metric>`, where metric is precision,
|
|
33
|
+
* recall, f1 or accuracy, or spans.precision / spans.recall.
|
|
34
|
+
*
|
|
35
|
+
* @param {object} report - An evaluateDetectors() report
|
|
36
|
+
* @param {Object<string, number>} [min]
|
|
37
|
+
* @returns {{passed: boolean, reasons: string[]}}
|
|
38
|
+
*/
|
|
39
|
+
export function detectorVerdict(report: object, min?: {
|
|
40
|
+
[x: string]: number;
|
|
41
|
+
}): {
|
|
42
|
+
passed: boolean;
|
|
43
|
+
reasons: string[];
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* Render an evaluateDetectors() report as Markdown or a standalone HTML page.
|
|
47
|
+
*
|
|
48
|
+
* @param {object} report
|
|
49
|
+
* @param {object} [options]
|
|
50
|
+
* @param {'markdown'|'html'} [options.format='markdown']
|
|
51
|
+
* @param {string} [options.title='Guardrail detectors']
|
|
52
|
+
* @param {Object<string, number>} [options.min] - Minimums, as for detectorVerdict()
|
|
53
|
+
* @returns {string}
|
|
54
|
+
*/
|
|
55
|
+
export function renderDetectorReport(report: object, options?: {
|
|
56
|
+
format?: "markdown" | "html";
|
|
57
|
+
title?: string;
|
|
58
|
+
min?: {
|
|
59
|
+
[x: string]: number;
|
|
60
|
+
};
|
|
61
|
+
}): string;
|
|
@@ -1,12 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
* Pipeline wrapper that applies all three guardrail layers (pre-retrieval,
|
|
3
|
-
* retrieval, and post-generation) around an existing RAG pipeline. In strict
|
|
4
|
-
* mode, any guard failure aborts the pipeline; otherwise issues are collected
|
|
5
|
-
* as warnings and execution continues.
|
|
6
|
-
*
|
|
7
|
-
* @extends EventEmitter
|
|
8
|
-
*/
|
|
9
|
-
export class GuardrailsPipeline extends EventEmitter<[never]> {
|
|
1
|
+
export class GuardrailsPipeline extends EventEmitter {
|
|
10
2
|
/**
|
|
11
3
|
* @param {object} pipeline - The underlying RAG pipeline (must have a .run() method)
|
|
12
4
|
* @param {object} [options] - Override default configuration
|
|
@@ -47,6 +39,7 @@ export class GuardrailsPipeline extends EventEmitter<[never]> {
|
|
|
47
39
|
* scrub every string in them with the same PII patterns as the answer.
|
|
48
40
|
*
|
|
49
41
|
* @param {*} value - Object, array or string to redact in place
|
|
42
|
+
* @param {string[]} [matches=[]] - Strings found by PII detectors, redacted too
|
|
50
43
|
* @returns {*} The redacted value
|
|
51
44
|
* @private
|
|
52
45
|
*/
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* @extends EventEmitter
|
|
7
7
|
*/
|
|
8
|
-
export class PostGenerationGuard extends EventEmitter
|
|
8
|
+
export class PostGenerationGuard extends EventEmitter {
|
|
9
9
|
/**
|
|
10
10
|
* @param {object} [options] - Override default configuration
|
|
11
11
|
* @param {boolean} [options.enablePIIDetection=true] - Whether to detect PII in output
|
|
@@ -14,6 +14,9 @@ export class PostGenerationGuard extends EventEmitter<[never]> {
|
|
|
14
14
|
* @param {number} [options.maxResponseLength=50000] - Maximum allowed response length
|
|
15
15
|
* @param {number} [options.minResponseLength=0] - Minimum response length
|
|
16
16
|
* @param {object} [options.piiPatterns] - Custom PII regex patterns (merged with defaults)
|
|
17
|
+
* @param {Array<object>} [options.detectors=[]] - PII detectors to run on the output too (see ./detectors); their matches are redacted. Async ones need checkAsync()
|
|
18
|
+
* @param {boolean} [options.builtIn=true] - false: use only `detectors`, not the built-in PII patterns
|
|
19
|
+
* @param {'block'|'skip'} [options.onDetectorError='block'] - A detector that throws makes the output unsafe ('block'; what it would have found can't be redacted) or is ignored ('skip')
|
|
17
20
|
*/
|
|
18
21
|
constructor(options?: {
|
|
19
22
|
enablePIIDetection?: boolean;
|
|
@@ -22,7 +25,12 @@ export class PostGenerationGuard extends EventEmitter<[never]> {
|
|
|
22
25
|
maxResponseLength?: number;
|
|
23
26
|
minResponseLength?: number;
|
|
24
27
|
piiPatterns?: object;
|
|
28
|
+
detectors?: Array<object>;
|
|
29
|
+
builtIn?: boolean;
|
|
30
|
+
onDetectorError?: "block" | "skip";
|
|
25
31
|
});
|
|
32
|
+
/** @private */
|
|
33
|
+
private _detectors;
|
|
26
34
|
config: any;
|
|
27
35
|
/**
|
|
28
36
|
* Check generated output for safety issues, PII, and groundedness.
|
|
@@ -43,15 +51,37 @@ export class PostGenerationGuard extends EventEmitter<[never]> {
|
|
|
43
51
|
sanitizedOutput: string;
|
|
44
52
|
groundednessScore?: number;
|
|
45
53
|
};
|
|
54
|
+
/**
|
|
55
|
+
* check() for async detectors, such as a model or a PII service.
|
|
56
|
+
* Detectors run concurrently.
|
|
57
|
+
*
|
|
58
|
+
* @param {string} output
|
|
59
|
+
* @param {Array<object>} [retrievedDocs=[]]
|
|
60
|
+
* @param {object} [options]
|
|
61
|
+
* @param {AbortSignal} [options.signal] - Passed to each detector in its context
|
|
62
|
+
* @returns {Promise<ReturnType<PostGenerationGuard['check']>>}
|
|
63
|
+
*/
|
|
64
|
+
checkAsync(output: string, retrievedDocs?: Array<object>, { signal }?: {
|
|
65
|
+
signal?: AbortSignal;
|
|
66
|
+
}): Promise<ReturnType<PostGenerationGuard["check"]>>;
|
|
67
|
+
/** @private */
|
|
68
|
+
private _runDetectors;
|
|
69
|
+
/** @private 1. The output must be a string. */
|
|
70
|
+
private _invalid;
|
|
71
|
+
/** @private */
|
|
72
|
+
private _check;
|
|
46
73
|
/**
|
|
47
74
|
* Replace PII matches in arbitrary text with [REDACTED], using the same
|
|
48
75
|
* patterns as check(). Used to scrub text derived from the answer, such as
|
|
49
|
-
* citation sentences.
|
|
76
|
+
* citation sentences. Detectors don't run here (they may be async):
|
|
77
|
+
* pass the strings a check found, `detectedText(result)`, as `matches`
|
|
78
|
+
* so they are removed too.
|
|
50
79
|
*
|
|
51
80
|
* @param {string} text
|
|
81
|
+
* @param {string[]} [matches=[]] - Exact strings to redact as well
|
|
52
82
|
* @returns {string}
|
|
53
83
|
*/
|
|
54
|
-
redact(text: string): string;
|
|
84
|
+
redact(text: string, matches?: string[]): string;
|
|
55
85
|
}
|
|
56
86
|
/**
|
|
57
87
|
* Default configuration for the PostGenerationGuard.
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* @extends EventEmitter
|
|
7
7
|
*/
|
|
8
|
-
export class PreRetrievalGuard extends EventEmitter
|
|
8
|
+
export class PreRetrievalGuard extends EventEmitter {
|
|
9
9
|
/**
|
|
10
10
|
* @param {object} [options] - Override default configuration
|
|
11
11
|
* @param {number} [options.maxQueryLength=10000] - Maximum allowed query length
|
|
@@ -14,6 +14,9 @@ export class PreRetrievalGuard extends EventEmitter<[never]> {
|
|
|
14
14
|
* @param {string[]} [options.blockedTopics=[]] - Topics to reject
|
|
15
15
|
* @param {string[]} [options.allowedTopics=[]] - If non-empty, only these topics are permitted
|
|
16
16
|
* @param {RegExp[]} [options.injectionPatterns] - Additional injection patterns (merged with defaults)
|
|
17
|
+
* @param {Array<object>} [options.detectors=[]] - Injection detectors to run on the query as well (see ./detectors); async ones need checkAsync()
|
|
18
|
+
* @param {boolean} [options.builtIn=true] - false: use only `detectors`, not the built-in patterns
|
|
19
|
+
* @param {'block'|'skip'} [options.onDetectorError='block'] - A detector that throws makes the query unsafe ('block') or is ignored ('skip')
|
|
17
20
|
*/
|
|
18
21
|
constructor(options?: {
|
|
19
22
|
maxQueryLength?: number;
|
|
@@ -22,7 +25,12 @@ export class PreRetrievalGuard extends EventEmitter<[never]> {
|
|
|
22
25
|
blockedTopics?: string[];
|
|
23
26
|
allowedTopics?: string[];
|
|
24
27
|
injectionPatterns?: RegExp[];
|
|
28
|
+
detectors?: Array<object>;
|
|
29
|
+
builtIn?: boolean;
|
|
30
|
+
onDetectorError?: "block" | "skip";
|
|
25
31
|
});
|
|
32
|
+
/** @private */
|
|
33
|
+
private _detectors;
|
|
26
34
|
config: any;
|
|
27
35
|
/** @private Extra patterns supplied by the caller */
|
|
28
36
|
private _customPatterns;
|
|
@@ -55,6 +63,26 @@ export class PreRetrievalGuard extends EventEmitter<[never]> {
|
|
|
55
63
|
}>;
|
|
56
64
|
};
|
|
57
65
|
};
|
|
66
|
+
/**
|
|
67
|
+
* check() for async detectors, such as a classifier model or a service.
|
|
68
|
+
* Detectors run concurrently.
|
|
69
|
+
*
|
|
70
|
+
* @param {string} query
|
|
71
|
+
* @param {object} [options]
|
|
72
|
+
* @param {AbortSignal} [options.signal] - Passed to each detector in its context
|
|
73
|
+
* @returns {Promise<ReturnType<PreRetrievalGuard['check']>>}
|
|
74
|
+
*/
|
|
75
|
+
checkAsync(query: string, { signal }?: {
|
|
76
|
+
signal?: AbortSignal;
|
|
77
|
+
}): Promise<ReturnType<PreRetrievalGuard["check"]>>;
|
|
78
|
+
/** @private */
|
|
79
|
+
private _runDetectors;
|
|
80
|
+
/** @private */
|
|
81
|
+
private _emit;
|
|
82
|
+
/** @private Validation and the built-in checks, before any detector runs. */
|
|
83
|
+
private _prepare;
|
|
84
|
+
/** @private Merge the detectors' findings and emit the result. */
|
|
85
|
+
private _finish;
|
|
58
86
|
}
|
|
59
87
|
/**
|
|
60
88
|
* Default configuration for the PreRetrievalGuard.
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* @extends EventEmitter
|
|
7
7
|
*/
|
|
8
|
-
export class RetrievalGuard extends EventEmitter
|
|
8
|
+
export class RetrievalGuard extends EventEmitter {
|
|
9
9
|
/**
|
|
10
10
|
* @param {object} [options] - Override default configuration
|
|
11
11
|
* @param {number} [options.minRelevanceScore=0] - Minimum relevance score threshold
|
|
@@ -14,6 +14,9 @@ export class RetrievalGuard extends EventEmitter<[never]> {
|
|
|
14
14
|
* @param {boolean} [options.detectInjection=false] - Scan document text for prompt injection
|
|
15
15
|
* @param {'remove'|'flag'} [options.injectionAction='remove'] - Remove flagged documents or keep and mark them
|
|
16
16
|
* @param {RegExp[]} [options.injectionPatterns=[]] - Extra injection patterns for the scan
|
|
17
|
+
* @param {Array<object>} [options.detectors=[]] - Injection detectors to run on each document's text too, when detectInjection is on (see ./detectors); async ones need filterAsync(), which calls them once per document, concurrently
|
|
18
|
+
* @param {boolean} [options.builtIn=true] - false: use only `detectors` for the injection scan
|
|
19
|
+
* @param {'block'|'skip'} [options.onDetectorError='block'] - A detector that throws counts as a finding ('block': the document is removed or flagged) or is ignored ('skip')
|
|
17
20
|
*/
|
|
18
21
|
constructor(options?: {
|
|
19
22
|
minRelevanceScore?: number;
|
|
@@ -22,8 +25,13 @@ export class RetrievalGuard extends EventEmitter<[never]> {
|
|
|
22
25
|
detectInjection?: boolean;
|
|
23
26
|
injectionAction?: "remove" | "flag";
|
|
24
27
|
injectionPatterns?: RegExp[];
|
|
28
|
+
detectors?: Array<object>;
|
|
29
|
+
builtIn?: boolean;
|
|
30
|
+
onDetectorError?: "block" | "skip";
|
|
25
31
|
});
|
|
26
32
|
config: any;
|
|
33
|
+
/** @private */
|
|
34
|
+
private _detectors;
|
|
27
35
|
/**
|
|
28
36
|
* Filter retrieval results based on configured guards.
|
|
29
37
|
*
|
|
@@ -48,6 +56,27 @@ export class RetrievalGuard extends EventEmitter<[never]> {
|
|
|
48
56
|
};
|
|
49
57
|
};
|
|
50
58
|
};
|
|
59
|
+
/**
|
|
60
|
+
* filter() for async detectors, such as a classifier model or a service. Each
|
|
61
|
+
* document that reaches the injection scan is checked by every detector,
|
|
62
|
+
* all concurrently.
|
|
63
|
+
*
|
|
64
|
+
* @param {Array<object>} results
|
|
65
|
+
* @param {object} [context={}]
|
|
66
|
+
* @param {object} [options]
|
|
67
|
+
* @param {AbortSignal} [options.signal] - Passed to each detector in its context
|
|
68
|
+
* @returns {Promise<ReturnType<RetrievalGuard['filter']>>}
|
|
69
|
+
*/
|
|
70
|
+
filterAsync(results: Array<object>, context?: object, { signal }?: {
|
|
71
|
+
signal?: AbortSignal;
|
|
72
|
+
}): Promise<ReturnType<RetrievalGuard["filter"]>>;
|
|
73
|
+
/**
|
|
74
|
+
* @private Relevance, freshness and ACL, then which documents go on to
|
|
75
|
+
* the injection scan.
|
|
76
|
+
*/
|
|
77
|
+
private _screen;
|
|
78
|
+
/** @private 4. Indirect prompt injection in document text (opt-in). */
|
|
79
|
+
private _finish;
|
|
51
80
|
}
|
|
52
81
|
/**
|
|
53
82
|
* Default configuration for the RetrievalGuard.
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* const tools = server.getToolDefinitions();
|
|
10
10
|
* const result = await server.handleToolUse('my_tool', { query: 'What is X?' });
|
|
11
11
|
*/
|
|
12
|
-
export class MCPServer extends EventEmitter
|
|
12
|
+
export class MCPServer extends EventEmitter {
|
|
13
13
|
/**
|
|
14
14
|
* Convenience factory: create an MCPServer with a single pipeline registered.
|
|
15
15
|
*
|
|
@@ -1,16 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Normalise the `telemetry` option of createRagPipeline. `api` is the
|
|
3
|
-
* @opentelemetry/api module, for builds that cannot require() it (/edge)
|
|
4
|
-
*
|
|
5
|
-
*
|
|
3
|
+
* @opentelemetry/api module, for builds that cannot require() it (/edge);
|
|
4
|
+
* `logs` is @opentelemetry/api-logs, for evaluation events with an SDK that
|
|
5
|
+
* registers its LoggerProvider there, and `logger` a Logger to emit them with.
|
|
6
|
+
* @param {boolean|{tracer?: object, api?: object, logs?: object, logger?: object, recordContent?: boolean}} option
|
|
7
|
+
* @returns {null|{api: object, tracer: object, logger: object|null, runKey: symbol, recordContent: boolean}}
|
|
6
8
|
*/
|
|
7
9
|
export function createTelemetry(option: boolean | {
|
|
8
10
|
tracer?: object;
|
|
9
11
|
api?: object;
|
|
12
|
+
logs?: object;
|
|
13
|
+
logger?: object;
|
|
10
14
|
recordContent?: boolean;
|
|
11
15
|
}): null | {
|
|
12
16
|
api: object;
|
|
13
17
|
tracer: object;
|
|
18
|
+
logger: object | null;
|
|
19
|
+
runKey: symbol;
|
|
14
20
|
recordContent: boolean;
|
|
15
21
|
};
|
|
16
22
|
/** Wrap a component so its stage methods run in spans. */
|