@devilsdev/rag-core 3.4.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/all.d.ts +58 -0
- package/dist/index.cjs +275 -15
- package/dist/index.d.ts +1 -1
- package/dist/index.mjs +1 -1
- package/dist/types/adapters/mastra.d.ts +6 -2
- package/dist/types/agentic/agentic-pipeline.d.ts +1 -1
- package/dist/types/agentic/iterative-retriever.d.ts +1 -1
- package/dist/types/agentic/query-planner.d.ts +1 -1
- package/dist/types/agentic/self-critique.d.ts +1 -1
- package/dist/types/ai/federation/federated-learning-coordinator.d.ts +1 -1
- package/dist/types/ai/training/model-training-orchestrator.d.ts +1 -1
- package/dist/types/chunking/chunking-engine.d.ts +1 -1
- package/dist/types/citation/citation-tracker.d.ts +1 -1
- package/dist/types/connectors/anthropic-connector.d.ts +35 -1
- package/dist/types/connectors/base-connector.d.ts +1 -1
- package/dist/types/connectors/connector-utils.d.ts +20 -0
- package/dist/types/cost/cost-calculator.d.ts +1 -1
- package/dist/types/cost/cost-tracker.d.ts +1 -1
- package/dist/types/cost/token-budget.d.ts +1 -1
- package/dist/types/dashboard/dashboard-generator.d.ts +1 -1
- package/dist/types/dashboard/metrics-aggregator.d.ts +1 -1
- package/dist/types/debug/execution-tracer.d.ts +1 -1
- package/dist/types/debug/trace-inspector.d.ts +1 -1
- package/dist/types/dev/dev-server.d.ts +1 -1
- package/dist/types/dev/hot-reload.d.ts +1 -1
- package/dist/types/ecosystem/plugin-analytics-dashboard.d.ts +1 -1
- package/dist/types/ecosystem/plugin-certification.d.ts +1 -1
- package/dist/types/ecosystem/plugin-hub.d.ts +1 -1
- package/dist/types/enterprise/audit-logging.d.ts +1 -1
- package/dist/types/enterprise/data-governance.d.ts +1 -1
- package/dist/types/enterprise/multi-tenancy.d.ts +1 -1
- package/dist/types/enterprise/sso-integration.d.ts +1 -1
- package/dist/types/evaluate/pipeline-evaluator.d.ts +1 -1
- package/dist/types/graph/entity-extractor.d.ts +1 -1
- package/dist/types/graph/graph-retriever.d.ts +1 -1
- package/dist/types/graph/knowledge-graph.d.ts +1 -1
- package/dist/types/grounding/llm-judge.d.ts +10 -3
- package/dist/types/grounding/provider-citations.d.ts +244 -0
- package/dist/types/guardrails/detectors.d.ts +88 -0
- package/dist/types/guardrails/evaluate-detectors.d.ts +61 -0
- package/dist/types/guardrails/guardrails-pipeline.d.ts +2 -9
- package/dist/types/guardrails/post-generation-guard.d.ts +33 -3
- package/dist/types/guardrails/pre-retrieval-guard.d.ts +29 -1
- package/dist/types/guardrails/retrieval-guard.d.ts +30 -1
- package/dist/types/mcp/mcp-server.d.ts +1 -1
- package/dist/types/observability/otel.d.ts +9 -3
- package/dist/types/observability/scores.d.ts +57 -0
- package/dist/types/reranker/cascade-reranker.d.ts +1 -1
- package/dist/types/reranker/embedding-reranker.d.ts +1 -1
- package/dist/types/reranker/scoring-reranker.d.ts +1 -1
- package/dist/types/retrieval/bm25-search.d.ts +1 -1
- package/dist/types/retrieval/hybrid-retriever.d.ts +1 -1
- package/dist/types/security/jwt-validator.d.ts +1 -1
- package/dist/types/security/plugin-sandbox.d.ts +1 -1
- package/dist/types/streaming/sse-adapter.d.ts +1 -1
- package/dist/types/streaming/stream-router.d.ts +1 -1
- package/dist/types/streaming/streaming-embedder.d.ts +1 -1
- package/dist/types/streaming/websocket-adapter.d.ts +1 -1
- package/dist/types/templates/scaffold.d.ts +1 -1
- package/dist/types/testing/contract-tests.d.ts +8 -4
- package/dist/types/utils/batch-processor.d.ts +1 -1
- package/dist/types/utils/connection-pool.d.ts +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
export namespace DEFAULT_PII_PATTERNS {
|
|
2
|
+
let email: RegExp;
|
|
3
|
+
let phone: RegExp;
|
|
4
|
+
let ssn: RegExp;
|
|
5
|
+
let creditCard: RegExp;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* The built-in prompt-injection check as a detector.
|
|
9
|
+
*
|
|
10
|
+
* @param {object} [options]
|
|
11
|
+
* @param {RegExp[]} [options.patterns] - Extra patterns, added to the built-in rules
|
|
12
|
+
* @returns {{name: string, kind: 'injection', detect: (text: string) => {flagged: boolean, labels: string[], matches: Array<{label: string, text: string}>}}}
|
|
13
|
+
*/
|
|
14
|
+
export function patternInjectionDetector({ patterns }?: {
|
|
15
|
+
patterns?: RegExp[];
|
|
16
|
+
}): {
|
|
17
|
+
name: string;
|
|
18
|
+
kind: "injection";
|
|
19
|
+
detect: (text: string) => {
|
|
20
|
+
flagged: boolean;
|
|
21
|
+
labels: string[];
|
|
22
|
+
matches: Array<{
|
|
23
|
+
label: string;
|
|
24
|
+
text: string;
|
|
25
|
+
}>;
|
|
26
|
+
};
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* The built-in PII check as a detector: emails, phone numbers, US social
|
|
30
|
+
* security numbers and card numbers, by pattern.
|
|
31
|
+
*
|
|
32
|
+
* @param {object} [options]
|
|
33
|
+
* @param {Object<string, RegExp>} [options.patterns] - Extra patterns by label, merged with the defaults
|
|
34
|
+
* @returns {{name: string, kind: 'pii', detect: (text: string) => {flagged: boolean, labels: string[], matches: Array<{label: string, text: string, start: number, end: number}>}}}
|
|
35
|
+
*/
|
|
36
|
+
export function patternPIIDetector({ patterns }?: {
|
|
37
|
+
patterns?: {
|
|
38
|
+
[x: string]: RegExp;
|
|
39
|
+
};
|
|
40
|
+
}): {
|
|
41
|
+
name: string;
|
|
42
|
+
kind: "pii";
|
|
43
|
+
detect: (text: string) => {
|
|
44
|
+
flagged: boolean;
|
|
45
|
+
labels: string[];
|
|
46
|
+
matches: Array<{
|
|
47
|
+
label: string;
|
|
48
|
+
text: string;
|
|
49
|
+
start: number;
|
|
50
|
+
end: number;
|
|
51
|
+
}>;
|
|
52
|
+
};
|
|
53
|
+
};
|
|
54
|
+
/**
|
|
55
|
+
* Check a guard's `detectors` and `builtIn` options.
|
|
56
|
+
*
|
|
57
|
+
* @param {string} guard - Guard name, for messages
|
|
58
|
+
* @param {'injection'|'pii'} kind - The kind of detector this guard runs
|
|
59
|
+
* @param {object} options - The guard's options
|
|
60
|
+
* @returns {object[]} The detectors
|
|
61
|
+
*/
|
|
62
|
+
export function resolveDetectors(guard: string, kind: "injection" | "pii", options?: object): object[];
|
|
63
|
+
/**
|
|
64
|
+
* Run detectors synchronously. A detector that returns a promise throws a
|
|
65
|
+
* TypeError naming the async method to use, rather than being ignored (an
|
|
66
|
+
* ignored promise is a check that never fires).
|
|
67
|
+
*
|
|
68
|
+
* @returns {object[]} One detection per detector
|
|
69
|
+
*/
|
|
70
|
+
export function runDetectorsSync(detectors: any, text: any, context: any, { guard, asyncMethod }: {
|
|
71
|
+
guard: any;
|
|
72
|
+
asyncMethod: any;
|
|
73
|
+
}): object[];
|
|
74
|
+
/**
|
|
75
|
+
* Run detectors concurrently, sync or async.
|
|
76
|
+
*
|
|
77
|
+
* @returns {Promise<object[]>} One detection per detector
|
|
78
|
+
*/
|
|
79
|
+
export function runDetectorsAsync(detectors: any, text: any, context: any): Promise<object[]>;
|
|
80
|
+
/**
|
|
81
|
+
* The strings PII detectors found in a PostGenerationGuard result, for
|
|
82
|
+
* `PostGenerationGuard.redact(text, matches)`. The result keeps them in a
|
|
83
|
+
* non-enumerable `matchedText`, so they don't appear in its JSON.
|
|
84
|
+
*
|
|
85
|
+
* @param {object} result - A PostGenerationGuard check() or checkAsync() result
|
|
86
|
+
* @returns {string[]}
|
|
87
|
+
*/
|
|
88
|
+
export function detectedText(result: object): string[];
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run detectors over labelled cases and score them.
|
|
3
|
+
*
|
|
4
|
+
* @param {Array<object>} cases - Labelled cases (see the module comment)
|
|
5
|
+
* @param {Array<{name: string, kind: 'injection'|'pii', detect: Function}>} detectors
|
|
6
|
+
* @param {object} [options]
|
|
7
|
+
* @param {number} [options.concurrency=4] - Cases in flight per detector
|
|
8
|
+
* @param {string[]|Object<string, string[]>} [options.coveredLabels] - Span labels each PII detector is meant to find, for the headline span recall; an array applies to every PII detector
|
|
9
|
+
* @param {AbortSignal} [options.signal] - Stops the run
|
|
10
|
+
* @returns {Promise<object>} The report: `{ createdAt, cases, detectors: { [name]: metrics } }`
|
|
11
|
+
*/
|
|
12
|
+
export function evaluateDetectors(cases: Array<object>, detectors: Array<{
|
|
13
|
+
name: string;
|
|
14
|
+
kind: "injection" | "pii";
|
|
15
|
+
detect: Function;
|
|
16
|
+
}>, options?: {
|
|
17
|
+
concurrency?: number;
|
|
18
|
+
coveredLabels?: string[] | {
|
|
19
|
+
[x: string]: string[];
|
|
20
|
+
};
|
|
21
|
+
signal?: AbortSignal;
|
|
22
|
+
}): Promise<object>;
|
|
23
|
+
/**
|
|
24
|
+
* Check labelled cases and fill in their ids.
|
|
25
|
+
*
|
|
26
|
+
* @param {Array<object>} cases
|
|
27
|
+
* @returns {Array<object>} The cases, each with an `id`
|
|
28
|
+
*/
|
|
29
|
+
export function validateDetectorCases(cases: Array<object>): Array<object>;
|
|
30
|
+
/**
|
|
31
|
+
* Check minimums such as `{ 'pattern-injection.recall': 0.5 }` against a
|
|
32
|
+
* report. A key is `<detector>.<metric>`, where metric is precision,
|
|
33
|
+
* recall, f1 or accuracy, or spans.precision / spans.recall.
|
|
34
|
+
*
|
|
35
|
+
* @param {object} report - An evaluateDetectors() report
|
|
36
|
+
* @param {Object<string, number>} [min]
|
|
37
|
+
* @returns {{passed: boolean, reasons: string[]}}
|
|
38
|
+
*/
|
|
39
|
+
export function detectorVerdict(report: object, min?: {
|
|
40
|
+
[x: string]: number;
|
|
41
|
+
}): {
|
|
42
|
+
passed: boolean;
|
|
43
|
+
reasons: string[];
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* Render an evaluateDetectors() report as Markdown or a standalone HTML page.
|
|
47
|
+
*
|
|
48
|
+
* @param {object} report
|
|
49
|
+
* @param {object} [options]
|
|
50
|
+
* @param {'markdown'|'html'} [options.format='markdown']
|
|
51
|
+
* @param {string} [options.title='Guardrail detectors']
|
|
52
|
+
* @param {Object<string, number>} [options.min] - Minimums, as for detectorVerdict()
|
|
53
|
+
* @returns {string}
|
|
54
|
+
*/
|
|
55
|
+
export function renderDetectorReport(report: object, options?: {
|
|
56
|
+
format?: "markdown" | "html";
|
|
57
|
+
title?: string;
|
|
58
|
+
min?: {
|
|
59
|
+
[x: string]: number;
|
|
60
|
+
};
|
|
61
|
+
}): string;
|
|
@@ -1,12 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
* Pipeline wrapper that applies all three guardrail layers (pre-retrieval,
|
|
3
|
-
* retrieval, and post-generation) around an existing RAG pipeline. In strict
|
|
4
|
-
* mode, any guard failure aborts the pipeline; otherwise issues are collected
|
|
5
|
-
* as warnings and execution continues.
|
|
6
|
-
*
|
|
7
|
-
* @extends EventEmitter
|
|
8
|
-
*/
|
|
9
|
-
export class GuardrailsPipeline extends EventEmitter<[never]> {
|
|
1
|
+
export class GuardrailsPipeline extends EventEmitter {
|
|
10
2
|
/**
|
|
11
3
|
* @param {object} pipeline - The underlying RAG pipeline (must have a .run() method)
|
|
12
4
|
* @param {object} [options] - Override default configuration
|
|
@@ -47,6 +39,7 @@ export class GuardrailsPipeline extends EventEmitter<[never]> {
|
|
|
47
39
|
* scrub every string in them with the same PII patterns as the answer.
|
|
48
40
|
*
|
|
49
41
|
* @param {*} value - Object, array or string to redact in place
|
|
42
|
+
* @param {string[]} [matches=[]] - Strings found by PII detectors, redacted too
|
|
50
43
|
* @returns {*} The redacted value
|
|
51
44
|
* @private
|
|
52
45
|
*/
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* @extends EventEmitter
|
|
7
7
|
*/
|
|
8
|
-
export class PostGenerationGuard extends EventEmitter
|
|
8
|
+
export class PostGenerationGuard extends EventEmitter {
|
|
9
9
|
/**
|
|
10
10
|
* @param {object} [options] - Override default configuration
|
|
11
11
|
* @param {boolean} [options.enablePIIDetection=true] - Whether to detect PII in output
|
|
@@ -14,6 +14,9 @@ export class PostGenerationGuard extends EventEmitter<[never]> {
|
|
|
14
14
|
* @param {number} [options.maxResponseLength=50000] - Maximum allowed response length
|
|
15
15
|
* @param {number} [options.minResponseLength=0] - Minimum response length
|
|
16
16
|
* @param {object} [options.piiPatterns] - Custom PII regex patterns (merged with defaults)
|
|
17
|
+
* @param {Array<object>} [options.detectors=[]] - PII detectors to run on the output too (see ./detectors); their matches are redacted. Async ones need checkAsync()
|
|
18
|
+
* @param {boolean} [options.builtIn=true] - false: use only `detectors`, not the built-in PII patterns
|
|
19
|
+
* @param {'block'|'skip'} [options.onDetectorError='block'] - A detector that throws makes the output unsafe ('block'; what it would have found can't be redacted) or is ignored ('skip')
|
|
17
20
|
*/
|
|
18
21
|
constructor(options?: {
|
|
19
22
|
enablePIIDetection?: boolean;
|
|
@@ -22,7 +25,12 @@ export class PostGenerationGuard extends EventEmitter<[never]> {
|
|
|
22
25
|
maxResponseLength?: number;
|
|
23
26
|
minResponseLength?: number;
|
|
24
27
|
piiPatterns?: object;
|
|
28
|
+
detectors?: Array<object>;
|
|
29
|
+
builtIn?: boolean;
|
|
30
|
+
onDetectorError?: "block" | "skip";
|
|
25
31
|
});
|
|
32
|
+
/** @private */
|
|
33
|
+
private _detectors;
|
|
26
34
|
config: any;
|
|
27
35
|
/**
|
|
28
36
|
* Check generated output for safety issues, PII, and groundedness.
|
|
@@ -43,15 +51,37 @@ export class PostGenerationGuard extends EventEmitter<[never]> {
|
|
|
43
51
|
sanitizedOutput: string;
|
|
44
52
|
groundednessScore?: number;
|
|
45
53
|
};
|
|
54
|
+
/**
|
|
55
|
+
* check() for async detectors, such as a model or a PII service.
|
|
56
|
+
* Detectors run concurrently.
|
|
57
|
+
*
|
|
58
|
+
* @param {string} output
|
|
59
|
+
* @param {Array<object>} [retrievedDocs=[]]
|
|
60
|
+
* @param {object} [options]
|
|
61
|
+
* @param {AbortSignal} [options.signal] - Passed to each detector in its context
|
|
62
|
+
* @returns {Promise<ReturnType<PostGenerationGuard['check']>>}
|
|
63
|
+
*/
|
|
64
|
+
checkAsync(output: string, retrievedDocs?: Array<object>, { signal }?: {
|
|
65
|
+
signal?: AbortSignal;
|
|
66
|
+
}): Promise<ReturnType<PostGenerationGuard["check"]>>;
|
|
67
|
+
/** @private */
|
|
68
|
+
private _runDetectors;
|
|
69
|
+
/** @private 1. The output must be a string. */
|
|
70
|
+
private _invalid;
|
|
71
|
+
/** @private */
|
|
72
|
+
private _check;
|
|
46
73
|
/**
|
|
47
74
|
* Replace PII matches in arbitrary text with [REDACTED], using the same
|
|
48
75
|
* patterns as check(). Used to scrub text derived from the answer, such as
|
|
49
|
-
* citation sentences.
|
|
76
|
+
* citation sentences. Detectors don't run here (they may be async):
|
|
77
|
+
* pass the strings a check found, `detectedText(result)`, as `matches`
|
|
78
|
+
* so they are removed too.
|
|
50
79
|
*
|
|
51
80
|
* @param {string} text
|
|
81
|
+
* @param {string[]} [matches=[]] - Exact strings to redact as well
|
|
52
82
|
* @returns {string}
|
|
53
83
|
*/
|
|
54
|
-
redact(text: string): string;
|
|
84
|
+
redact(text: string, matches?: string[]): string;
|
|
55
85
|
}
|
|
56
86
|
/**
|
|
57
87
|
* Default configuration for the PostGenerationGuard.
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* @extends EventEmitter
|
|
7
7
|
*/
|
|
8
|
-
export class PreRetrievalGuard extends EventEmitter
|
|
8
|
+
export class PreRetrievalGuard extends EventEmitter {
|
|
9
9
|
/**
|
|
10
10
|
* @param {object} [options] - Override default configuration
|
|
11
11
|
* @param {number} [options.maxQueryLength=10000] - Maximum allowed query length
|
|
@@ -14,6 +14,9 @@ export class PreRetrievalGuard extends EventEmitter<[never]> {
|
|
|
14
14
|
* @param {string[]} [options.blockedTopics=[]] - Topics to reject
|
|
15
15
|
* @param {string[]} [options.allowedTopics=[]] - If non-empty, only these topics are permitted
|
|
16
16
|
* @param {RegExp[]} [options.injectionPatterns] - Additional injection patterns (merged with defaults)
|
|
17
|
+
* @param {Array<object>} [options.detectors=[]] - Injection detectors to run on the query as well (see ./detectors); async ones need checkAsync()
|
|
18
|
+
* @param {boolean} [options.builtIn=true] - false: use only `detectors`, not the built-in patterns
|
|
19
|
+
* @param {'block'|'skip'} [options.onDetectorError='block'] - A detector that throws makes the query unsafe ('block') or is ignored ('skip')
|
|
17
20
|
*/
|
|
18
21
|
constructor(options?: {
|
|
19
22
|
maxQueryLength?: number;
|
|
@@ -22,7 +25,12 @@ export class PreRetrievalGuard extends EventEmitter<[never]> {
|
|
|
22
25
|
blockedTopics?: string[];
|
|
23
26
|
allowedTopics?: string[];
|
|
24
27
|
injectionPatterns?: RegExp[];
|
|
28
|
+
detectors?: Array<object>;
|
|
29
|
+
builtIn?: boolean;
|
|
30
|
+
onDetectorError?: "block" | "skip";
|
|
25
31
|
});
|
|
32
|
+
/** @private */
|
|
33
|
+
private _detectors;
|
|
26
34
|
config: any;
|
|
27
35
|
/** @private Extra patterns supplied by the caller */
|
|
28
36
|
private _customPatterns;
|
|
@@ -55,6 +63,26 @@ export class PreRetrievalGuard extends EventEmitter<[never]> {
|
|
|
55
63
|
}>;
|
|
56
64
|
};
|
|
57
65
|
};
|
|
66
|
+
/**
|
|
67
|
+
* check() for async detectors, such as a classifier model or a service.
|
|
68
|
+
* Detectors run concurrently.
|
|
69
|
+
*
|
|
70
|
+
* @param {string} query
|
|
71
|
+
* @param {object} [options]
|
|
72
|
+
* @param {AbortSignal} [options.signal] - Passed to each detector in its context
|
|
73
|
+
* @returns {Promise<ReturnType<PreRetrievalGuard['check']>>}
|
|
74
|
+
*/
|
|
75
|
+
checkAsync(query: string, { signal }?: {
|
|
76
|
+
signal?: AbortSignal;
|
|
77
|
+
}): Promise<ReturnType<PreRetrievalGuard["check"]>>;
|
|
78
|
+
/** @private */
|
|
79
|
+
private _runDetectors;
|
|
80
|
+
/** @private */
|
|
81
|
+
private _emit;
|
|
82
|
+
/** @private Validation and the built-in checks, before any detector runs. */
|
|
83
|
+
private _prepare;
|
|
84
|
+
/** @private Merge the detectors' findings and emit the result. */
|
|
85
|
+
private _finish;
|
|
58
86
|
}
|
|
59
87
|
/**
|
|
60
88
|
* Default configuration for the PreRetrievalGuard.
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* @extends EventEmitter
|
|
7
7
|
*/
|
|
8
|
-
export class RetrievalGuard extends EventEmitter
|
|
8
|
+
export class RetrievalGuard extends EventEmitter {
|
|
9
9
|
/**
|
|
10
10
|
* @param {object} [options] - Override default configuration
|
|
11
11
|
* @param {number} [options.minRelevanceScore=0] - Minimum relevance score threshold
|
|
@@ -14,6 +14,9 @@ export class RetrievalGuard extends EventEmitter<[never]> {
|
|
|
14
14
|
* @param {boolean} [options.detectInjection=false] - Scan document text for prompt injection
|
|
15
15
|
* @param {'remove'|'flag'} [options.injectionAction='remove'] - Remove flagged documents or keep and mark them
|
|
16
16
|
* @param {RegExp[]} [options.injectionPatterns=[]] - Extra injection patterns for the scan
|
|
17
|
+
* @param {Array<object>} [options.detectors=[]] - Injection detectors to run on each document's text too, when detectInjection is on (see ./detectors); async ones need filterAsync(), which calls them once per document, concurrently
|
|
18
|
+
* @param {boolean} [options.builtIn=true] - false: use only `detectors` for the injection scan
|
|
19
|
+
* @param {'block'|'skip'} [options.onDetectorError='block'] - A detector that throws counts as a finding ('block': the document is removed or flagged) or is ignored ('skip')
|
|
17
20
|
*/
|
|
18
21
|
constructor(options?: {
|
|
19
22
|
minRelevanceScore?: number;
|
|
@@ -22,8 +25,13 @@ export class RetrievalGuard extends EventEmitter<[never]> {
|
|
|
22
25
|
detectInjection?: boolean;
|
|
23
26
|
injectionAction?: "remove" | "flag";
|
|
24
27
|
injectionPatterns?: RegExp[];
|
|
28
|
+
detectors?: Array<object>;
|
|
29
|
+
builtIn?: boolean;
|
|
30
|
+
onDetectorError?: "block" | "skip";
|
|
25
31
|
});
|
|
26
32
|
config: any;
|
|
33
|
+
/** @private */
|
|
34
|
+
private _detectors;
|
|
27
35
|
/**
|
|
28
36
|
* Filter retrieval results based on configured guards.
|
|
29
37
|
*
|
|
@@ -48,6 +56,27 @@ export class RetrievalGuard extends EventEmitter<[never]> {
|
|
|
48
56
|
};
|
|
49
57
|
};
|
|
50
58
|
};
|
|
59
|
+
/**
|
|
60
|
+
* filter() for async detectors, such as a classifier model or a service. Each
|
|
61
|
+
* document that reaches the injection scan is checked by every detector,
|
|
62
|
+
* all concurrently.
|
|
63
|
+
*
|
|
64
|
+
* @param {Array<object>} results
|
|
65
|
+
* @param {object} [context={}]
|
|
66
|
+
* @param {object} [options]
|
|
67
|
+
* @param {AbortSignal} [options.signal] - Passed to each detector in its context
|
|
68
|
+
* @returns {Promise<ReturnType<RetrievalGuard['filter']>>}
|
|
69
|
+
*/
|
|
70
|
+
filterAsync(results: Array<object>, context?: object, { signal }?: {
|
|
71
|
+
signal?: AbortSignal;
|
|
72
|
+
}): Promise<ReturnType<RetrievalGuard["filter"]>>;
|
|
73
|
+
/**
|
|
74
|
+
* @private Relevance, freshness and ACL, then which documents go on to
|
|
75
|
+
* the injection scan.
|
|
76
|
+
*/
|
|
77
|
+
private _screen;
|
|
78
|
+
/** @private 4. Indirect prompt injection in document text (opt-in). */
|
|
79
|
+
private _finish;
|
|
51
80
|
}
|
|
52
81
|
/**
|
|
53
82
|
* Default configuration for the RetrievalGuard.
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* const tools = server.getToolDefinitions();
|
|
10
10
|
* const result = await server.handleToolUse('my_tool', { query: 'What is X?' });
|
|
11
11
|
*/
|
|
12
|
-
export class MCPServer extends EventEmitter
|
|
12
|
+
export class MCPServer extends EventEmitter {
|
|
13
13
|
/**
|
|
14
14
|
* Convenience factory: create an MCPServer with a single pipeline registered.
|
|
15
15
|
*
|
|
@@ -1,16 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Normalise the `telemetry` option of createRagPipeline. `api` is the
|
|
3
|
-
* @opentelemetry/api module, for builds that cannot require() it (/edge)
|
|
4
|
-
*
|
|
5
|
-
*
|
|
3
|
+
* @opentelemetry/api module, for builds that cannot require() it (/edge);
|
|
4
|
+
* `logs` is @opentelemetry/api-logs, for evaluation events with an SDK that
|
|
5
|
+
* registers its LoggerProvider there, and `logger` a Logger to emit them with.
|
|
6
|
+
* @param {boolean|{tracer?: object, api?: object, logs?: object, logger?: object, recordContent?: boolean}} option
|
|
7
|
+
* @returns {null|{api: object, tracer: object, logger: object|null, runKey: symbol, recordContent: boolean}}
|
|
6
8
|
*/
|
|
7
9
|
export function createTelemetry(option: boolean | {
|
|
8
10
|
tracer?: object;
|
|
9
11
|
api?: object;
|
|
12
|
+
logs?: object;
|
|
13
|
+
logger?: object;
|
|
10
14
|
recordContent?: boolean;
|
|
11
15
|
}): null | {
|
|
12
16
|
api: object;
|
|
13
17
|
tracer: object;
|
|
18
|
+
logger: object | null;
|
|
19
|
+
runKey: symbol;
|
|
14
20
|
recordContent: boolean;
|
|
15
21
|
};
|
|
16
22
|
/** Wrap a component so its stage methods run in spans. */
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @param {object} result - A pipeline run result (or a stream's done
|
|
3
|
+
* chunk) with `evaluation` and/or `citations`, or a checkAnswer() result
|
|
4
|
+
* @param {object} [options]
|
|
5
|
+
* @param {boolean} [options.explanations=false] - Include explanations,
|
|
6
|
+
* which quote the answer
|
|
7
|
+
* @returns {Array<{name: string, value: number, target: 'retrieval'|'generation', label?: string, explanation?: string}>}
|
|
8
|
+
*/
|
|
9
|
+
export function evaluationScores(result: object, { explanations }?: {
|
|
10
|
+
explanations?: boolean;
|
|
11
|
+
}): Array<{
|
|
12
|
+
name: string;
|
|
13
|
+
value: number;
|
|
14
|
+
target: "retrieval" | "generation";
|
|
15
|
+
label?: string;
|
|
16
|
+
explanation?: string;
|
|
17
|
+
}>;
|
|
18
|
+
/**
|
|
19
|
+
* The scores of a run result or answer check as Langfuse scores: the
|
|
20
|
+
* objects `langfuse.score.create()` takes (from `@langfuse/client`), and,
|
|
21
|
+
* without a trace id, the evaluations a Langfuse experiment evaluator
|
|
22
|
+
* returns. Langfuse is not a dependency.
|
|
23
|
+
*
|
|
24
|
+
* With `telemetry` on, a run result carries its OpenTelemetry trace and span
|
|
25
|
+
* ids in `result.telemetry`; Langfuse uses the same ids when it receives the
|
|
26
|
+
* spans (LangfuseSpanProcessor), so each score is attached to the trace and
|
|
27
|
+
* to the span it judges: the retrieval for contextPrecision and
|
|
28
|
+
* contextRecall, the generation for the rest. A score whose stage had no
|
|
29
|
+
* span (a cache hit) gets the trace only: Langfuse's default span filter
|
|
30
|
+
* keeps the stage spans (they have gen_ai.* attributes) but not the run
|
|
31
|
+
* span, so an observation id for it could point at nothing.
|
|
32
|
+
*
|
|
33
|
+
* @param {object} result - A pipeline run result, a stream's done chunk, or a checkAnswer() result
|
|
34
|
+
* @param {object} [options]
|
|
35
|
+
* @param {string} [options.traceId] - Overrides `result.telemetry.traceId`
|
|
36
|
+
* @param {string} [options.observationId] - Attach every score to this observation
|
|
37
|
+
* @param {string} [options.sessionId]
|
|
38
|
+
* @param {boolean} [options.comments=false] - Add explanations as comments; they quote the answer
|
|
39
|
+
* @returns {Array<{name: string, value: number, dataType: 'NUMERIC', traceId?: string, observationId?: string, sessionId?: string, comment?: string, metadata?: {label: string}}>}
|
|
40
|
+
*/
|
|
41
|
+
export function toLangfuseScores(result: object, options?: {
|
|
42
|
+
traceId?: string;
|
|
43
|
+
observationId?: string;
|
|
44
|
+
sessionId?: string;
|
|
45
|
+
comments?: boolean;
|
|
46
|
+
}): Array<{
|
|
47
|
+
name: string;
|
|
48
|
+
value: number;
|
|
49
|
+
dataType: "NUMERIC";
|
|
50
|
+
traceId?: string;
|
|
51
|
+
observationId?: string;
|
|
52
|
+
sessionId?: string;
|
|
53
|
+
comment?: string;
|
|
54
|
+
metadata?: {
|
|
55
|
+
label: string;
|
|
56
|
+
};
|
|
57
|
+
}>;
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* @extends EventEmitter
|
|
7
7
|
*/
|
|
8
|
-
export class CascadeReranker extends EventEmitter
|
|
8
|
+
export class CascadeReranker extends EventEmitter {
|
|
9
9
|
/**
|
|
10
10
|
* @param {object[]} [rerankers=[]] - Array of objects with .rerank() method
|
|
11
11
|
* @param {object} [options]
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*
|
|
9
9
|
* @extends EventEmitter
|
|
10
10
|
*/
|
|
11
|
-
export class EmbeddingReranker extends EventEmitter
|
|
11
|
+
export class EmbeddingReranker extends EventEmitter {
|
|
12
12
|
/**
|
|
13
13
|
* @param {object} embedder - Must expose .embedQuery(text) or .embed(text)
|
|
14
14
|
* @param {object} [options]
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Keyword search with BM25 ranking. Index documents with index(), then
|
|
3
3
|
* search(query) returns them scored by term frequency and rarity.
|
|
4
4
|
*/
|
|
5
|
-
export class BM25Search extends EventEmitter
|
|
5
|
+
export class BM25Search extends EventEmitter {
|
|
6
6
|
constructor(options?: {});
|
|
7
7
|
config: {
|
|
8
8
|
k1: number;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Runs several retrievers in parallel (for example vector and BM25) and
|
|
3
3
|
* fuses their results with weighted reciprocal rank fusion.
|
|
4
4
|
*/
|
|
5
|
-
export class HybridRetriever extends EventEmitter
|
|
5
|
+
export class HybridRetriever extends EventEmitter {
|
|
6
6
|
/**
|
|
7
7
|
* @param {Array<{retriever: object, weight: number, name: string}>} retrievers
|
|
8
8
|
* @param {object} options
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* adapter.on('token', (t) => console.log('sent', t));
|
|
12
12
|
* await adapter.stream(asyncTokenGenerator, res);
|
|
13
13
|
*/
|
|
14
|
-
export class SSEAdapter extends EventEmitter
|
|
14
|
+
export class SSEAdapter extends EventEmitter {
|
|
15
15
|
/**
|
|
16
16
|
* @param {Object} [options] - Configuration overrides
|
|
17
17
|
* @param {string} [options.eventName='token'] - SSE event name for token frames
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* const handler = router.createHandler(pipeline);
|
|
14
14
|
* app.post('/api/query', handler);
|
|
15
15
|
*/
|
|
16
|
-
export class StreamRouter extends EventEmitter
|
|
16
|
+
export class StreamRouter extends EventEmitter {
|
|
17
17
|
/**
|
|
18
18
|
* @param {Object} [options] - Configuration overrides
|
|
19
19
|
* @param {string} [options.defaultTransport='sse'] - Fallback transport type
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* se.on('done', (stats) => console.log('finished', stats));
|
|
17
17
|
* const all = await se.embedAll(documents);
|
|
18
18
|
*/
|
|
19
|
-
export class StreamingEmbedder extends EventEmitter
|
|
19
|
+
export class StreamingEmbedder extends EventEmitter {
|
|
20
20
|
/**
|
|
21
21
|
* @param {Object} embedder - Embedder instance with `.embed()` or `.embedQuery()`
|
|
22
22
|
* @param {Object} [options] - Override any key from {@link DEFAULT_CONFIG}
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* adapter.on('done', () => console.log('stream complete'));
|
|
13
13
|
* await adapter.stream(asyncTokenGenerator, ws);
|
|
14
14
|
*/
|
|
15
|
-
export class WebSocketAdapter extends EventEmitter
|
|
15
|
+
export class WebSocketAdapter extends EventEmitter {
|
|
16
16
|
/**
|
|
17
17
|
* @param {Object} [options] - Configuration overrides
|
|
18
18
|
* @param {string} [options.messageType='token'] - Type field for token messages
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* const result = scaffolder.create('document-qa', '/path/to/my-project');
|
|
10
10
|
* console.log(result.files); // list of created files
|
|
11
11
|
*/
|
|
12
|
-
export class ProjectScaffolder extends EventEmitter
|
|
12
|
+
export class ProjectScaffolder extends EventEmitter {
|
|
13
13
|
/**
|
|
14
14
|
* @param {Object} [options={}] - Scaffolder configuration
|
|
15
15
|
* @param {boolean} [options.installDependencies=false] - Whether to auto-run npm install
|
|
@@ -11,20 +11,22 @@
|
|
|
11
11
|
* import { describe, it } from "vitest"; // or "node:test"
|
|
12
12
|
* runContractTests("llm", () => new MyLLM(), { describe, it });
|
|
13
13
|
*
|
|
14
|
-
* @param {'loader'|'embedder'|'retriever'|'reranker'|'llm'} type - Plugin type
|
|
14
|
+
* @param {'loader'|'embedder'|'retriever'|'reranker'|'llm'|'detector'} type - Plugin type
|
|
15
15
|
* @param {() => (object|Promise<object>)} factory - Returns a new, empty plugin for each check
|
|
16
16
|
* @param {object} [options]
|
|
17
17
|
* @param {string} [options.name] - Name of the test group (default: "<type> contract")
|
|
18
18
|
* @param {boolean} [options.registry=false] - Also require the `metadata` pluginRegistry.register() needs
|
|
19
19
|
* @param {(retriever: object, documents: object[]) => (void|Promise<void>)} [options.setup] - Retrievers: loads the test documents (default: `retriever.store(documents)`), for retrievers without store()
|
|
20
|
+
* @param {string} [options.positive] - Detectors: text the detector must flag, instead of the default sample for its kind
|
|
20
21
|
* @param {Function} [options.describe] - Your framework's describe (default: the global one)
|
|
21
22
|
* @param {Function} [options.it] - Your framework's it/test (default: the global it or test)
|
|
22
23
|
* @returns {void}
|
|
23
24
|
*/
|
|
24
|
-
export function runContractTests(type: "loader" | "embedder" | "retriever" | "reranker" | "llm", factory: () => (object | Promise<object>), options?: {
|
|
25
|
+
export function runContractTests(type: "loader" | "embedder" | "retriever" | "reranker" | "llm" | "detector", factory: () => (object | Promise<object>), options?: {
|
|
25
26
|
name?: string;
|
|
26
27
|
registry?: boolean;
|
|
27
28
|
setup?: (retriever: object, documents: object[]) => (void | Promise<void>);
|
|
29
|
+
positive?: string;
|
|
28
30
|
describe?: Function;
|
|
29
31
|
it?: Function;
|
|
30
32
|
}): void;
|
|
@@ -35,16 +37,18 @@ export function runContractTests(type: "loader" | "embedder" | "retriever" | "re
|
|
|
35
37
|
* const report = await checkContract("embedder", () => new MyEmbedder());
|
|
36
38
|
* if (!report.passed) console.log(report.results.filter((r) => !r.passed));
|
|
37
39
|
*
|
|
38
|
-
* @param {'loader'|'embedder'|'retriever'|'reranker'|'llm'} type - Plugin type
|
|
40
|
+
* @param {'loader'|'embedder'|'retriever'|'reranker'|'llm'|'detector'} type - Plugin type
|
|
39
41
|
* @param {() => (object|Promise<object>)} factory - Returns a new, empty plugin for each check
|
|
40
42
|
* @param {object} [options]
|
|
41
43
|
* @param {boolean} [options.registry=false] - Also require the `metadata` pluginRegistry.register() needs
|
|
42
44
|
* @param {(retriever: object, documents: object[]) => (void|Promise<void>)} [options.setup] - Retrievers: loads the test documents (default: `retriever.store(documents)`), for retrievers without store()
|
|
45
|
+
* @param {string} [options.positive] - Detectors: text the detector must flag, instead of the default sample for its kind
|
|
43
46
|
* @returns {Promise<{ type: string, passed: boolean, results: Array<{ name: string, passed: boolean, skipped: boolean, error?: string }> }>}
|
|
44
47
|
*/
|
|
45
|
-
export function checkContract(type: "loader" | "embedder" | "retriever" | "reranker" | "llm", factory: () => (object | Promise<object>), options?: {
|
|
48
|
+
export function checkContract(type: "loader" | "embedder" | "retriever" | "reranker" | "llm" | "detector", factory: () => (object | Promise<object>), options?: {
|
|
46
49
|
registry?: boolean;
|
|
47
50
|
setup?: (retriever: object, documents: object[]) => (void | Promise<void>);
|
|
51
|
+
positive?: string;
|
|
48
52
|
}): Promise<{
|
|
49
53
|
type: string;
|
|
50
54
|
passed: boolean;
|