@devilsdev/rag-pipeline-utils 2.4.5 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ragrc.schema.json +83 -192
- package/CHANGELOG.md +345 -0
- package/README.md +418 -117
- package/SECURITY.md +5 -5
- package/dist/experimental.cjs +2 -0
- package/dist/experimental.d.ts +1 -0
- package/dist/experimental.mjs +5 -0
- package/dist/index.cjs +4992 -1242
- package/dist/index.d.ts +80 -2
- package/dist/index.mjs +1 -1
- package/dist/types/agentic/agentic-pipeline.d.ts +103 -0
- package/dist/types/agentic/iterative-retriever.d.ts +85 -0
- package/dist/types/agentic/query-planner.d.ts +145 -0
- package/dist/types/agentic/self-critique.d.ts +45 -0
- package/dist/types/ai/adaptive-retrieval.d.ts +44 -0
- package/dist/types/ai/federation/federated-learning-coordinator.d.ts +62 -0
- package/dist/types/ai/multimodal-processing.d.ts +32 -0
- package/dist/types/ai/multimodal.d.ts +39 -0
- package/dist/types/ai/retrieval-engine.d.ts +37 -0
- package/dist/types/ai/training/model-training-orchestrator.d.ts +31 -0
- package/dist/types/chunking/chunking-engine.d.ts +70 -0
- package/dist/types/chunking/strategies/fixed-size.d.ts +17 -0
- package/dist/types/chunking/strategies/recursive.d.ts +20 -0
- package/dist/types/chunking/strategies/semantic.d.ts +19 -0
- package/dist/types/chunking/strategies/sentence.d.ts +13 -0
- package/dist/types/chunking/strategies/structure-aware.d.ts +19 -0
- package/dist/types/citation/citation-tracker.d.ts +64 -0
- package/dist/types/citation/hallucination-detector.d.ts +40 -0
- package/dist/types/citation/source-mapper.d.ts +37 -0
- package/dist/types/connectors/anthropic-connector.d.ts +92 -0
- package/dist/types/connectors/base-connector.d.ts +87 -0
- package/dist/types/connectors/cohere-connector.d.ts +89 -0
- package/dist/types/connectors/connector-utils.d.ts +110 -0
- package/dist/types/connectors/local-embedder.d.ts +86 -0
- package/dist/types/connectors/memory-retriever.d.ts +91 -0
- package/dist/types/connectors/ollama-connector.d.ts +110 -0
- package/dist/types/connectors/openai-connector.d.ts +114 -0
- package/dist/types/connectors/rag-prompt.d.ts +40 -0
- package/dist/types/cost/cost-calculator.d.ts +152 -0
- package/dist/types/cost/cost-tracker.d.ts +126 -0
- package/dist/types/cost/token-budget.d.ts +110 -0
- package/dist/types/dag/core/error-context.d.ts +239 -0
- package/dist/types/dag/core/execution-scheduler.d.ts +191 -0
- package/dist/types/dag/core/topology-validation.d.ts +226 -0
- package/dist/types/dag/dag-engine.d.ts +156 -0
- package/dist/types/dag/dag-node.d.ts +37 -0
- package/dist/types/dashboard/dashboard-generator.d.ts +127 -0
- package/dist/types/dashboard/metrics-aggregator.d.ts +218 -0
- package/dist/types/debug/execution-tracer.d.ts +57 -0
- package/dist/types/debug/trace-inspector.d.ts +31 -0
- package/dist/types/ecosystem/plugin-certification.d.ts +169 -0
- package/dist/types/enterprise/data-governance.d.ts +203 -0
- package/dist/types/enterprise/multi-tenancy.d.ts +137 -0
- package/dist/types/enterprise/sso-integration.d.ts +420 -0
- package/dist/types/evaluate/context-metrics.d.ts +55 -0
- package/dist/types/evaluate/faithfulness.d.ts +41 -0
- package/dist/types/evaluate/groundedness.d.ts +60 -0
- package/dist/types/evaluate/pipeline-evaluator.d.ts +88 -0
- package/dist/types/evaluate/relevance.d.ts +21 -0
- package/dist/types/evaluate/scoring.d.ts +123 -0
- package/dist/types/experimental.d.ts +15 -0
- package/dist/types/graph/entity-extractor.d.ts +84 -0
- package/dist/types/graph/graph-index.d.ts +49 -0
- package/dist/types/graph/graph-retriever.d.ts +81 -0
- package/dist/types/graph/knowledge-graph.d.ts +129 -0
- package/dist/types/guardrails/guardrails-pipeline.d.ts +63 -0
- package/dist/types/guardrails/injection-detector.d.ts +47 -0
- package/dist/types/guardrails/post-generation-guard.d.ts +61 -0
- package/dist/types/guardrails/pre-retrieval-guard.d.ts +64 -0
- package/dist/types/guardrails/retrieval-guard.d.ts +57 -0
- package/dist/types/mcp/mcp-server.d.ts +85 -0
- package/dist/types/mcp/mcp-tool-builder.d.ts +55 -0
- package/dist/types/reranker/cascade-reranker.d.ts +40 -0
- package/dist/types/reranker/embedding-reranker.d.ts +77 -0
- package/dist/types/reranker/scoring-reranker.d.ts +41 -0
- package/dist/types/retrieval/bm25-search.d.ts +43 -0
- package/dist/types/retrieval/file-vector-store.d.ts +87 -0
- package/dist/types/retrieval/hybrid-retriever.d.ts +35 -0
- package/dist/types/retrieval/rank-fusion.d.ts +10 -0
- package/dist/types/security/jwt-config.d.ts +80 -0
- package/dist/types/security/jwt-validator.d.ts +195 -0
- package/dist/types/security/plugin-sandbox.d.ts +158 -0
- package/dist/types/security/plugin-signature-verifier.d.ts +139 -0
- package/dist/types/security/validator.d.ts +5 -0
- package/dist/types/streaming/embedding-stream.d.ts +65 -0
- package/dist/types/streaming/sse-adapter.d.ts +59 -0
- package/dist/types/streaming/stream-router.d.ts +88 -0
- package/dist/types/streaming/streaming-embedder.d.ts +99 -0
- package/dist/types/streaming/websocket-adapter.d.ts +71 -0
- package/dist/types/templates/scaffold.d.ts +55 -0
- package/dist/types/templates/template-registry.d.ts +63 -0
- package/dist/types/utils/embed-query.d.ts +20 -0
- package/dist/types/utils/logger.d.ts +50 -0
- package/dist/types/utils/rate-limiter.d.ts +66 -0
- package/dist/types/utils/retry.d.ts +27 -0
- package/dist/types/utils/run-abort.d.ts +34 -0
- package/dist/types/utils/secure-logging.d.ts +51 -0
- package/dist/types/utils/structured-logger.d.ts +126 -0
- package/package.json +53 -29
- package/src/agentic/agentic-pipeline.js +75 -7
- package/src/agentic/iterative-retriever.js +47 -10
- package/src/agentic/query-planner.js +329 -78
- package/src/citation/citation-tracker.js +6 -4
- package/src/citation/hallucination-detector.js +2 -2
- package/src/citation/source-mapper.js +15 -28
- package/src/cli/commands/ai.js +101 -0
- package/src/cli/commands/plugin-hub/event-handlers.js +6 -0
- package/src/cli/commands/plugin-hub/hub-commands.js +0 -74
- package/src/cli/commands/plugin-hub/local-commands.js +143 -0
- package/src/cli/commands/plugin-hub.js +96 -36
- package/src/cli/diagnostics/config-checker.js +76 -63
- package/src/cli/diagnostics/environment-checker.js +62 -31
- package/src/cli/diagnostics/plugin-checker.js +36 -89
- package/src/cli/doctor-command.js +39 -5
- package/src/cli/enhanced-cli-commands.js +72 -217
- package/src/cli/handlers/ai-handler.js +364 -0
- package/src/cli/handlers/diagnostics-handler.js +78 -31
- package/src/cli/handlers/ingest-handler.js +28 -22
- package/src/cli/handlers/init-handler.js +148 -25
- package/src/cli/handlers/query-handler.js +41 -26
- package/src/cli/handlers/run-handler.js +16 -167
- package/src/cli/handlers/serve-handler.js +60 -0
- package/src/cli/services/bench-stats.js +54 -0
- package/src/cli/services/benchmark.js +161 -0
- package/src/cli/services/feedback.js +155 -0
- package/src/cli/services/rag-service.js +322 -0
- package/src/cli/shared/cli-utils.js +24 -7
- package/src/cli/shared/config-context.js +21 -0
- package/src/cli/wizard/config-builder.js +81 -0
- package/src/cli/wizard/init-wizard.js +617 -0
- package/src/config/component-factory.js +125 -0
- package/src/config/load-config.js +29 -10
- package/src/config/load-plugin-config.js +46 -2
- package/src/connectors/anthropic-connector.js +93 -55
- package/src/connectors/base-connector.js +74 -19
- package/src/connectors/cohere-connector.js +88 -32
- package/src/connectors/connector-utils.js +302 -0
- package/src/connectors/local-embedder.js +43 -5
- package/src/connectors/memory-retriever.js +108 -15
- package/src/connectors/ollama-connector.js +83 -49
- package/src/connectors/openai-connector.js +108 -44
- package/src/connectors/rag-prompt.js +88 -0
- package/src/core/create-pipeline.js +209 -60
- package/src/core/plugin-registry.js +87 -11
- package/src/cost/cost-calculator.js +205 -32
- package/src/dev/dev-server.js +166 -11
- package/src/dev/hot-reload.js +40 -1
- package/src/ecosystem/plugin-checks.js +181 -0
- package/src/ecosystem/plugin-hub.js +50 -125
- package/src/ecosystem/plugin-installer.js +298 -0
- package/src/ecosystem/plugin-probe.js +127 -0
- package/src/ecosystem/plugin-scanner.js +185 -0
- package/src/ecosystem/tar-extract.js +114 -0
- package/src/enterprise/audit-logging.js +158 -29
- package/src/enterprise/sso-integration.js +715 -228
- package/src/evaluate/context-metrics.js +28 -16
- package/src/evaluate/evaluator.js +3 -3
- package/src/evaluate/faithfulness.js +18 -14
- package/src/evaluate/groundedness.js +43 -12
- package/src/evaluate/pipeline-evaluator.js +35 -8
- package/src/evaluate/scoring.js +251 -3
- package/src/experimental.js +71 -0
- package/src/guardrails/guardrails-pipeline.js +69 -7
- package/src/guardrails/injection-detector.js +283 -0
- package/src/guardrails/post-generation-guard.js +19 -0
- package/src/guardrails/pre-retrieval-guard.js +36 -36
- package/src/guardrails/retrieval-guard.js +58 -8
- package/src/index.js +36 -1
- package/src/loader/document-reader.js +91 -0
- package/src/reranker/cascade-reranker.js +3 -0
- package/src/reranker/embedding-reranker.js +136 -21
- package/src/reranker/llm-reranker.js +79 -12
- package/src/reranker/scoring-reranker.js +3 -0
- package/src/retrieval/bm25-search.js +12 -6
- package/src/retrieval/file-vector-store.js +212 -0
- package/src/retrieval/hybrid-retriever.js +22 -19
- package/src/security/jwt-config.js +13 -2
- package/src/security/jwt-validator.js +134 -30
- package/src/security/plugin-sandbox.js +18 -1
- package/src/security/plugin-signature-verifier.js +245 -70
- package/src/utils/deprecate.js +69 -0
- package/src/utils/embed-query.js +33 -0
- package/src/utils/input-sanitizer.js +19 -4
- package/src/utils/logger.js +34 -4
- package/src/utils/run-abort.js +102 -0
- package/src/ai/federated-learning.js +0 -211
- package/src/ai/federation.js +0 -71
- package/src/ai/index.js +0 -22
- package/src/ai/model-trainer.js +0 -217
- package/src/ai/model-training.js +0 -202
- package/src/ai/multimodal/multi-modal-processor.js +0 -345
- package/src/ai/orchestrator.js +0 -65
- package/src/ai/retrieval/adaptive-retrieval-engine.js +0 -398
- package/src/cli/commands/ai-ml/adaptive-commands.js +0 -178
- package/src/cli/commands/ai-ml/federated-commands.js +0 -207
- package/src/cli/commands/ai-ml/index.js +0 -17
- package/src/cli/commands/ai-ml/multimodal-commands.js +0 -187
- package/src/cli/commands/ai-ml/shared.js +0 -51
- package/src/cli/commands/ai-ml/training-commands.js +0 -228
- package/src/cli/commands/ai-ml.js +0 -386
- package/src/cli/commands/docs.js +0 -228
- package/src/cli/commands/dx.js +0 -538
- package/src/cli/commands/wizard.js +0 -627
- package/src/cli/input-sanitization-middleware.js +0 -356
- package/src/cli/interactive-wizard.js +0 -373
- package/src/cli/templates/development.template.json +0 -64
- package/src/cli/templates/minimal.template.json +0 -33
- package/src/cli/templates/production.template.json +0 -104
- package/src/cli/templates/testing.template.json +0 -45
- package/src/cli/wizard/configuration.js +0 -148
- package/src/cli/wizard/observability.js +0 -79
- package/src/cli/wizard/performance.js +0 -76
- package/src/cli/wizard/plugin-selection.js +0 -297
- package/src/cli/wizard/preview-save.js +0 -172
- package/src/cli/wizard/project-setup.js +0 -66
- package/src/core/pipeline-factory.js +0 -59
- package/src/dag/enhanced-dag-engine.js +0 -894
- package/src/dx/index.js +0 -40
- package/src/dx/integration-templates.js +0 -662
- package/src/dx/performance-profiler.js +0 -968
- package/src/dx/realtime-debugger.js +0 -755
- package/src/dx/visual-pipeline-builder.js +0 -578
- package/src/ingest.js +0 -59
- package/src/pipeline/pipeline.js +0 -18
- package/src/query.js +0 -117
|
@@ -16,6 +16,20 @@ const DEFAULT_CONFIG = {
|
|
|
16
16
|
strict: false, // If true, block on any guard failure; if false, warn and continue
|
|
17
17
|
};
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* Reduce removed documents to their ids so filtered-out (possibly
|
|
21
|
+
* access-controlled) content is never returned to the caller.
|
|
22
|
+
*
|
|
23
|
+
* @param {{results: Array, removed: Array, metadata: object}} filterOutput
|
|
24
|
+
* @returns {{results: Array, removed: Array<{id: *}>, metadata: object}}
|
|
25
|
+
*/
|
|
26
|
+
function summarizeFilter(filterOutput) {
|
|
27
|
+
return {
|
|
28
|
+
...filterOutput,
|
|
29
|
+
removed: filterOutput.removed.map((doc) => ({ id: doc.id })),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
19
33
|
/**
|
|
20
34
|
* Pipeline wrapper that applies all three guardrail layers (pre-retrieval,
|
|
21
35
|
* retrieval, and post-generation) around an existing RAG pipeline. In strict
|
|
@@ -92,21 +106,49 @@ class GuardrailsPipeline extends EventEmitter {
|
|
|
92
106
|
query = guardrails.preRetrieval.sanitizedQuery;
|
|
93
107
|
}
|
|
94
108
|
|
|
95
|
-
// 2. Run the underlying pipeline with (possibly sanitized) query
|
|
109
|
+
// 2. Run the underlying pipeline with (possibly sanitized) query. The
|
|
110
|
+
// retrieval guard is passed in as options.filterResults so it runs
|
|
111
|
+
// between retrieval and generation (supported by createRagPipeline).
|
|
112
|
+
let preGenerationFilter = null;
|
|
113
|
+
const filterOptions = this.retrievalGuard
|
|
114
|
+
? {
|
|
115
|
+
filterResults: (docs) => {
|
|
116
|
+
preGenerationFilter = this.retrievalGuard.filter(
|
|
117
|
+
docs,
|
|
118
|
+
input.context,
|
|
119
|
+
);
|
|
120
|
+
return preGenerationFilter.results;
|
|
121
|
+
},
|
|
122
|
+
}
|
|
123
|
+
: {};
|
|
96
124
|
const result = await this.pipeline.run({
|
|
97
125
|
...input,
|
|
98
126
|
query,
|
|
99
|
-
options: { ...input.options, citations: true },
|
|
127
|
+
options: { ...input.options, citations: true, ...filterOptions },
|
|
100
128
|
});
|
|
101
129
|
|
|
102
130
|
// 3. Retrieval guard on results
|
|
103
|
-
if (this.retrievalGuard &&
|
|
104
|
-
guardrails.retrieval =
|
|
105
|
-
|
|
106
|
-
|
|
131
|
+
if (this.retrievalGuard && preGenerationFilter) {
|
|
132
|
+
guardrails.retrieval = summarizeFilter(preGenerationFilter);
|
|
133
|
+
this.emit('retrieval', guardrails.retrieval);
|
|
134
|
+
} else if (this.retrievalGuard && result.results) {
|
|
135
|
+
// The wrapped pipeline ignored options.filterResults, so it generated
|
|
136
|
+
// from unfiltered documents. Filtering now only fixes result.results;
|
|
137
|
+
// if ACL removed anything the answer may contain restricted content.
|
|
138
|
+
guardrails.retrieval = summarizeFilter(
|
|
139
|
+
this.retrievalGuard.filter(result.results, input.context),
|
|
107
140
|
);
|
|
108
141
|
this.emit('retrieval', guardrails.retrieval);
|
|
109
142
|
result.results = guardrails.retrieval.results;
|
|
143
|
+
|
|
144
|
+
if (guardrails.retrieval.metadata.reasons.acl) {
|
|
145
|
+
return {
|
|
146
|
+
success: false,
|
|
147
|
+
error:
|
|
148
|
+
'Retrieval guard removed access-controlled documents after generation; answer withheld',
|
|
149
|
+
guardrails,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
110
152
|
}
|
|
111
153
|
|
|
112
154
|
// 4. Post-generation guard on answer
|
|
@@ -118,19 +160,39 @@ class GuardrailsPipeline extends EventEmitter {
|
|
|
118
160
|
this.emit('postGeneration', guardrails.postGeneration);
|
|
119
161
|
|
|
120
162
|
if (!guardrails.postGeneration.safe && this.config.strict) {
|
|
163
|
+
// Do not spread result: it carries the raw answer and citations.
|
|
121
164
|
return {
|
|
122
|
-
...result,
|
|
123
165
|
success: false,
|
|
124
166
|
error: 'Post-generation guard flagged output',
|
|
125
167
|
guardrails,
|
|
126
168
|
};
|
|
127
169
|
}
|
|
128
170
|
result.answer = guardrails.postGeneration.sanitizedOutput;
|
|
171
|
+
this._redactDeep(result.citations);
|
|
172
|
+
this._redactDeep(result.evaluation);
|
|
129
173
|
}
|
|
130
174
|
|
|
131
175
|
result.guardrails = guardrails;
|
|
132
176
|
return result;
|
|
133
177
|
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Citations and evaluation details copy sentences from the raw answer, so
|
|
181
|
+
* scrub every string in them with the same PII patterns as the answer.
|
|
182
|
+
*
|
|
183
|
+
* @param {*} value - Object, array or string to redact in place
|
|
184
|
+
* @returns {*} The redacted value
|
|
185
|
+
* @private
|
|
186
|
+
*/
|
|
187
|
+
_redactDeep(value) {
|
|
188
|
+
if (typeof value === 'string') return this.postGuard.redact(value);
|
|
189
|
+
if (value && typeof value === 'object') {
|
|
190
|
+
for (const key of Object.keys(value)) {
|
|
191
|
+
value[key] = this._redactDeep(value[key]);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return value;
|
|
195
|
+
}
|
|
134
196
|
}
|
|
135
197
|
|
|
136
198
|
module.exports = { GuardrailsPipeline, DEFAULT_CONFIG };
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Prompt-injection detection shared by PreRetrievalGuard (user queries) and
|
|
5
|
+
* RetrievalGuard (retrieved documents, opt-in).
|
|
6
|
+
*
|
|
7
|
+
* Text is normalised before matching so trivial obfuscation does not bypass
|
|
8
|
+
* the patterns: Unicode NFKC, removal of zero-width and format characters,
|
|
9
|
+
* control characters turned into spaces, diacritics stripped, common
|
|
10
|
+
* Cyrillic/Greek homoglyphs and leetspeak folded to ASCII letters, and
|
|
11
|
+
* whitespace collapsed. Detection never rewrites the caller's text.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Build a character class from code points / [start, end] ranges, so the
|
|
16
|
+
* source stays ASCII-only.
|
|
17
|
+
* @param {Array<number|[number, number]>} items
|
|
18
|
+
* @returns {string}
|
|
19
|
+
*/
|
|
20
|
+
function charClass(items) {
|
|
21
|
+
const hex = (cp) => '\\u' + cp.toString(16).padStart(4, '0');
|
|
22
|
+
return items
|
|
23
|
+
.map((it) => (Array.isArray(it) ? `${hex(it[0])}-${hex(it[1])}` : hex(it)))
|
|
24
|
+
.join('');
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Zero-width / invisible format characters removed before matching. */
|
|
28
|
+
const INVISIBLE_RE = new RegExp(
|
|
29
|
+
`[${charClass([
|
|
30
|
+
0x00ad, // soft hyphen
|
|
31
|
+
0x034f, // combining grapheme joiner
|
|
32
|
+
0x061c, // arabic letter mark
|
|
33
|
+
0x115f,
|
|
34
|
+
0x1160, // hangul fillers
|
|
35
|
+
0x17b4,
|
|
36
|
+
0x17b5,
|
|
37
|
+
[0x180b, 0x180f], // mongolian variation selectors / vowel separator
|
|
38
|
+
[0x200b, 0x200f], // zero-width space/joiners, LRM/RLM
|
|
39
|
+
[0x202a, 0x202e], // bidi embeddings/overrides
|
|
40
|
+
[0x2060, 0x206f], // word joiner, invisible operators, bidi isolates
|
|
41
|
+
0x3164, // hangul filler
|
|
42
|
+
[0xfe00, 0xfe0f], // variation selectors
|
|
43
|
+
0xfeff, // BOM / zero-width no-break space
|
|
44
|
+
0xffa0, // halfwidth hangul filler
|
|
45
|
+
])}]`,
|
|
46
|
+
'g',
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
/** C0/C1 control characters (other than whitespace), replaced by a space. */
|
|
50
|
+
const CONTROL_RE = new RegExp(
|
|
51
|
+
`[${charClass([[0x00, 0x08], 0x0b, 0x0c, [0x0e, 0x1f], [0x7f, 0x9f]])}]`,
|
|
52
|
+
'g',
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
/** Cyrillic and Greek letters that look like Latin letters (code point -> letter). */
|
|
56
|
+
const HOMOGLYPH_CODEPOINTS = [
|
|
57
|
+
// Cyrillic
|
|
58
|
+
[0x0430, 'a'],
|
|
59
|
+
[0x0432, 'b'],
|
|
60
|
+
[0x0435, 'e'],
|
|
61
|
+
[0x0451, 'e'],
|
|
62
|
+
[0x043a, 'k'],
|
|
63
|
+
[0x043c, 'm'],
|
|
64
|
+
[0x043d, 'h'],
|
|
65
|
+
[0x043e, 'o'],
|
|
66
|
+
[0x0440, 'p'],
|
|
67
|
+
[0x0441, 'c'],
|
|
68
|
+
[0x0442, 't'],
|
|
69
|
+
[0x0443, 'y'],
|
|
70
|
+
[0x0445, 'x'],
|
|
71
|
+
[0x0455, 's'],
|
|
72
|
+
[0x0456, 'i'],
|
|
73
|
+
[0x0457, 'i'],
|
|
74
|
+
[0x0458, 'j'],
|
|
75
|
+
[0x0501, 'd'],
|
|
76
|
+
[0x051b, 'q'],
|
|
77
|
+
[0x051d, 'w'],
|
|
78
|
+
[0x04cf, 'l'],
|
|
79
|
+
[0x0261, 'g'],
|
|
80
|
+
// Greek
|
|
81
|
+
[0x03b1, 'a'],
|
|
82
|
+
[0x03b2, 'b'],
|
|
83
|
+
[0x03b5, 'e'],
|
|
84
|
+
[0x03b7, 'n'],
|
|
85
|
+
[0x03b9, 'i'],
|
|
86
|
+
[0x03ba, 'k'],
|
|
87
|
+
[0x03bd, 'v'],
|
|
88
|
+
[0x03bf, 'o'],
|
|
89
|
+
[0x03c1, 'p'],
|
|
90
|
+
[0x03c4, 't'],
|
|
91
|
+
[0x03c5, 'u'],
|
|
92
|
+
[0x03c7, 'x'],
|
|
93
|
+
[0x03c9, 'w'],
|
|
94
|
+
[0x03f2, 'c'],
|
|
95
|
+
];
|
|
96
|
+
const HOMOGLYPHS = Object.fromEntries(
|
|
97
|
+
HOMOGLYPH_CODEPOINTS.map(([cp, letter]) => [
|
|
98
|
+
String.fromCodePoint(cp),
|
|
99
|
+
letter,
|
|
100
|
+
]),
|
|
101
|
+
);
|
|
102
|
+
|
|
103
|
+
/** Leetspeak substitutions folded to letters (only in the folded copy). */
|
|
104
|
+
const LEET = {
|
|
105
|
+
0: 'o',
|
|
106
|
+
1: 'i',
|
|
107
|
+
3: 'e',
|
|
108
|
+
4: 'a',
|
|
109
|
+
5: 's',
|
|
110
|
+
7: 't',
|
|
111
|
+
8: 'b',
|
|
112
|
+
'@': 'a',
|
|
113
|
+
$: 's',
|
|
114
|
+
'!': 'i',
|
|
115
|
+
'|': 'l',
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Normalise text for matching, without leetspeak/homoglyph folding.
|
|
120
|
+
* Case is preserved so case-sensitive custom patterns still work.
|
|
121
|
+
* @param {string} text
|
|
122
|
+
* @returns {string}
|
|
123
|
+
*/
|
|
124
|
+
function normalizeText(text) {
|
|
125
|
+
return String(text)
|
|
126
|
+
.normalize('NFKC')
|
|
127
|
+
.replace(INVISIBLE_RE, '')
|
|
128
|
+
.replace(CONTROL_RE, ' ')
|
|
129
|
+
.normalize('NFD')
|
|
130
|
+
.replace(/\p{Mn}+/gu, '')
|
|
131
|
+
.normalize('NFC')
|
|
132
|
+
.replace(/[^\S\n]+/g, ' ')
|
|
133
|
+
.replace(/ ?\n\s*/g, '\n')
|
|
134
|
+
.trim();
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Fully folded, lower-case form used by the built-in patterns.
|
|
139
|
+
* @param {string} text
|
|
140
|
+
* @returns {string}
|
|
141
|
+
*/
|
|
142
|
+
function foldText(text) {
|
|
143
|
+
const lower = normalizeText(text).toLowerCase();
|
|
144
|
+
let out = '';
|
|
145
|
+
for (const ch of lower) {
|
|
146
|
+
out += HOMOGLYPHS[ch] || LEET[ch] || ch;
|
|
147
|
+
}
|
|
148
|
+
// Separator tricks such as "i.g.n.o.r.e" or "i-g-n-o-r-e": join single
|
|
149
|
+
// letters separated by one punctuation/space character.
|
|
150
|
+
return out.replace(/\b(?:[a-z][.\-_* ]){3,}[a-z]\b/g, (m) =>
|
|
151
|
+
m.replace(/[.\-_* ]/g, ''),
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Built-in injection patterns. They run against the folded, lower-case text,
|
|
157
|
+
* so they are written in lower case and must stay narrow enough not to flag
|
|
158
|
+
* ordinary questions (e.g. "How does Redis act as a cache?").
|
|
159
|
+
* @type {Array<{ id: string, pattern: RegExp }>}
|
|
160
|
+
*/
|
|
161
|
+
const DEFAULT_RULES = [
|
|
162
|
+
{
|
|
163
|
+
id: 'ignore-instructions',
|
|
164
|
+
pattern:
|
|
165
|
+
/\b(?:ignore|disregard|forget|skip|override|bypass)\s+(?:all\s+|any\s+|the\s+|your\s+|my\s+|of\s+)*(?:previous|prior|above|preceding|earlier|former|original|system|initial)\s+(?:instructions?|prompts?|directions?|directives?|rules|guidelines|context|messages?)\b/,
|
|
166
|
+
},
|
|
167
|
+
{
|
|
168
|
+
id: 'forget-everything',
|
|
169
|
+
pattern:
|
|
170
|
+
/\bforget\s+(?:everything|all)\s+(?:you(?:'ve|\s+have|\s+were|\s+know|\s+learned)|(?:previous|prior|above|before|that\s+came))/,
|
|
171
|
+
},
|
|
172
|
+
{
|
|
173
|
+
id: 'role-override',
|
|
174
|
+
pattern:
|
|
175
|
+
/\byou\s+are\s+now\s+(?:an?\s+|my\s+|the\s+)?(?:[a-z'-]+\s+){0,2}(?:ai|assistant|bot|chatbot|model|llm|dan|hacker|unrestricted|unfiltered|uncensored|jailbroken|evil|persona|character)\b|\byou\s+are\s+now\s+(?:in\s+)?(?:developer|dan|god|admin|jailbreak|unrestricted)\s+mode\b|\byou\s+are\s+no\s+longer\s+(?:an?\s+)?(?:ai|assistant|bound|restricted|limited|chatgpt|claude)\b|\byou\s+are\s+dan\b/,
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
// "act as" only in imperative position: start of text/sentence, or after
|
|
179
|
+
// please/now/you must/will/should. "Redis acts as a cache" is not flagged.
|
|
180
|
+
id: 'act-as',
|
|
181
|
+
pattern:
|
|
182
|
+
/(?:^|[.!?:;\n]\s*|\b(?:please|now|you\s+(?:must|will|should|shall|can|are\s+to)|i\s+want\s+you\s+to)\s+)act\s+as\s+(?:if\s+you\b|an?\s+|my\s+|the\s+|though\b)/,
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
id: 'pretend',
|
|
186
|
+
pattern:
|
|
187
|
+
/\bpretend\s+(?:that\s+)?(?:you\s+are|you're|youre|you\s+have\s+no|to\s+be\s+(?:an?\s+)?(?:\w+\s+)?(?:ai|assistant|model|bot|chatbot|dan|unrestricted|unfiltered|uncensored|evil|hacker|jailbroken))\b/,
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
// Chat-template role markers: "system:" as a line prefix, [system],
|
|
191
|
+
// <|im_start|>system, [INST], <<SYS>>, "### system".
|
|
192
|
+
id: 'role-marker',
|
|
193
|
+
pattern:
|
|
194
|
+
/(?:^|\n)\s*(?:system|assistant|developer)\s*:|\[\s*(?:system|inst)\s*\]|<\|?\s*(?:im_start|system)\s*\|?>|<<\s*sys\s*>>|#{2,}\s*(?:system|instruction)s?\b/,
|
|
195
|
+
},
|
|
196
|
+
{
|
|
197
|
+
id: 'dan',
|
|
198
|
+
pattern: /\bdan\s+(?:mode|prompt|jailbreak)\b|\bdo\s+anything\s+now\b/,
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
id: 'bypass-restrictions',
|
|
202
|
+
pattern:
|
|
203
|
+
/\b(?:bypass|circumvent|evade)\s+(?:all\s+|any\s+|your\s+|the\s+|of\s+)*(?:ai\s+|safety\s+|content\s+|ethical\s+)?(?:restrictions|guardrails|safeguards|safety\s+(?:rules|measures|guidelines|filters?)|content\s+(?:polic(?:y|ies)|filters?)|moderation)\b|\b(?:disable|ignore|remove|turn\s+off|drop)\s+(?:all\s+(?:of\s+)?)?your\s+(?:safety\s+|content\s+|ethical\s+)?(?:restrictions|filters?|guardrails|safeguards|safety|rules|guidelines|programming)\b/,
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
id: 'override-safety',
|
|
207
|
+
pattern:
|
|
208
|
+
/\boverride\s+(?:your\s+|the\s+|all\s+)?(?:safety|content\s+(?:filters?|polic(?:y|ies))|system\s+(?:prompt|instructions?))\b/,
|
|
209
|
+
},
|
|
210
|
+
{
|
|
211
|
+
id: 'jailbreak',
|
|
212
|
+
pattern:
|
|
213
|
+
/\bjailbr(?:ea|o)k(?:ing|en)?\s+(?:the\s+|this\s+|your\s+|an?\s+)?(?:model|ai|assistant|llm|chatbot|gpt|claude|system|prompt|you)\b|\bjailbreak\s+(?:mode|prompt)\b|\byou\s+are\s+jailbroken\b/,
|
|
214
|
+
},
|
|
215
|
+
{
|
|
216
|
+
id: 'prompt-exfiltration',
|
|
217
|
+
pattern:
|
|
218
|
+
/\b(?:reveal|show|print|repeat|output|leak|display|tell\s+me)\s+(?:me\s+)?(?:your|the)\s+(?:system\s+prompt|hidden\s+(?:prompt|instructions)|initial\s+(?:prompt|instructions)|original\s+(?:prompt|instructions)|instructions\s+above)\b/,
|
|
219
|
+
},
|
|
220
|
+
];
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Prepare a user-supplied pattern for repeated .test() calls: a global or
|
|
224
|
+
* sticky flag would keep lastIndex state between calls.
|
|
225
|
+
* @param {RegExp|string} pattern
|
|
226
|
+
* @returns {RegExp}
|
|
227
|
+
*/
|
|
228
|
+
function toStatelessRegExp(pattern) {
|
|
229
|
+
if (pattern instanceof RegExp) {
|
|
230
|
+
if (!pattern.global && !pattern.sticky) return pattern;
|
|
231
|
+
return new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, ''));
|
|
232
|
+
}
|
|
233
|
+
return new RegExp(String(pattern), 'i');
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Detect prompt-injection attempts in text.
|
|
238
|
+
*
|
|
239
|
+
* Built-in rules run on the folded text. Custom patterns run on the
|
|
240
|
+
* normalised text (case preserved) and on the folded text.
|
|
241
|
+
*
|
|
242
|
+
* @param {string} text
|
|
243
|
+
* @param {object} [opts]
|
|
244
|
+
* @param {Array<RegExp|string>} [opts.patterns=[]] - Extra patterns
|
|
245
|
+
* @param {boolean} [opts.useDefaults=true] - Include the built-in rules
|
|
246
|
+
* @returns {{ detected: boolean, matches: Array<{ id: string, pattern: string, match: string }>, normalized: string }}
|
|
247
|
+
*/
|
|
248
|
+
function detectInjection(text, opts = {}) {
|
|
249
|
+
const normalized = normalizeText(text == null ? '' : text);
|
|
250
|
+
const lower = normalized.toLowerCase();
|
|
251
|
+
const folded = foldText(normalized);
|
|
252
|
+
const matches = [];
|
|
253
|
+
|
|
254
|
+
if (opts.useDefaults !== false) {
|
|
255
|
+
for (const rule of DEFAULT_RULES) {
|
|
256
|
+
const m = rule.pattern.exec(lower) || rule.pattern.exec(folded);
|
|
257
|
+
if (m) {
|
|
258
|
+
matches.push({
|
|
259
|
+
id: rule.id,
|
|
260
|
+
pattern: String(rule.pattern),
|
|
261
|
+
match: m[0].trim(),
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
for (const raw of opts.patterns || []) {
|
|
268
|
+
const re = toStatelessRegExp(raw);
|
|
269
|
+
const m = re.exec(normalized) || re.exec(folded);
|
|
270
|
+
if (m) {
|
|
271
|
+
matches.push({ id: 'custom', pattern: String(raw), match: m[0].trim() });
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
return { detected: matches.length > 0, matches, normalized };
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
module.exports = {
|
|
279
|
+
detectInjection,
|
|
280
|
+
normalizeText,
|
|
281
|
+
foldText,
|
|
282
|
+
DEFAULT_RULES,
|
|
283
|
+
};
|
|
@@ -165,6 +165,25 @@ class PostGenerationGuard extends EventEmitter {
|
|
|
165
165
|
this.emit('checked', result);
|
|
166
166
|
return result;
|
|
167
167
|
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Replace PII matches in arbitrary text with [REDACTED], using the same
|
|
171
|
+
* patterns as check(). Used to scrub text derived from the answer, such as
|
|
172
|
+
* citation sentences.
|
|
173
|
+
*
|
|
174
|
+
* @param {string} text
|
|
175
|
+
* @returns {string}
|
|
176
|
+
*/
|
|
177
|
+
redact(text) {
|
|
178
|
+
if (typeof text !== 'string' || !this.config.enablePIIDetection) {
|
|
179
|
+
return text;
|
|
180
|
+
}
|
|
181
|
+
let redacted = text;
|
|
182
|
+
for (const pattern of Object.values(this.config.piiPatterns)) {
|
|
183
|
+
redacted = redacted.replace(pattern, '[REDACTED]');
|
|
184
|
+
}
|
|
185
|
+
return redacted;
|
|
186
|
+
}
|
|
168
187
|
}
|
|
169
188
|
|
|
170
189
|
module.exports = { PostGenerationGuard, DEFAULT_CONFIG };
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
const { EventEmitter } = require('events');
|
|
4
4
|
const { validateInput } = require('../security/validator');
|
|
5
5
|
const { tokenize } = require('../evaluate/scoring');
|
|
6
|
+
const { detectInjection, DEFAULT_RULES } = require('./injection-detector');
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* Default configuration for the PreRetrievalGuard.
|
|
@@ -14,18 +15,10 @@ const DEFAULT_CONFIG = {
|
|
|
14
15
|
enableTopicFiltering: false,
|
|
15
16
|
blockedTopics: [],
|
|
16
17
|
allowedTopics: [],
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
/act\s+as\s+(a|an)?\s*/i,
|
|
22
|
-
/pretend\s+(you\s+are|to\s+be)/i,
|
|
23
|
-
/system\s*:/i,
|
|
24
|
-
/\bDAN\b/,
|
|
25
|
-
/bypass\s+(all\s+)?restrictions/i,
|
|
26
|
-
/override\s+(safety|content|system)/i,
|
|
27
|
-
/jailbreak/i,
|
|
28
|
-
],
|
|
18
|
+
// Built-in rules (see ./injection-detector). They match against normalised
|
|
19
|
+
// text (NFKC, zero-width/control characters removed, homoglyph and
|
|
20
|
+
// leetspeak folding). options.injectionPatterns adds extra patterns.
|
|
21
|
+
injectionPatterns: DEFAULT_RULES.map((rule) => rule.pattern),
|
|
29
22
|
};
|
|
30
23
|
|
|
31
24
|
/**
|
|
@@ -48,6 +41,10 @@ class PreRetrievalGuard extends EventEmitter {
|
|
|
48
41
|
constructor(options = {}) {
|
|
49
42
|
super();
|
|
50
43
|
this.config = { ...DEFAULT_CONFIG, ...options };
|
|
44
|
+
/** @private Extra patterns supplied by the caller */
|
|
45
|
+
this._customPatterns = options.injectionPatterns
|
|
46
|
+
? [...options.injectionPatterns]
|
|
47
|
+
: [];
|
|
51
48
|
if (options.injectionPatterns) {
|
|
52
49
|
this.config.injectionPatterns = [
|
|
53
50
|
...DEFAULT_CONFIG.injectionPatterns,
|
|
@@ -63,8 +60,13 @@ class PreRetrievalGuard extends EventEmitter {
|
|
|
63
60
|
* @returns {{
|
|
64
61
|
* safe: boolean,
|
|
65
62
|
* issues: string[],
|
|
66
|
-
* sanitizedQuery: string
|
|
63
|
+
* sanitizedQuery: string,
|
|
64
|
+
* injection: { detected: boolean, matches: Array<{id: string, pattern: string, match: string}> }
|
|
67
65
|
* }}
|
|
66
|
+
* `sanitizedQuery` is the query truncated to maxQueryLength; it is never
|
|
67
|
+
* rewritten. Injection findings are reported in `issues` and `injection`
|
|
68
|
+
* and make `safe` false; blocking is up to the caller (GuardrailsPipeline
|
|
69
|
+
* blocks in strict mode).
|
|
68
70
|
* @fires PreRetrievalGuard#checked
|
|
69
71
|
*/
|
|
70
72
|
check(query) {
|
|
@@ -77,7 +79,12 @@ class PreRetrievalGuard extends EventEmitter {
|
|
|
77
79
|
query.trim().length === 0
|
|
78
80
|
) {
|
|
79
81
|
issues.push('Query must be a non-empty string');
|
|
80
|
-
const result = {
|
|
82
|
+
const result = {
|
|
83
|
+
safe: false,
|
|
84
|
+
issues,
|
|
85
|
+
sanitizedQuery: '',
|
|
86
|
+
injection: { detected: false, matches: [] },
|
|
87
|
+
};
|
|
81
88
|
this.emit('checked', result);
|
|
82
89
|
return result;
|
|
83
90
|
}
|
|
@@ -92,16 +99,22 @@ class PreRetrievalGuard extends EventEmitter {
|
|
|
92
99
|
}
|
|
93
100
|
}
|
|
94
101
|
|
|
95
|
-
|
|
102
|
+
// The query is passed on unchanged (apart from length truncation).
|
|
103
|
+
// Deleting only the matched phrase used to leave the rest of an attack
|
|
104
|
+
// payload in place, so detections are reported instead.
|
|
105
|
+
const sanitizedQuery = validation.sanitized;
|
|
96
106
|
|
|
97
|
-
// 3. Injection detection
|
|
98
|
-
|
|
107
|
+
// 3. Injection detection on a normalised copy of the query
|
|
108
|
+
let injection = { detected: false, matches: [] };
|
|
99
109
|
if (this.config.enableInjectionDetection) {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
110
|
+
const found = detectInjection(sanitizedQuery, {
|
|
111
|
+
patterns: this._customPatterns,
|
|
112
|
+
});
|
|
113
|
+
injection = { detected: found.detected, matches: found.matches };
|
|
114
|
+
for (const m of found.matches) {
|
|
115
|
+
issues.push(
|
|
116
|
+
`Potential prompt injection detected: ${m.id} (${JSON.stringify(m.match)})`,
|
|
117
|
+
);
|
|
105
118
|
}
|
|
106
119
|
}
|
|
107
120
|
|
|
@@ -133,24 +146,11 @@ class PreRetrievalGuard extends EventEmitter {
|
|
|
133
146
|
}
|
|
134
147
|
}
|
|
135
148
|
|
|
136
|
-
// 5. Sanitize: strip matched injection patterns from query
|
|
137
|
-
if (matchedPatterns.length > 0) {
|
|
138
|
-
for (const pattern of matchedPatterns) {
|
|
139
|
-
// Create a global version of the pattern for replacement
|
|
140
|
-
const flags = pattern.flags.includes('g')
|
|
141
|
-
? pattern.flags
|
|
142
|
-
: pattern.flags + 'g';
|
|
143
|
-
const globalPattern = new RegExp(pattern.source, flags);
|
|
144
|
-
sanitizedQuery = sanitizedQuery.replace(globalPattern, '').trim();
|
|
145
|
-
}
|
|
146
|
-
// Collapse multiple spaces left after stripping
|
|
147
|
-
sanitizedQuery = sanitizedQuery.replace(/\s{2,}/g, ' ').trim();
|
|
148
|
-
}
|
|
149
|
-
|
|
150
149
|
const result = {
|
|
151
150
|
safe: issues.length === 0,
|
|
152
151
|
issues,
|
|
153
152
|
sanitizedQuery,
|
|
153
|
+
injection,
|
|
154
154
|
};
|
|
155
155
|
|
|
156
156
|
this.emit('checked', result);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
const { EventEmitter } = require('events');
|
|
4
|
+
const { detectInjection } = require('./injection-detector');
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* Default configuration for the RetrievalGuard.
|
|
@@ -10,6 +11,12 @@ const DEFAULT_CONFIG = {
|
|
|
10
11
|
minRelevanceScore: 0,
|
|
11
12
|
maxAgeDays: 0, // 0 = disabled
|
|
12
13
|
enforceACL: false,
|
|
14
|
+
// Opt-in scan of retrieved document text for prompt injection (indirect
|
|
15
|
+
// injection). 'remove' drops flagged documents, 'flag' keeps them and
|
|
16
|
+
// marks them with `injection` details.
|
|
17
|
+
detectInjection: false,
|
|
18
|
+
injectionAction: 'remove',
|
|
19
|
+
injectionPatterns: [],
|
|
13
20
|
};
|
|
14
21
|
|
|
15
22
|
/**
|
|
@@ -25,6 +32,9 @@ class RetrievalGuard extends EventEmitter {
|
|
|
25
32
|
* @param {number} [options.minRelevanceScore=0] - Minimum relevance score threshold
|
|
26
33
|
* @param {number} [options.maxAgeDays=0] - Maximum document age in days (0 = disabled)
|
|
27
34
|
* @param {boolean} [options.enforceACL=false] - Whether to enforce access control
|
|
35
|
+
* @param {boolean} [options.detectInjection=false] - Scan document text for prompt injection
|
|
36
|
+
* @param {'remove'|'flag'} [options.injectionAction='remove'] - Remove flagged documents or keep and mark them
|
|
37
|
+
* @param {RegExp[]} [options.injectionPatterns=[]] - Extra injection patterns for the scan
|
|
28
38
|
*/
|
|
29
39
|
constructor(options = {}) {
|
|
30
40
|
super();
|
|
@@ -86,17 +96,19 @@ class RetrievalGuard extends EventEmitter {
|
|
|
86
96
|
}
|
|
87
97
|
}
|
|
88
98
|
|
|
89
|
-
// 3. ACL filter
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
99
|
+
// 3. ACL filter. Missing userPermissions fails closed: the user is
|
|
100
|
+
// treated as having no roles, so restricted documents are removed.
|
|
101
|
+
if (!dominated && this.config.enforceACL) {
|
|
102
|
+
const userPermissions = context.userPermissions || [];
|
|
103
|
+
// Read the restriction from the result itself or its metadata, so a
|
|
104
|
+
// top-level `acl` is not ignored when the result also has metadata.
|
|
105
|
+
const restriction =
|
|
106
|
+
result.requiredRole ?? result.acl ?? doc.requiredRole ?? doc.acl;
|
|
107
|
+
const requiredRoles = restriction ? [].concat(restriction) : null;
|
|
96
108
|
|
|
97
109
|
if (requiredRoles && requiredRoles.length > 0) {
|
|
98
110
|
const hasAccess = requiredRoles.some((role) =>
|
|
99
|
-
|
|
111
|
+
userPermissions.includes(role),
|
|
100
112
|
);
|
|
101
113
|
if (!hasAccess) {
|
|
102
114
|
dominated = true;
|
|
@@ -105,6 +117,25 @@ class RetrievalGuard extends EventEmitter {
|
|
|
105
117
|
}
|
|
106
118
|
}
|
|
107
119
|
|
|
120
|
+
// 4. Indirect prompt injection in document text (opt-in)
|
|
121
|
+
if (!dominated && this.config.detectInjection) {
|
|
122
|
+
const text = documentText(result);
|
|
123
|
+
const found = detectInjection(text, {
|
|
124
|
+
patterns: this.config.injectionPatterns,
|
|
125
|
+
});
|
|
126
|
+
if (found.detected) {
|
|
127
|
+
reasons.injection = (reasons.injection || 0) + 1;
|
|
128
|
+
if (this.config.injectionAction === 'flag') {
|
|
129
|
+
kept.push({
|
|
130
|
+
...result,
|
|
131
|
+
injection: { detected: true, matches: found.matches },
|
|
132
|
+
});
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
dominated = true;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
108
139
|
if (dominated) {
|
|
109
140
|
removed.push(result);
|
|
110
141
|
} else {
|
|
@@ -123,4 +154,23 @@ class RetrievalGuard extends EventEmitter {
|
|
|
123
154
|
}
|
|
124
155
|
}
|
|
125
156
|
|
|
157
|
+
/**
|
|
158
|
+
* Text of a retrieval result for injection scanning.
|
|
159
|
+
* @param {object|string} result
|
|
160
|
+
* @returns {string}
|
|
161
|
+
*/
|
|
162
|
+
function documentText(result) {
|
|
163
|
+
if (typeof result === 'string') return result;
|
|
164
|
+
if (!result || typeof result !== 'object') return '';
|
|
165
|
+
const doc = result.doc && typeof result.doc === 'object' ? result.doc : {};
|
|
166
|
+
return String(
|
|
167
|
+
result.content ||
|
|
168
|
+
result.text ||
|
|
169
|
+
result.pageContent ||
|
|
170
|
+
doc.content ||
|
|
171
|
+
doc.text ||
|
|
172
|
+
'',
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
|
|
126
176
|
module.exports = { RetrievalGuard, DEFAULT_CONFIG };
|