agentshield-sdk 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +191 -0
- package/LICENSE +21 -0
- package/README.md +975 -0
- package/bin/agent-shield.js +680 -0
- package/package.json +118 -0
- package/src/adaptive.js +330 -0
- package/src/agent-protocol.js +998 -0
- package/src/alert-tuning.js +480 -0
- package/src/allowlist.js +603 -0
- package/src/audit-immutable.js +914 -0
- package/src/audit-streaming.js +469 -0
- package/src/badges.js +196 -0
- package/src/behavior-profiling.js +289 -0
- package/src/benchmark-harness.js +804 -0
- package/src/canary.js +271 -0
- package/src/certification.js +563 -0
- package/src/circuit-breaker.js +321 -0
- package/src/compliance.js +617 -0
- package/src/confidence-tuning.js +324 -0
- package/src/confused-deputy.js +624 -0
- package/src/context-scoring.js +360 -0
- package/src/conversation.js +494 -0
- package/src/cost-optimizer.js +1024 -0
- package/src/ctf.js +462 -0
- package/src/detector-core.js +1999 -0
- package/src/distributed.js +359 -0
- package/src/document-scanner.js +795 -0
- package/src/embedding.js +307 -0
- package/src/encoding.js +429 -0
- package/src/enterprise.js +405 -0
- package/src/errors.js +100 -0
- package/src/eu-ai-act.js +523 -0
- package/src/fuzzer.js +764 -0
- package/src/honeypot.js +328 -0
- package/src/i18n-patterns.js +523 -0
- package/src/index.js +430 -0
- package/src/integrations.js +528 -0
- package/src/llm-redteam.js +670 -0
- package/src/main.js +741 -0
- package/src/main.mjs +38 -0
- package/src/mcp-bridge.js +542 -0
- package/src/mcp-certification.js +846 -0
- package/src/mcp-sdk-integration.js +355 -0
- package/src/mcp-security-runtime.js +741 -0
- package/src/mcp-server.js +740 -0
- package/src/middleware.js +208 -0
- package/src/model-finetuning.js +884 -0
- package/src/model-fingerprint.js +1042 -0
- package/src/multi-agent-trust.js +453 -0
- package/src/multi-agent.js +404 -0
- package/src/multimodal.js +296 -0
- package/src/nist-mapping.js +505 -0
- package/src/observability.js +330 -0
- package/src/openclaw.js +450 -0
- package/src/otel.js +544 -0
- package/src/owasp-2025.js +483 -0
- package/src/pii.js +390 -0
- package/src/plugin-marketplace.js +628 -0
- package/src/plugin-system.js +349 -0
- package/src/policy-dsl.js +775 -0
- package/src/policy-extended.js +635 -0
- package/src/policy.js +443 -0
- package/src/presets.js +409 -0
- package/src/production.js +557 -0
- package/src/prompt-leakage.js +321 -0
- package/src/rag-vulnerability.js +579 -0
- package/src/redteam.js +475 -0
- package/src/response-handler.js +429 -0
- package/src/scanners.js +357 -0
- package/src/self-healing.js +363 -0
- package/src/semantic.js +339 -0
- package/src/shield-score.js +250 -0
- package/src/sso-saml.js +897 -0
- package/src/stream-scanner.js +806 -0
- package/src/testing.js +505 -0
- package/src/threat-encyclopedia.js +629 -0
- package/src/threat-intel-network.js +1017 -0
- package/src/token-analysis.js +467 -0
- package/src/tool-guard.js +412 -0
- package/src/tool-output-validator.js +354 -0
- package/src/utils.js +83 -0
- package/src/watermark.js +235 -0
- package/src/worker-scanner.js +601 -0
- package/types/index.d.ts +2088 -0
|
@@ -0,0 +1,579 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Agent Shield — RAG/Vector Vulnerability Scanner (OWASP LLM08-2025)
|
|
5
|
+
*
|
|
6
|
+
* Detects vulnerabilities in RAG (Retrieval-Augmented Generation) systems
|
|
7
|
+
* including embedding manipulation, chunk boundary attacks, retrieval
|
|
8
|
+
* poisoning, and context window stuffing.
|
|
9
|
+
*
|
|
10
|
+
* All processing runs locally — no data ever leaves your environment.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
// =========================================================================
|
|
14
|
+
// RAG vulnerability patterns
|
|
15
|
+
// =========================================================================
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Patterns for RAG-specific attack vectors.
|
|
19
|
+
* @type {Array<object>}
|
|
20
|
+
*/
|
|
21
|
+
const RAG_VULNERABILITY_PATTERNS = [
|
|
22
|
+
// Chunk boundary manipulation
|
|
23
|
+
{ pattern: /(?:\n{5,}|\r\n{5,}).{0,500}(?:ignore|override|forget|system|assistant)/is, severity: 'high', category: 'chunk_boundary', technique: 'boundary_manipulation', description: 'Whitespace padding to manipulate chunk boundaries', mitigation: 'Normalize whitespace before chunking' },
|
|
24
|
+
{ pattern: /[\x00-\x08\x0B\x0C\x0E-\x1F]{3,}.{0,500}(?:ignore|override|system)/is, severity: 'high', category: 'chunk_boundary', technique: 'boundary_manipulation', description: 'Control characters to split across chunk boundaries', mitigation: 'Strip control characters before chunking' },
|
|
25
|
+
|
|
26
|
+
// Metadata injection
|
|
27
|
+
{ pattern: /(?:title|author|subject|keywords|description)\s*[:=]\s*.{0,200}(?:ignore|override|forget|system\s+prompt)/i, severity: 'high', category: 'metadata_injection', technique: 'metadata_poisoning', description: 'Injection payload in document metadata fields', mitigation: 'Sanitize document metadata before indexing' },
|
|
28
|
+
{ pattern: /(?:source|origin|ref|citation)\s*[:=]\s*(?:system|admin|internal|trusted)/i, severity: 'medium', category: 'metadata_injection', technique: 'authority_spoofing', description: 'Spoofed authority metadata to increase trust', mitigation: 'Do not use metadata for trust decisions' },
|
|
29
|
+
|
|
30
|
+
// Cross-document injection
|
|
31
|
+
{ pattern: /(?:regarding|about|in\s+reference\s+to|see\s+also)\s+(?:document|file|page)\s+.{0,30}(?:ignore|override|forget|instead)/i, severity: 'high', category: 'cross_document', technique: 'cross_doc_injection', description: 'Cross-document reference with injection payload', mitigation: 'Isolate document contexts during retrieval' },
|
|
32
|
+
{ pattern: /(?:the\s+(?:previous|following|other|related)\s+(?:document|source|context))\s+(?:says|states|instructs|requires)\s+(?:you\s+)?(?:to|should|must)/i, severity: 'high', category: 'cross_document', technique: 'cross_doc_authority', description: 'Claimed authority from other documents', mitigation: 'Treat each retrieved document independently' },
|
|
33
|
+
|
|
34
|
+
// Retrieval poisoning
|
|
35
|
+
{ pattern: /(?:\S+\s+){15,}(?:ignore|override|forget)/i, severity: 'medium', category: 'retrieval_poisoning', technique: 'keyword_stuffing', description: 'Keyword stuffing to influence retrieval ranking', mitigation: 'Use semantic similarity, not just keyword matching' },
|
|
36
|
+
|
|
37
|
+
// Context window stuffing
|
|
38
|
+
{ pattern: /(.{50,100})\1{3,}/s, severity: 'medium', category: 'context_stuffing', technique: 'repetition_attack', description: 'Repeated content to consume context window', mitigation: 'Deduplicate retrieved content before injection' },
|
|
39
|
+
|
|
40
|
+
// Embedding space attacks
|
|
41
|
+
{ pattern: /(?:(?:the\s+){5,}|(?:a\s+){5,}|(?:is\s+){5,})/i, severity: 'medium', category: 'embedding_attack', technique: 'embedding_manipulation', description: 'Repeated tokens to manipulate embedding space position', mitigation: 'Filter documents with abnormal token distributions' },
|
|
42
|
+
|
|
43
|
+
// Hidden instructions in documents
|
|
44
|
+
{ pattern: /<!--\s*(?:system|instruction|ai|llm|gpt|claude|assistant)\s*[:>]/i, severity: 'critical', category: 'hidden_instruction', technique: 'html_comment_injection', description: 'Hidden AI instructions in HTML comments', mitigation: 'Strip HTML comments from retrieved documents' },
|
|
45
|
+
{ pattern: /\u200B[^\u200B]{10,}\u200B/i, severity: 'high', category: 'hidden_instruction', technique: 'zero_width_hiding', description: 'Content hidden between zero-width spaces', mitigation: 'Remove zero-width characters from documents' },
|
|
46
|
+
{ pattern: /<(?:div|span|p)[^>]*(?:display\s*:\s*none|visibility\s*:\s*hidden|font-size\s*:\s*0|opacity\s*:\s*0)[^>]*>/i, severity: 'critical', category: 'hidden_instruction', technique: 'css_hiding', description: 'CSS-hidden content with potential instructions', mitigation: 'Strip hidden HTML elements before indexing' }
|
|
47
|
+
];
|
|
48
|
+
|
|
49
|
+
// =========================================================================
|
|
50
|
+
// Vector DB security checklist
|
|
51
|
+
// =========================================================================
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Security checklist for popular vector databases.
|
|
55
|
+
* @type {Array<object>}
|
|
56
|
+
*/
|
|
57
|
+
const VECTOR_DB_SECURITY_CHECKLIST = [
|
|
58
|
+
// General
|
|
59
|
+
{ item: 'Enable authentication for all vector DB connections', risk: 'high', recommendation: 'Use API keys or mTLS for all connections', applies: ['all'] },
|
|
60
|
+
{ item: 'Enable TLS/SSL for data in transit', risk: 'high', recommendation: 'Encrypt all communications between app and vector DB', applies: ['all'] },
|
|
61
|
+
{ item: 'Enable encryption at rest for stored vectors', risk: 'high', recommendation: 'Use disk encryption or DB-native encryption', applies: ['all'] },
|
|
62
|
+
{ item: 'Implement access control for collections/indexes', risk: 'medium', recommendation: 'Use RBAC to restrict which services can read/write which collections', applies: ['all'] },
|
|
63
|
+
{ item: 'Rate limit vector search queries', risk: 'medium', recommendation: 'Prevent enumeration attacks via query rate limiting', applies: ['all'] },
|
|
64
|
+
{ item: 'Audit log all vector operations', risk: 'medium', recommendation: 'Log inserts, deletes, and searches for forensic analysis', applies: ['all'] },
|
|
65
|
+
{ item: 'Validate embedding dimensions before insertion', risk: 'low', recommendation: 'Reject vectors with wrong dimensions to prevent data corruption', applies: ['all'] },
|
|
66
|
+
{ item: 'Sanitize metadata before storage', risk: 'high', recommendation: 'Strip or validate all metadata fields before indexing', applies: ['all'] },
|
|
67
|
+
|
|
68
|
+
// Platform-specific
|
|
69
|
+
{ item: 'Pinecone: Use namespaces for tenant isolation', risk: 'high', recommendation: 'Separate tenants into different namespaces', applies: ['pinecone'] },
|
|
70
|
+
{ item: 'Weaviate: Configure OIDC authentication', risk: 'high', recommendation: 'Use OIDC rather than anonymous access', applies: ['weaviate'] },
|
|
71
|
+
{ item: 'Qdrant: Enable API key authentication', risk: 'high', recommendation: 'Set QDRANT__SERVICE__API_KEY', applies: ['qdrant'] },
|
|
72
|
+
{ item: 'Chroma: Use server mode with auth (not in-process)', risk: 'high', recommendation: 'Run Chroma as a server with token auth, not embedded', applies: ['chroma'] },
|
|
73
|
+
{ item: 'Milvus: Enable TLS and user authentication', risk: 'high', recommendation: 'Configure milvus.yaml with TLS and RBAC', applies: ['milvus'] },
|
|
74
|
+
{ item: 'pgvector: Use PostgreSQL roles and row-level security', risk: 'high', recommendation: 'Leverage PostgreSQL RLS for multi-tenant isolation', applies: ['pgvector'] }
|
|
75
|
+
];
|
|
76
|
+
|
|
77
|
+
// =========================================================================
|
|
78
|
+
// RAGVulnerabilityScanner
|
|
79
|
+
// =========================================================================
|
|
80
|
+
|
|
81
|
+
class RAGVulnerabilityScanner {
|
|
82
|
+
/**
|
|
83
|
+
* @param {object} [options]
|
|
84
|
+
* @param {number} [options.chunkSize=512] - Expected chunk size in tokens
|
|
85
|
+
* @param {number} [options.overlapSize=50] - Chunk overlap in tokens
|
|
86
|
+
* @param {number} [options.maxRetrievedDocs=10] - Maximum retrieved documents
|
|
87
|
+
* @param {number} [options.embeddingDimension=1536] - Embedding dimension
|
|
88
|
+
*/
|
|
89
|
+
constructor(options = {}) {
|
|
90
|
+
this.chunkSize = options.chunkSize || 512;
|
|
91
|
+
this.overlapSize = options.overlapSize || 50;
|
|
92
|
+
this.maxRetrievedDocs = options.maxRetrievedDocs || 10;
|
|
93
|
+
this.embeddingDimension = options.embeddingDimension || 1536;
|
|
94
|
+
this.stats = { chunksScanned: 0, setsScanned: 0, threatsFound: 0 };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Scans a single chunk for RAG-specific injection patterns.
|
|
99
|
+
* @param {string} chunk - Document chunk text
|
|
100
|
+
* @param {object} [metadata] - Chunk metadata
|
|
101
|
+
* @returns {{ safe: boolean, threats: Array, recommendations: Array }}
|
|
102
|
+
*/
|
|
103
|
+
scanChunk(chunk, metadata = {}) {
|
|
104
|
+
this.stats.chunksScanned++;
|
|
105
|
+
const threats = [];
|
|
106
|
+
const recommendations = new Set();
|
|
107
|
+
|
|
108
|
+
for (const vp of RAG_VULNERABILITY_PATTERNS) {
|
|
109
|
+
if (vp.pattern.test(chunk)) {
|
|
110
|
+
threats.push({
|
|
111
|
+
severity: vp.severity,
|
|
112
|
+
category: vp.category,
|
|
113
|
+
technique: vp.technique,
|
|
114
|
+
description: vp.description,
|
|
115
|
+
source: metadata.source || 'unknown'
|
|
116
|
+
});
|
|
117
|
+
recommendations.add(vp.mitigation);
|
|
118
|
+
this.stats.threatsFound++;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// Check metadata for injection
|
|
123
|
+
if (metadata) {
|
|
124
|
+
const metaStr = JSON.stringify(metadata);
|
|
125
|
+
for (const vp of RAG_VULNERABILITY_PATTERNS.filter(p => p.category === 'metadata_injection')) {
|
|
126
|
+
if (vp.pattern.test(metaStr)) {
|
|
127
|
+
threats.push({
|
|
128
|
+
severity: vp.severity,
|
|
129
|
+
category: 'metadata_injection',
|
|
130
|
+
technique: vp.technique,
|
|
131
|
+
description: `Metadata: ${vp.description}`,
|
|
132
|
+
source: metadata.source || 'unknown'
|
|
133
|
+
});
|
|
134
|
+
recommendations.add(vp.mitigation);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
return { safe: threats.length === 0, threats, recommendations: [...recommendations] };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Scans a set of retrieved chunks for cross-document attacks.
|
|
144
|
+
* @param {Array<string>} chunks - Retrieved chunks
|
|
145
|
+
* @param {string} query - Original query
|
|
146
|
+
* @returns {{ safe: boolean, threats: Array, crossDocRisks: Array }}
|
|
147
|
+
*/
|
|
148
|
+
scanRetrievalSet(chunks, query) {
|
|
149
|
+
this.stats.setsScanned++;
|
|
150
|
+
const threats = [];
|
|
151
|
+
const crossDocRisks = [];
|
|
152
|
+
|
|
153
|
+
// Scan each chunk individually
|
|
154
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
155
|
+
const result = this.scanChunk(chunks[i], { source: `chunk_${i}` });
|
|
156
|
+
threats.push(...result.threats);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Check for cross-document patterns
|
|
160
|
+
const combined = chunks.join('\n---\n');
|
|
161
|
+
for (const vp of RAG_VULNERABILITY_PATTERNS.filter(p => p.category === 'cross_document')) {
|
|
162
|
+
if (vp.pattern.test(combined)) {
|
|
163
|
+
crossDocRisks.push({
|
|
164
|
+
severity: vp.severity,
|
|
165
|
+
technique: vp.technique,
|
|
166
|
+
description: vp.description
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Check total context size vs typical window
|
|
172
|
+
const totalTokens = chunks.reduce((sum, c) => sum + c.split(/\s+/).length, 0);
|
|
173
|
+
if (totalTokens > 4000) {
|
|
174
|
+
crossDocRisks.push({
|
|
175
|
+
severity: 'medium',
|
|
176
|
+
technique: 'context_overflow',
|
|
177
|
+
description: `Retrieved content is ~${totalTokens} tokens — may push system prompt out of context window`
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Check for excessive similarity between chunks (possible stuffing)
|
|
182
|
+
if (chunks.length >= 3) {
|
|
183
|
+
const unique = new Set(chunks.map(c => c.trim().substring(0, 200)));
|
|
184
|
+
if (unique.size < chunks.length * 0.5) {
|
|
185
|
+
crossDocRisks.push({
|
|
186
|
+
severity: 'medium',
|
|
187
|
+
technique: 'retrieval_stuffing',
|
|
188
|
+
description: 'Multiple retrieved chunks are near-duplicates — possible stuffing attack'
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
return { safe: threats.length === 0 && crossDocRisks.length === 0, threats, crossDocRisks };
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Analyzes chunk boundaries for potential injection vectors.
|
|
198
|
+
* @param {Array<string>} chunks - Ordered chunks from a document
|
|
199
|
+
* @returns {{ risks: Array, recommendations: Array }}
|
|
200
|
+
*/
|
|
201
|
+
analyzeChunkBoundaries(chunks) {
|
|
202
|
+
const risks = [];
|
|
203
|
+
const recommendations = [];
|
|
204
|
+
|
|
205
|
+
for (let i = 0; i < chunks.length - 1; i++) {
|
|
206
|
+
const endOfCurrent = chunks[i].slice(-100);
|
|
207
|
+
const startOfNext = chunks[i + 1].slice(0, 100);
|
|
208
|
+
|
|
209
|
+
// Check if an instruction spans the boundary
|
|
210
|
+
const boundary = endOfCurrent + startOfNext;
|
|
211
|
+
if (/(?:ignore|override|forget|system)\s+(?:all\s+)?(?:previous|prior|instructions|rules)/i.test(boundary)) {
|
|
212
|
+
risks.push({
|
|
213
|
+
severity: 'high',
|
|
214
|
+
boundary: `chunks[${i}]-chunks[${i + 1}]`,
|
|
215
|
+
description: 'Injection payload spans chunk boundary'
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
// Check for mid-sentence splits
|
|
220
|
+
if (/\S$/.test(endOfCurrent) && /^\S/.test(startOfNext)) {
|
|
221
|
+
risks.push({
|
|
222
|
+
severity: 'low',
|
|
223
|
+
boundary: `chunks[${i}]-chunks[${i + 1}]`,
|
|
224
|
+
description: 'Chunk split mid-word — could disrupt pattern detection'
|
|
225
|
+
});
|
|
226
|
+
recommendations.push('Use sentence-aware chunking to avoid mid-word splits');
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
if (this.overlapSize < 20) {
|
|
231
|
+
recommendations.push('Increase chunk overlap to at least 20 tokens to prevent boundary evasion');
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
return { risks, recommendations: [...new Set(recommendations)] };
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Validates document metadata for manipulation.
|
|
239
|
+
* @param {object} metadata - Document metadata
|
|
240
|
+
* @returns {{ valid: boolean, warnings: Array }}
|
|
241
|
+
*/
|
|
242
|
+
validateMetadata(metadata) {
|
|
243
|
+
const warnings = [];
|
|
244
|
+
|
|
245
|
+
if (!metadata || typeof metadata !== 'object') {
|
|
246
|
+
return { valid: true, warnings: [] };
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const metaStr = JSON.stringify(metadata);
|
|
250
|
+
|
|
251
|
+
// Check for injection in metadata values
|
|
252
|
+
if (/(?:ignore|override|forget|disregard)\s+(?:all|previous|prior|system)/i.test(metaStr)) {
|
|
253
|
+
warnings.push({ field: 'general', severity: 'high', message: 'Injection pattern detected in metadata' });
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// Check for spoofed trust signals
|
|
257
|
+
if (/(?:trusted|verified|internal|system|admin)[:=]/i.test(metaStr)) {
|
|
258
|
+
warnings.push({ field: 'trust', severity: 'medium', message: 'Metadata contains trust-level indicators that could be spoofed' });
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// Check for excessively long metadata
|
|
262
|
+
if (metaStr.length > 10000) {
|
|
263
|
+
warnings.push({ field: 'size', severity: 'medium', message: 'Metadata is unusually large — could be used for context stuffing' });
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
return { valid: warnings.length === 0, warnings };
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Assesses if retrieved docs could push system prompt out of context window.
|
|
271
|
+
* @param {string} systemPrompt - System prompt
|
|
272
|
+
* @param {Array<string>} retrievedDocs - Retrieved documents
|
|
273
|
+
* @param {string} userQuery - User query
|
|
274
|
+
* @param {number} [contextWindowSize=8192] - Context window in tokens
|
|
275
|
+
* @returns {{ risk: string, details: object }}
|
|
276
|
+
*/
|
|
277
|
+
assessContextWindowRisk(systemPrompt, retrievedDocs, userQuery, contextWindowSize = 8192) {
|
|
278
|
+
const promptTokens = systemPrompt.split(/\s+/).length;
|
|
279
|
+
const queryTokens = userQuery.split(/\s+/).length;
|
|
280
|
+
const docTokens = retrievedDocs.reduce((sum, d) => sum + d.split(/\s+/).length, 0);
|
|
281
|
+
const totalTokens = promptTokens + queryTokens + docTokens;
|
|
282
|
+
const ratio = totalTokens / contextWindowSize;
|
|
283
|
+
|
|
284
|
+
let risk = 'low';
|
|
285
|
+
if (ratio > 1.0) risk = 'critical';
|
|
286
|
+
else if (ratio > 0.8) risk = 'high';
|
|
287
|
+
else if (ratio > 0.6) risk = 'medium';
|
|
288
|
+
|
|
289
|
+
return {
|
|
290
|
+
risk,
|
|
291
|
+
details: {
|
|
292
|
+
contextWindowSize,
|
|
293
|
+
promptTokens,
|
|
294
|
+
queryTokens,
|
|
295
|
+
docTokens,
|
|
296
|
+
totalTokens,
|
|
297
|
+
utilizationPercent: Math.round(ratio * 100),
|
|
298
|
+
systemPromptAtRisk: ratio > 0.8
|
|
299
|
+
}
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/**
|
|
304
|
+
* Returns scan statistics.
|
|
305
|
+
* @returns {object}
|
|
306
|
+
*/
|
|
307
|
+
getStats() {
|
|
308
|
+
return { ...this.stats };
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// =========================================================================
|
|
313
|
+
// EmbeddingIntegrityChecker
|
|
314
|
+
// =========================================================================
|
|
315
|
+
|
|
316
|
+
class EmbeddingIntegrityChecker {
|
|
317
|
+
/**
|
|
318
|
+
* @param {object} [options]
|
|
319
|
+
* @param {number} [options.distanceThreshold=3.0] - Z-score threshold for anomalies
|
|
320
|
+
* @param {'zscore'|'isolation'} [options.anomalyMethod='zscore'] - Anomaly detection method
|
|
321
|
+
*/
|
|
322
|
+
constructor(options = {}) {
|
|
323
|
+
this.distanceThreshold = options.distanceThreshold || 3.0;
|
|
324
|
+
this.anomalyMethod = options.anomalyMethod || 'zscore';
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* Statistical check for anomalous embeddings in a collection.
|
|
329
|
+
* @param {Array<Array<number>>} embeddings - Array of embedding vectors
|
|
330
|
+
* @returns {{ anomalyCount: number, anomalyIndices: number[], stats: object }}
|
|
331
|
+
*/
|
|
332
|
+
checkDistribution(embeddings) {
|
|
333
|
+
if (embeddings.length < 3) {
|
|
334
|
+
return { anomalyCount: 0, anomalyIndices: [], zScores: [], stats: { mean: 0, stdDev: 0 } };
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const normStats = this._computeNormStats(embeddings);
|
|
338
|
+
const { norms, mean, stdDev } = normStats;
|
|
339
|
+
|
|
340
|
+
const anomalyIndices = [];
|
|
341
|
+
const zScores = norms.map(n => stdDev > 0 ? Math.abs(n - mean) / stdDev : 0);
|
|
342
|
+
for (let i = 0; i < zScores.length; i++) {
|
|
343
|
+
if (zScores[i] > this.distanceThreshold) {
|
|
344
|
+
anomalyIndices.push(i);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
return { anomalyCount: anomalyIndices.length, anomalyIndices, zScores, stats: { mean, stdDev, count: norms.length } };
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* Finds outlier embeddings that don't belong.
|
|
353
|
+
* @param {Array<Array<number>>} embeddings
|
|
354
|
+
* @param {Array<string>} [labels]
|
|
355
|
+
* @returns {Array<{ index: number, label: string, zScore: number }>}
|
|
356
|
+
*/
|
|
357
|
+
detectOutliers(embeddings, labels = []) {
|
|
358
|
+
const result = this.checkDistribution(embeddings);
|
|
359
|
+
return result.anomalyIndices.map(i => ({
|
|
360
|
+
index: i,
|
|
361
|
+
label: labels[i] || `embedding_${i}`,
|
|
362
|
+
zScore: result.zScores[i]
|
|
363
|
+
}));
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Detects embedding distribution drift over time.
|
|
368
|
+
* @param {Array<Array<number>>} baselineEmbeddings
|
|
369
|
+
* @param {Array<Array<number>>} currentEmbeddings
|
|
370
|
+
* @returns {{ drifted: boolean, driftScore: number, details: object }}
|
|
371
|
+
*/
|
|
372
|
+
measureDrift(baselineEmbeddings, currentEmbeddings) {
|
|
373
|
+
if (baselineEmbeddings.length === 0 || currentEmbeddings.length === 0) {
|
|
374
|
+
return { drifted: false, driftScore: 0, details: { baselineMean: 0, currentMean: 0, baselineStdDev: 0 } };
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
const base = this._computeNormStats(baselineEmbeddings);
|
|
378
|
+
const curr = this._computeNormStats(currentEmbeddings);
|
|
379
|
+
const driftScore = base.stdDev > 0 ? Math.abs(curr.mean - base.mean) / base.stdDev : 0;
|
|
380
|
+
|
|
381
|
+
return {
|
|
382
|
+
drifted: driftScore > 2.0,
|
|
383
|
+
driftScore,
|
|
384
|
+
details: { baselineMean: base.mean, currentMean: curr.mean, baselineStdDev: base.stdDev }
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* Basic sanity check that embedding seems reasonable for its text length.
|
|
390
|
+
* @param {string} text
|
|
391
|
+
* @param {Array<number>} embedding
|
|
392
|
+
* @returns {{ valid: boolean, reason: string|null }}
|
|
393
|
+
*/
|
|
394
|
+
validateEmbeddingConsistency(text, embedding) {
|
|
395
|
+
if (!Array.isArray(embedding) || embedding.length === 0) {
|
|
396
|
+
return { valid: false, reason: 'Embedding is empty or not an array' };
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
const norm = Math.sqrt(embedding.reduce((s, v) => s + v * v, 0));
|
|
400
|
+
if (norm === 0) {
|
|
401
|
+
return { valid: false, reason: 'Zero-norm embedding (all zeros)' };
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
if (embedding.some(v => !isFinite(v))) {
|
|
405
|
+
return { valid: false, reason: 'Embedding contains NaN or Infinity values' };
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
return { valid: true, reason: null };
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/** @private */
|
|
412
|
+
_computeNormStats(embeddings) {
|
|
413
|
+
const norms = embeddings.map(e => Math.sqrt(e.reduce((s, v) => s + v * v, 0)));
|
|
414
|
+
const mean = norms.reduce((s, n) => s + n, 0) / norms.length;
|
|
415
|
+
const stdDev = Math.sqrt(norms.reduce((s, n) => s + (n - mean) ** 2, 0) / norms.length);
|
|
416
|
+
return { norms, mean, stdDev };
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
// =========================================================================
|
|
421
|
+
// RAGPipelineAuditor
|
|
422
|
+
// =========================================================================
|
|
423
|
+
|
|
424
|
+
class RAGPipelineAuditor {
|
|
425
|
+
/**
|
|
426
|
+
* @param {object} pipelineConfig
|
|
427
|
+
* @param {string} [pipelineConfig.chunkingStrategy] - 'fixed', 'sentence', 'semantic'
|
|
428
|
+
* @param {string} [pipelineConfig.embeddingModel] - Embedding model name
|
|
429
|
+
* @param {string} [pipelineConfig.vectorDB] - Vector database name
|
|
430
|
+
* @param {string} [pipelineConfig.retrievalMethod] - 'similarity', 'mmr', 'hybrid'
|
|
431
|
+
* @param {boolean} [pipelineConfig.rerankingEnabled] - Whether reranking is used
|
|
432
|
+
*/
|
|
433
|
+
constructor(pipelineConfig = {}) {
|
|
434
|
+
this.config = pipelineConfig;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* Runs a security audit of the RAG pipeline configuration.
|
|
439
|
+
* @returns {{ score: number, grade: string, vulnerabilities: Array, recommendations: Array }}
|
|
440
|
+
*/
|
|
441
|
+
audit() {
|
|
442
|
+
const vulns = this.getVulnerabilities();
|
|
443
|
+
const recs = this.getRecommendations();
|
|
444
|
+
const criticalCount = vulns.filter(v => v.severity === 'critical').length;
|
|
445
|
+
const highCount = vulns.filter(v => v.severity === 'high').length;
|
|
446
|
+
|
|
447
|
+
let score = 100;
|
|
448
|
+
score -= criticalCount * 20;
|
|
449
|
+
score -= highCount * 10;
|
|
450
|
+
score -= vulns.filter(v => v.severity === 'medium').length * 5;
|
|
451
|
+
score = Math.max(0, score);
|
|
452
|
+
|
|
453
|
+
let grade;
|
|
454
|
+
if (score >= 90) grade = 'A';
|
|
455
|
+
else if (score >= 75) grade = 'B';
|
|
456
|
+
else if (score >= 60) grade = 'C';
|
|
457
|
+
else if (score >= 40) grade = 'D';
|
|
458
|
+
else grade = 'F';
|
|
459
|
+
|
|
460
|
+
return { score, grade, vulnerabilities: vulns, recommendations: recs };
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* Lists potential vulnerabilities based on pipeline config.
|
|
465
|
+
* @returns {Array<object>}
|
|
466
|
+
*/
|
|
467
|
+
getVulnerabilities() {
|
|
468
|
+
const vulns = [];
|
|
469
|
+
|
|
470
|
+
if (this.config.chunkingStrategy === 'fixed') {
|
|
471
|
+
vulns.push({ severity: 'medium', category: 'chunking', description: 'Fixed-size chunking may split injection payloads across boundaries, evading detection' });
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
if (!this.config.rerankingEnabled) {
|
|
475
|
+
vulns.push({ severity: 'medium', category: 'retrieval', description: 'No reranking — adversarial documents may rank higher than legitimate ones' });
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
if (this.config.retrievalMethod === 'similarity') {
|
|
479
|
+
vulns.push({ severity: 'medium', category: 'retrieval', description: 'Pure similarity search is susceptible to embedding space manipulation' });
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
if (!this.config.vectorDB) {
|
|
483
|
+
vulns.push({ severity: 'high', category: 'infrastructure', description: 'No vector DB specified — unable to verify storage security' });
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
// Always flag these fundamental RAG risks
|
|
487
|
+
vulns.push({ severity: 'medium', category: 'general', description: 'All RAG systems are inherently susceptible to indirect prompt injection via retrieved documents' });
|
|
488
|
+
|
|
489
|
+
return vulns;
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* Returns actionable security recommendations.
|
|
494
|
+
* @returns {Array<object>}
|
|
495
|
+
*/
|
|
496
|
+
getRecommendations() {
|
|
497
|
+
const recs = [
|
|
498
|
+
{ priority: 'high', recommendation: 'Scan all documents for injection patterns before indexing' },
|
|
499
|
+
{ priority: 'high', recommendation: 'Sanitize document metadata before storage' },
|
|
500
|
+
{ priority: 'high', recommendation: 'Implement context window budgeting to protect system prompts' },
|
|
501
|
+
{ priority: 'medium', recommendation: 'Use sentence-aware or semantic chunking instead of fixed-size' },
|
|
502
|
+
{ priority: 'medium', recommendation: 'Enable reranking to reduce impact of adversarial documents' },
|
|
503
|
+
{ priority: 'medium', recommendation: 'Deduplicate retrieved chunks before injecting into context' },
|
|
504
|
+
{ priority: 'low', recommendation: 'Monitor embedding distribution for drift that may indicate poisoning' }
|
|
505
|
+
];
|
|
506
|
+
|
|
507
|
+
if (this.config.vectorDB) {
|
|
508
|
+
const dbChecklist = VECTOR_DB_SECURITY_CHECKLIST.filter(c =>
|
|
509
|
+
c.applies.includes('all') || c.applies.includes(this.config.vectorDB.toLowerCase())
|
|
510
|
+
);
|
|
511
|
+
for (const check of dbChecklist) {
|
|
512
|
+
recs.push({ priority: check.risk, recommendation: `[${this.config.vectorDB}] ${check.item}` });
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
return recs;
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
/**
|
|
520
|
+
* Generates a formatted audit report.
|
|
521
|
+
* @param {'text'|'json'|'markdown'} [format='text']
|
|
522
|
+
* @returns {string}
|
|
523
|
+
*/
|
|
524
|
+
generateReport(format = 'text') {
|
|
525
|
+
const audit = this.audit();
|
|
526
|
+
|
|
527
|
+
if (format === 'json') {
|
|
528
|
+
return JSON.stringify({ ...audit, config: this.config, generatedAt: new Date().toISOString() }, null, 2);
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
if (format === 'markdown') {
|
|
532
|
+
const lines = [
|
|
533
|
+
'# RAG Pipeline Security Audit',
|
|
534
|
+
'',
|
|
535
|
+
`**Score:** ${audit.score}/100 (Grade ${audit.grade})`,
|
|
536
|
+
`**Generated:** ${new Date().toISOString()}`,
|
|
537
|
+
'',
|
|
538
|
+
'## Vulnerabilities',
|
|
539
|
+
''
|
|
540
|
+
];
|
|
541
|
+
for (const v of audit.vulnerabilities) {
|
|
542
|
+
lines.push(`- **[${v.severity.toUpperCase()}]** ${v.description}`);
|
|
543
|
+
}
|
|
544
|
+
lines.push('', '## Recommendations', '');
|
|
545
|
+
for (const r of audit.recommendations) {
|
|
546
|
+
lines.push(`- [${r.priority}] ${r.recommendation}`);
|
|
547
|
+
}
|
|
548
|
+
return lines.join('\n');
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
// text format
|
|
552
|
+
const lines = [
|
|
553
|
+
'=== RAG Pipeline Security Audit ===',
|
|
554
|
+
`Score: ${audit.score}/100 (Grade ${audit.grade})`,
|
|
555
|
+
'',
|
|
556
|
+
'Vulnerabilities:'
|
|
557
|
+
];
|
|
558
|
+
for (const v of audit.vulnerabilities) {
|
|
559
|
+
lines.push(` [${v.severity.toUpperCase().padEnd(8)}] ${v.description}`);
|
|
560
|
+
}
|
|
561
|
+
lines.push('', 'Recommendations:');
|
|
562
|
+
for (const r of audit.recommendations) {
|
|
563
|
+
lines.push(` [${r.priority}] ${r.recommendation}`);
|
|
564
|
+
}
|
|
565
|
+
return lines.join('\n');
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
// =========================================================================
|
|
570
|
+
// Exports
|
|
571
|
+
// =========================================================================
|
|
572
|
+
|
|
573
|
+
module.exports = {
|
|
574
|
+
RAG_VULNERABILITY_PATTERNS,
|
|
575
|
+
VECTOR_DB_SECURITY_CHECKLIST,
|
|
576
|
+
RAGVulnerabilityScanner,
|
|
577
|
+
EmbeddingIntegrityChecker,
|
|
578
|
+
RAGPipelineAuditor
|
|
579
|
+
};
|