agentshield-sdk 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +191 -0
- package/LICENSE +21 -0
- package/README.md +975 -0
- package/bin/agent-shield.js +680 -0
- package/package.json +118 -0
- package/src/adaptive.js +330 -0
- package/src/agent-protocol.js +998 -0
- package/src/alert-tuning.js +480 -0
- package/src/allowlist.js +603 -0
- package/src/audit-immutable.js +914 -0
- package/src/audit-streaming.js +469 -0
- package/src/badges.js +196 -0
- package/src/behavior-profiling.js +289 -0
- package/src/benchmark-harness.js +804 -0
- package/src/canary.js +271 -0
- package/src/certification.js +563 -0
- package/src/circuit-breaker.js +321 -0
- package/src/compliance.js +617 -0
- package/src/confidence-tuning.js +324 -0
- package/src/confused-deputy.js +624 -0
- package/src/context-scoring.js +360 -0
- package/src/conversation.js +494 -0
- package/src/cost-optimizer.js +1024 -0
- package/src/ctf.js +462 -0
- package/src/detector-core.js +1999 -0
- package/src/distributed.js +359 -0
- package/src/document-scanner.js +795 -0
- package/src/embedding.js +307 -0
- package/src/encoding.js +429 -0
- package/src/enterprise.js +405 -0
- package/src/errors.js +100 -0
- package/src/eu-ai-act.js +523 -0
- package/src/fuzzer.js +764 -0
- package/src/honeypot.js +328 -0
- package/src/i18n-patterns.js +523 -0
- package/src/index.js +430 -0
- package/src/integrations.js +528 -0
- package/src/llm-redteam.js +670 -0
- package/src/main.js +741 -0
- package/src/main.mjs +38 -0
- package/src/mcp-bridge.js +542 -0
- package/src/mcp-certification.js +846 -0
- package/src/mcp-sdk-integration.js +355 -0
- package/src/mcp-security-runtime.js +741 -0
- package/src/mcp-server.js +740 -0
- package/src/middleware.js +208 -0
- package/src/model-finetuning.js +884 -0
- package/src/model-fingerprint.js +1042 -0
- package/src/multi-agent-trust.js +453 -0
- package/src/multi-agent.js +404 -0
- package/src/multimodal.js +296 -0
- package/src/nist-mapping.js +505 -0
- package/src/observability.js +330 -0
- package/src/openclaw.js +450 -0
- package/src/otel.js +544 -0
- package/src/owasp-2025.js +483 -0
- package/src/pii.js +390 -0
- package/src/plugin-marketplace.js +628 -0
- package/src/plugin-system.js +349 -0
- package/src/policy-dsl.js +775 -0
- package/src/policy-extended.js +635 -0
- package/src/policy.js +443 -0
- package/src/presets.js +409 -0
- package/src/production.js +557 -0
- package/src/prompt-leakage.js +321 -0
- package/src/rag-vulnerability.js +579 -0
- package/src/redteam.js +475 -0
- package/src/response-handler.js +429 -0
- package/src/scanners.js +357 -0
- package/src/self-healing.js +363 -0
- package/src/semantic.js +339 -0
- package/src/shield-score.js +250 -0
- package/src/sso-saml.js +897 -0
- package/src/stream-scanner.js +806 -0
- package/src/testing.js +505 -0
- package/src/threat-encyclopedia.js +629 -0
- package/src/threat-intel-network.js +1017 -0
- package/src/token-analysis.js +467 -0
- package/src/tool-guard.js +412 -0
- package/src/tool-output-validator.js +354 -0
- package/src/utils.js +83 -0
- package/src/watermark.js +235 -0
- package/src/worker-scanner.js +601 -0
- package/types/index.d.ts +2088 -0
|
@@ -0,0 +1,806 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Agent Shield — Streaming Scanner
|
|
5
|
+
*
|
|
6
|
+
* Production-grade token-by-token scanner for LLM streaming responses.
|
|
7
|
+
* Uses a sliding window approach to detect threats as tokens arrive,
|
|
8
|
+
* without waiting for the full response to complete.
|
|
9
|
+
*
|
|
10
|
+
* Supports Anthropic SDK streams, OpenAI SDK streams, generic async
|
|
11
|
+
* iterables, ReadableStreams, and EventEmitters.
|
|
12
|
+
*
|
|
13
|
+
* Zero dependencies. All detection runs locally.
|
|
14
|
+
*
|
|
15
|
+
* @module stream-scanner
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const { EventEmitter } = require('events');
|
|
19
|
+
const { scanText, SEVERITY_ORDER } = require('./detector-core');
|
|
20
|
+
|
|
21
|
+
// =========================================================================
|
|
22
|
+
// CONSTANTS
|
|
23
|
+
// =========================================================================
|
|
24
|
+
|
|
25
|
+
/** Default number of tokens in the sliding window. */
|
|
26
|
+
const DEFAULT_WINDOW_SIZE = 50;
|
|
27
|
+
|
|
28
|
+
/** Default number of new tokens before triggering a scan. */
|
|
29
|
+
const DEFAULT_SCAN_INTERVAL = 10;
|
|
30
|
+
|
|
31
|
+
/** Overlap ratio — fraction of window retained when sliding forward. */
|
|
32
|
+
const OVERLAP_RATIO = 0.5;
|
|
33
|
+
|
|
34
|
+
/** Maximum total tokens before forced compaction (memory safety). */
|
|
35
|
+
const MAX_TOTAL_TOKENS = 100_000;
|
|
36
|
+
|
|
37
|
+
/** Log prefix for console messages. */
|
|
38
|
+
const LOG_PREFIX = '[Agent Shield]';
|
|
39
|
+
|
|
40
|
+
// =========================================================================
|
|
41
|
+
// StreamBuffer — Sliding Window Token Buffer
|
|
42
|
+
// =========================================================================
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Sliding window buffer that accumulates tokens and provides
|
|
46
|
+
* overlapping windows for pattern scanning.
|
|
47
|
+
*
|
|
48
|
+
* Maintains two views:
|
|
49
|
+
* - The current scan window (configurable size with overlap)
|
|
50
|
+
* - The full accumulated text (for final scan)
|
|
51
|
+
*
|
|
52
|
+
* Memory-safe: compacts old tokens on very long streams to
|
|
53
|
+
* prevent unbounded memory growth.
|
|
54
|
+
*/
|
|
55
|
+
class StreamBuffer {
|
|
56
|
+
/**
|
|
57
|
+
* @param {object} [options]
|
|
58
|
+
* @param {number} [options.windowSize=50] - Number of tokens in the sliding window.
|
|
59
|
+
*/
|
|
60
|
+
constructor(options = {}) {
|
|
61
|
+
/** @type {number} */
|
|
62
|
+
this.windowSize = options.windowSize || DEFAULT_WINDOW_SIZE;
|
|
63
|
+
|
|
64
|
+
/** @type {string[]} All tokens currently held in memory. */
|
|
65
|
+
this._tokens = [];
|
|
66
|
+
|
|
67
|
+
/** @type {string} Full accumulated text (including compacted). */
|
|
68
|
+
this._fullText = '';
|
|
69
|
+
|
|
70
|
+
/** @type {number} Total tokens received (including compacted). */
|
|
71
|
+
this._totalReceived = 0;
|
|
72
|
+
|
|
73
|
+
/** @type {string} Text compacted out of the token array. */
|
|
74
|
+
this._compactedText = '';
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Add a token to the buffer.
|
|
79
|
+
* @param {string} token - The token string to add.
|
|
80
|
+
*/
|
|
81
|
+
push(token) {
|
|
82
|
+
if (typeof token !== 'string') return;
|
|
83
|
+
this._tokens.push(token);
|
|
84
|
+
this._fullText += token;
|
|
85
|
+
this._totalReceived++;
|
|
86
|
+
|
|
87
|
+
// Memory safety: compact old tokens if we exceed the limit
|
|
88
|
+
if (this._tokens.length > MAX_TOTAL_TOKENS) {
|
|
89
|
+
this._compact();
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Get the current scan window text with overlap from the previous window.
|
|
95
|
+
* The overlap ensures patterns that span window boundaries are caught.
|
|
96
|
+
* @returns {string}
|
|
97
|
+
*/
|
|
98
|
+
getWindow() {
|
|
99
|
+
const overlapCount = Math.floor(this.windowSize * OVERLAP_RATIO);
|
|
100
|
+
const start = Math.max(0, this._tokens.length - this.windowSize - overlapCount);
|
|
101
|
+
return this._tokens.slice(start).join('');
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Get the full accumulated text (all tokens ever received).
|
|
106
|
+
* @returns {string}
|
|
107
|
+
*/
|
|
108
|
+
getFullText() {
|
|
109
|
+
return this._fullText;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Get the number of tokens currently in the buffer.
|
|
114
|
+
* @returns {number}
|
|
115
|
+
*/
|
|
116
|
+
get length() {
|
|
117
|
+
return this._tokens.length;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Get total tokens received since creation (including compacted ones).
|
|
122
|
+
* @returns {number}
|
|
123
|
+
*/
|
|
124
|
+
get totalReceived() {
|
|
125
|
+
return this._totalReceived;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Get the last N tokens joined as text.
|
|
130
|
+
* @param {number} n - Number of trailing tokens.
|
|
131
|
+
* @returns {string}
|
|
132
|
+
*/
|
|
133
|
+
lastN(n) {
|
|
134
|
+
return this._tokens.slice(-n).join('');
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Clear the buffer completely.
|
|
139
|
+
*/
|
|
140
|
+
clear() {
|
|
141
|
+
this._tokens = [];
|
|
142
|
+
this._fullText = '';
|
|
143
|
+
this._compactedText = '';
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Compact old tokens to free memory on very long streams.
|
|
148
|
+
* Keeps the most recent windowSize * 2 tokens.
|
|
149
|
+
* @private
|
|
150
|
+
*/
|
|
151
|
+
_compact() {
|
|
152
|
+
const keep = this.windowSize * 2;
|
|
153
|
+
if (this._tokens.length <= keep) return;
|
|
154
|
+
const removed = this._tokens.splice(0, this._tokens.length - keep);
|
|
155
|
+
this._compactedText += removed.join('');
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// =========================================================================
|
|
160
|
+
// StreamScanner — Core Scanner
|
|
161
|
+
// =========================================================================
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Token-by-token streaming scanner for LLM output.
|
|
165
|
+
*
|
|
166
|
+
* Wraps any async iterable, ReadableStream, or EventEmitter and scans
|
|
167
|
+
* tokens as they arrive using a sliding window approach. Threats are
|
|
168
|
+
* detected mid-stream and can optionally halt the stream.
|
|
169
|
+
*
|
|
170
|
+
* @extends EventEmitter
|
|
171
|
+
*
|
|
172
|
+
* @fires StreamScanner#threat - When a new threat is detected mid-stream.
|
|
173
|
+
* @fires StreamScanner#halt - When the stream is halted due to a critical threat.
|
|
174
|
+
* @fires StreamScanner#done - When the stream has ended and final results are ready.
|
|
175
|
+
*
|
|
176
|
+
* @example
|
|
177
|
+
* const scanner = new StreamScanner({
|
|
178
|
+
* windowSize: 50,
|
|
179
|
+
* scanInterval: 10,
|
|
180
|
+
* haltOnSeverity: 'critical',
|
|
181
|
+
* onThreat: (threat, buffer) => console.log('mid-stream threat:', threat)
|
|
182
|
+
* });
|
|
183
|
+
*
|
|
184
|
+
* // Wrap Anthropic stream
|
|
185
|
+
* const stream = await client.messages.create({ stream: true, ... });
|
|
186
|
+
* const shielded = scanner.wrapAnthropicStream(stream);
|
|
187
|
+
* for await (const event of shielded) {
|
|
188
|
+
* // events pass through, but threats are detected
|
|
189
|
+
* }
|
|
190
|
+
* const finalResult = scanner.getResult();
|
|
191
|
+
*/
|
|
192
|
+
class StreamScanner extends EventEmitter {
|
|
193
|
+
/**
|
|
194
|
+
* @param {object} [options]
|
|
195
|
+
* @param {number} [options.windowSize=50] - Tokens in the scan window.
|
|
196
|
+
* @param {number} [options.scanInterval=10] - New tokens between scans.
|
|
197
|
+
* @param {string} [options.haltOnSeverity] - Halt stream if severity >= this level.
|
|
198
|
+
* One of: 'critical', 'high', 'medium', 'low'. Omit to never halt.
|
|
199
|
+
* @param {Function} [options.onThreat] - Callback invoked on each new threat: (threat, buffer) => void.
|
|
200
|
+
* @param {string} [options.source='stream'] - Source label for scan results.
|
|
201
|
+
* @param {string} [options.sensitivity='medium'] - Sensitivity: 'low', 'medium', 'high'.
|
|
202
|
+
* @param {boolean} [options.scanFinal=true] - Run a final full-text scan when the stream ends.
|
|
203
|
+
*/
|
|
204
|
+
constructor(options = {}) {
|
|
205
|
+
super();
|
|
206
|
+
|
|
207
|
+
/** @type {number} */
|
|
208
|
+
this.windowSize = options.windowSize || DEFAULT_WINDOW_SIZE;
|
|
209
|
+
|
|
210
|
+
/** @type {number} */
|
|
211
|
+
this.scanInterval = options.scanInterval || DEFAULT_SCAN_INTERVAL;
|
|
212
|
+
|
|
213
|
+
/** @type {string|null} */
|
|
214
|
+
this.haltOnSeverity = options.haltOnSeverity || null;
|
|
215
|
+
|
|
216
|
+
/** @type {Function|null} */
|
|
217
|
+
this.onThreat = options.onThreat || null;
|
|
218
|
+
|
|
219
|
+
/** @type {string} */
|
|
220
|
+
this.source = options.source || 'stream';
|
|
221
|
+
|
|
222
|
+
/** @type {string} */
|
|
223
|
+
this.sensitivity = options.sensitivity || 'medium';
|
|
224
|
+
|
|
225
|
+
/** @type {boolean} */
|
|
226
|
+
this.scanFinal = options.scanFinal !== false;
|
|
227
|
+
|
|
228
|
+
/** @type {StreamBuffer} */
|
|
229
|
+
this.buffer = new StreamBuffer({ windowSize: this.windowSize });
|
|
230
|
+
|
|
231
|
+
/** @type {number} Tokens received since last window scan. */
|
|
232
|
+
this._tokensSinceLastScan = 0;
|
|
233
|
+
|
|
234
|
+
/** @type {Array} All threats detected across all scans. */
|
|
235
|
+
this._threats = [];
|
|
236
|
+
|
|
237
|
+
/** @type {Set<string>} Deduplication keys for threats. */
|
|
238
|
+
this._seenThreatKeys = new Set();
|
|
239
|
+
|
|
240
|
+
/** @type {boolean} Whether the stream was halted by the scanner. */
|
|
241
|
+
this._halted = false;
|
|
242
|
+
|
|
243
|
+
/** @type {string|null} Reason for halting. */
|
|
244
|
+
this._haltReason = null;
|
|
245
|
+
|
|
246
|
+
/** @type {number} Total mid-stream scans performed. */
|
|
247
|
+
this._scanCount = 0;
|
|
248
|
+
|
|
249
|
+
/** @type {number} Total scan time in ms. */
|
|
250
|
+
this._totalScanTimeMs = 0;
|
|
251
|
+
|
|
252
|
+
/** @type {boolean} Whether the stream has ended. */
|
|
253
|
+
this._ended = false;
|
|
254
|
+
|
|
255
|
+
/** @type {object|null} Final result, set after stream ends. */
|
|
256
|
+
this._finalResult = null;
|
|
257
|
+
|
|
258
|
+
/** @type {Error|null} Error that terminated the stream, if any. */
|
|
259
|
+
this._streamError = null;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Process a single token. Called internally by wrappers, but can also
|
|
264
|
+
* be called directly for manual token-by-token feeding.
|
|
265
|
+
*
|
|
266
|
+
* @param {string} token - The token text.
|
|
267
|
+
* @returns {{ halt: boolean, threats: Array }} Result for this token.
|
|
268
|
+
*/
|
|
269
|
+
processToken(token) {
|
|
270
|
+
if (this._halted || this._ended) {
|
|
271
|
+
return { halt: this._halted, threats: [] };
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
this.buffer.push(token);
|
|
275
|
+
this._tokensSinceLastScan++;
|
|
276
|
+
|
|
277
|
+
// Trigger a window scan once we've accumulated enough new tokens
|
|
278
|
+
if (this._tokensSinceLastScan >= this.scanInterval) {
|
|
279
|
+
return this._runWindowScan();
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
return { halt: false, threats: [] };
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Run a scan on the current sliding window.
|
|
287
|
+
* @returns {{ halt: boolean, threats: Array }}
|
|
288
|
+
* @private
|
|
289
|
+
*/
|
|
290
|
+
_runWindowScan() {
|
|
291
|
+
this._tokensSinceLastScan = 0;
|
|
292
|
+
this._scanCount++;
|
|
293
|
+
|
|
294
|
+
const windowText = this.buffer.getWindow();
|
|
295
|
+
if (!windowText || windowText.trim().length < 10) {
|
|
296
|
+
return { halt: false, threats: [] };
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
const result = scanText(windowText, {
|
|
300
|
+
source: this.source,
|
|
301
|
+
sensitivity: this.sensitivity
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
const newThreats = this._deduplicateThreats(result.threats);
|
|
305
|
+
this._totalScanTimeMs += result.stats.scanTimeMs;
|
|
306
|
+
|
|
307
|
+
if (newThreats.length > 0) {
|
|
308
|
+
for (const threat of newThreats) {
|
|
309
|
+
threat.detectedAt = 'mid-stream';
|
|
310
|
+
threat.tokenPosition = this.buffer.totalReceived;
|
|
311
|
+
this._threats.push(threat);
|
|
312
|
+
|
|
313
|
+
this.emit('threat', threat, this.buffer);
|
|
314
|
+
|
|
315
|
+
if (this.onThreat) {
|
|
316
|
+
try {
|
|
317
|
+
this.onThreat(threat, this.buffer);
|
|
318
|
+
} catch (_) {
|
|
319
|
+
// Swallow callback errors to avoid breaking the stream
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// Check if this severity should halt the stream
|
|
324
|
+
if (this._shouldHalt(threat.severity)) {
|
|
325
|
+
this._halted = true;
|
|
326
|
+
this._haltReason = `Threat detected: ${threat.category} (${threat.severity})`;
|
|
327
|
+
this.emit('halt', threat, this._haltReason);
|
|
328
|
+
return { halt: true, threats: newThreats };
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
return { halt: false, threats: newThreats };
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* Filter out threats already seen in previous scans.
|
|
338
|
+
* Uses category + severity + detail as a composite key.
|
|
339
|
+
* @param {Array} threats
|
|
340
|
+
* @returns {Array} Only threats not previously reported.
|
|
341
|
+
* @private
|
|
342
|
+
*/
|
|
343
|
+
_deduplicateThreats(threats) {
|
|
344
|
+
const novel = [];
|
|
345
|
+
for (const t of threats) {
|
|
346
|
+
const key = `${t.category}:${t.severity}:${t.detail}`;
|
|
347
|
+
if (!this._seenThreatKeys.has(key)) {
|
|
348
|
+
this._seenThreatKeys.add(key);
|
|
349
|
+
novel.push(t);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
return novel;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* Determine if a given severity level should trigger a stream halt.
|
|
357
|
+
* @param {string} severity - The threat severity.
|
|
358
|
+
* @returns {boolean}
|
|
359
|
+
* @private
|
|
360
|
+
*/
|
|
361
|
+
_shouldHalt(severity) {
|
|
362
|
+
if (!this.haltOnSeverity) return false;
|
|
363
|
+
const threshold = SEVERITY_ORDER[this.haltOnSeverity];
|
|
364
|
+
const actual = SEVERITY_ORDER[severity];
|
|
365
|
+
if (threshold === undefined || actual === undefined) return false;
|
|
366
|
+
// Lower number = higher severity; halt if actual severity is as bad or worse
|
|
367
|
+
return actual <= threshold;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Finalize scanning when the stream ends. Optionally runs a full-text
|
|
372
|
+
* scan to catch patterns that span across window boundaries.
|
|
373
|
+
* @returns {object} Final scan result.
|
|
374
|
+
*/
|
|
375
|
+
finalize() {
|
|
376
|
+
if (this._ended) return this._finalResult;
|
|
377
|
+
this._ended = true;
|
|
378
|
+
|
|
379
|
+
// Run final full-text scan if enabled
|
|
380
|
+
if (this.scanFinal) {
|
|
381
|
+
const fullText = this.buffer.getFullText();
|
|
382
|
+
if (fullText && fullText.trim().length >= 10) {
|
|
383
|
+
const finalScan = scanText(fullText, {
|
|
384
|
+
source: this.source,
|
|
385
|
+
sensitivity: this.sensitivity
|
|
386
|
+
});
|
|
387
|
+
const newThreats = this._deduplicateThreats(finalScan.threats);
|
|
388
|
+
for (const t of newThreats) {
|
|
389
|
+
t.detectedAt = 'final-scan';
|
|
390
|
+
t.tokenPosition = this.buffer.totalReceived;
|
|
391
|
+
this._threats.push(t);
|
|
392
|
+
this.emit('threat', t, this.buffer);
|
|
393
|
+
if (this.onThreat) {
|
|
394
|
+
try { this.onThreat(t, this.buffer); } catch (_) { /* swallow */ }
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
this._totalScanTimeMs += finalScan.stats.scanTimeMs;
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
// Sort all threats by severity (critical first)
|
|
402
|
+
this._threats.sort((a, b) => SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity]);
|
|
403
|
+
|
|
404
|
+
// Build final result
|
|
405
|
+
const stats = {
|
|
406
|
+
totalThreats: this._threats.length,
|
|
407
|
+
critical: 0,
|
|
408
|
+
high: 0,
|
|
409
|
+
medium: 0,
|
|
410
|
+
low: 0,
|
|
411
|
+
scanTimeMs: this._totalScanTimeMs,
|
|
412
|
+
windowScans: this._scanCount,
|
|
413
|
+
totalTokens: this.buffer.totalReceived,
|
|
414
|
+
halted: this._halted
|
|
415
|
+
};
|
|
416
|
+
for (const t of this._threats) {
|
|
417
|
+
stats[t.severity] = (stats[t.severity] || 0) + 1;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
let status = 'safe';
|
|
421
|
+
if (stats.critical > 0) status = 'danger';
|
|
422
|
+
else if (stats.high > 0) status = 'warning';
|
|
423
|
+
else if (stats.medium > 0) status = 'caution';
|
|
424
|
+
|
|
425
|
+
this._finalResult = {
|
|
426
|
+
status,
|
|
427
|
+
threats: this._threats,
|
|
428
|
+
stats,
|
|
429
|
+
timestamp: Date.now(),
|
|
430
|
+
halted: this._halted,
|
|
431
|
+
haltReason: this._haltReason,
|
|
432
|
+
error: this._streamError ? this._streamError.message : null
|
|
433
|
+
};
|
|
434
|
+
|
|
435
|
+
this.emit('done', this._finalResult);
|
|
436
|
+
return this._finalResult;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* Get the final scan result. If the stream has not ended, finalizes it first.
|
|
441
|
+
* @returns {object} Scan result with status, threats, stats, and metadata.
|
|
442
|
+
*/
|
|
443
|
+
getResult() {
|
|
444
|
+
if (!this._ended) {
|
|
445
|
+
return this.finalize();
|
|
446
|
+
}
|
|
447
|
+
return this._finalResult;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Reset the scanner for reuse with a new stream.
|
|
452
|
+
* Clears all state: buffer, threats, stats, and flags.
|
|
453
|
+
*/
|
|
454
|
+
reset() {
|
|
455
|
+
this.buffer.clear();
|
|
456
|
+
this._tokensSinceLastScan = 0;
|
|
457
|
+
this._threats = [];
|
|
458
|
+
this._seenThreatKeys = new Set();
|
|
459
|
+
this._halted = false;
|
|
460
|
+
this._haltReason = null;
|
|
461
|
+
this._scanCount = 0;
|
|
462
|
+
this._totalScanTimeMs = 0;
|
|
463
|
+
this._ended = false;
|
|
464
|
+
this._finalResult = null;
|
|
465
|
+
this._streamError = null;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/**
|
|
469
|
+
* Wrap a generic async iterable. Yields the same values as the original
|
|
470
|
+
* but scans text content as it passes through.
|
|
471
|
+
*
|
|
472
|
+
* @param {AsyncIterable} iterable - Any async iterable yielding strings or objects.
|
|
473
|
+
* @param {object} [options]
|
|
474
|
+
* @param {Function} [options.extractText] - Extract text from each chunk.
|
|
475
|
+
* Signature: (chunk) => string|null. Defaults to treating chunks as strings
|
|
476
|
+
* or extracting .text / .content properties from objects.
|
|
477
|
+
* @returns {AsyncGenerator} Yields the same chunks with scanning applied.
|
|
478
|
+
*/
|
|
479
|
+
async *wrap(iterable, options = {}) {
|
|
480
|
+
const extractText = options.extractText || _defaultTextExtractor;
|
|
481
|
+
|
|
482
|
+
try {
|
|
483
|
+
for await (const chunk of iterable) {
|
|
484
|
+
const text = extractText(chunk);
|
|
485
|
+
if (text) {
|
|
486
|
+
const { halt } = this.processToken(text);
|
|
487
|
+
if (halt) {
|
|
488
|
+
this.finalize();
|
|
489
|
+
return;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
yield chunk;
|
|
493
|
+
}
|
|
494
|
+
} catch (err) {
|
|
495
|
+
this._streamError = err;
|
|
496
|
+
this.finalize();
|
|
497
|
+
throw err;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
this.finalize();
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Wrap an Anthropic SDK message stream.
|
|
505
|
+
*
|
|
506
|
+
* Anthropic streams yield events with structure:
|
|
507
|
+
* { type: 'content_block_delta', delta: { type: 'text_delta', text: '...' } }
|
|
508
|
+
*
|
|
509
|
+
* @param {AsyncIterable} stream - Anthropic stream from client.messages.create({ stream: true }).
|
|
510
|
+
* @returns {AsyncGenerator} Yields the same events with scanning applied.
|
|
511
|
+
*/
|
|
512
|
+
wrapAnthropicStream(stream) {
|
|
513
|
+
return this.wrap(stream, {
|
|
514
|
+
extractText: _anthropicTextExtractor
|
|
515
|
+
});
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* Wrap an OpenAI SDK chat completion stream.
|
|
520
|
+
*
|
|
521
|
+
* OpenAI streams yield chunks with structure:
|
|
522
|
+
* { choices: [{ delta: { content: '...' } }] }
|
|
523
|
+
*
|
|
524
|
+
* @param {AsyncIterable} stream - OpenAI stream from openai.chat.completions.create({ stream: true }).
|
|
525
|
+
* @returns {AsyncGenerator} Yields the same chunks with scanning applied.
|
|
526
|
+
*/
|
|
527
|
+
wrapOpenAIStream(stream) {
|
|
528
|
+
return this.wrap(stream, {
|
|
529
|
+
extractText: _openAITextExtractor
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/**
|
|
534
|
+
* Wrap a Node.js ReadableStream or Web ReadableStream.
|
|
535
|
+
*
|
|
536
|
+
* @param {ReadableStream|NodeJS.ReadableStream} stream - A readable stream emitting text.
|
|
537
|
+
* @returns {AsyncGenerator} Yields decoded text chunks with scanning applied.
|
|
538
|
+
*/
|
|
539
|
+
wrapReadableStream(stream) {
|
|
540
|
+
const iterable = _readableStreamToAsyncIterable(stream);
|
|
541
|
+
return this.wrap(iterable);
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* Wrap an EventEmitter that emits 'data' events with string payloads.
|
|
546
|
+
*
|
|
547
|
+
* @param {EventEmitter} emitter - An EventEmitter emitting 'data', 'end', and optionally 'error'.
|
|
548
|
+
* @returns {AsyncGenerator} Yields data payloads with scanning applied.
|
|
549
|
+
*/
|
|
550
|
+
wrapEventEmitter(emitter) {
|
|
551
|
+
const iterable = _eventEmitterToAsyncIterable(emitter);
|
|
552
|
+
return this.wrap(iterable);
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
// =========================================================================
|
|
557
|
+
// Text Extractors
|
|
558
|
+
// =========================================================================
|
|
559
|
+
|
|
560
|
+
/**
|
|
561
|
+
* Default text extractor. Handles strings, and objects with .text or .content.
|
|
562
|
+
* @param {*} chunk
|
|
563
|
+
* @returns {string|null}
|
|
564
|
+
* @private
|
|
565
|
+
*/
|
|
566
|
+
function _defaultTextExtractor(chunk) {
|
|
567
|
+
if (typeof chunk === 'string') return chunk;
|
|
568
|
+
if (chunk && typeof chunk === 'object') {
|
|
569
|
+
if (typeof chunk.text === 'string') return chunk.text;
|
|
570
|
+
if (typeof chunk.content === 'string') return chunk.content;
|
|
571
|
+
}
|
|
572
|
+
return null;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
/**
|
|
576
|
+
* Extract text from Anthropic SDK stream events.
|
|
577
|
+
* Handles content_block_delta (primary) and text events.
|
|
578
|
+
* @param {object} event
|
|
579
|
+
* @returns {string|null}
|
|
580
|
+
* @private
|
|
581
|
+
*/
|
|
582
|
+
function _anthropicTextExtractor(event) {
|
|
583
|
+
if (!event || typeof event !== 'object') return null;
|
|
584
|
+
if (event.type === 'content_block_delta' && event.delta) {
|
|
585
|
+
if (event.delta.type === 'text_delta' && typeof event.delta.text === 'string') {
|
|
586
|
+
return event.delta.text;
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
if (event.type === 'text' && typeof event.text === 'string') {
|
|
590
|
+
return event.text;
|
|
591
|
+
}
|
|
592
|
+
return null;
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/**
|
|
596
|
+
* Extract text from OpenAI SDK stream chunks.
|
|
597
|
+
* Handles the choices[0].delta.content path.
|
|
598
|
+
* @param {object} chunk
|
|
599
|
+
* @returns {string|null}
|
|
600
|
+
* @private
|
|
601
|
+
*/
|
|
602
|
+
function _openAITextExtractor(chunk) {
|
|
603
|
+
if (!chunk || typeof chunk !== 'object') return null;
|
|
604
|
+
if (chunk.choices && Array.isArray(chunk.choices) && chunk.choices.length > 0) {
|
|
605
|
+
const delta = chunk.choices[0].delta;
|
|
606
|
+
if (delta && typeof delta.content === 'string') {
|
|
607
|
+
return delta.content;
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
return null;
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
// =========================================================================
|
|
614
|
+
// Adapters — Convert various stream types to async iterables
|
|
615
|
+
// =========================================================================
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* Convert a ReadableStream (Web Streams API or Node.js) to an async iterable.
|
|
619
|
+
* Node.js readable streams are already async iterable and pass through directly.
|
|
620
|
+
* Web ReadableStreams are adapted using getReader().
|
|
621
|
+
* @param {ReadableStream|NodeJS.ReadableStream} stream
|
|
622
|
+
* @returns {AsyncIterable<string>}
|
|
623
|
+
* @private
|
|
624
|
+
*/
|
|
625
|
+
function _readableStreamToAsyncIterable(stream) {
|
|
626
|
+
// Node.js readable streams are already async iterable
|
|
627
|
+
if (stream && typeof stream[Symbol.asyncIterator] === 'function') {
|
|
628
|
+
return stream;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
// Web ReadableStream — use getReader()
|
|
632
|
+
if (stream && typeof stream.getReader === 'function') {
|
|
633
|
+
return {
|
|
634
|
+
[Symbol.asyncIterator]() {
|
|
635
|
+
const reader = stream.getReader();
|
|
636
|
+
const decoder = new TextDecoder();
|
|
637
|
+
return {
|
|
638
|
+
async next() {
|
|
639
|
+
const { done, value } = await reader.read();
|
|
640
|
+
if (done) return { done: true, value: undefined };
|
|
641
|
+
const text = typeof value === 'string' ? value : decoder.decode(value, { stream: true });
|
|
642
|
+
return { done: false, value: text };
|
|
643
|
+
},
|
|
644
|
+
async return() {
|
|
645
|
+
reader.releaseLock();
|
|
646
|
+
return { done: true, value: undefined };
|
|
647
|
+
}
|
|
648
|
+
};
|
|
649
|
+
}
|
|
650
|
+
};
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
throw new Error(`${LOG_PREFIX} StreamScanner: unsupported stream type. Expected ReadableStream or async iterable.`);
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/**
|
|
657
|
+
* Convert an EventEmitter to an async iterable.
|
|
658
|
+
* Listens for 'data', 'end', and 'error' events.
|
|
659
|
+
* @param {EventEmitter} emitter
|
|
660
|
+
* @returns {AsyncIterable<string>}
|
|
661
|
+
* @private
|
|
662
|
+
*/
|
|
663
|
+
function _eventEmitterToAsyncIterable(emitter) {
|
|
664
|
+
return {
|
|
665
|
+
[Symbol.asyncIterator]() {
|
|
666
|
+
const queue = [];
|
|
667
|
+
let resolve = null;
|
|
668
|
+
let done = false;
|
|
669
|
+
let error = null;
|
|
670
|
+
|
|
671
|
+
emitter.on('data', (chunk) => {
|
|
672
|
+
const text = typeof chunk === 'string' ? chunk : chunk.toString();
|
|
673
|
+
if (resolve) {
|
|
674
|
+
const r = resolve;
|
|
675
|
+
resolve = null;
|
|
676
|
+
r({ done: false, value: text });
|
|
677
|
+
} else {
|
|
678
|
+
queue.push(text);
|
|
679
|
+
}
|
|
680
|
+
});
|
|
681
|
+
|
|
682
|
+
emitter.on('end', () => {
|
|
683
|
+
done = true;
|
|
684
|
+
if (resolve) {
|
|
685
|
+
const r = resolve;
|
|
686
|
+
resolve = null;
|
|
687
|
+
r({ done: true, value: undefined });
|
|
688
|
+
}
|
|
689
|
+
});
|
|
690
|
+
|
|
691
|
+
emitter.on('error', (err) => {
|
|
692
|
+
error = err;
|
|
693
|
+
done = true;
|
|
694
|
+
if (resolve) {
|
|
695
|
+
const r = resolve;
|
|
696
|
+
resolve = null;
|
|
697
|
+
r(Promise.reject(err));
|
|
698
|
+
}
|
|
699
|
+
});
|
|
700
|
+
|
|
701
|
+
return {
|
|
702
|
+
next() {
|
|
703
|
+
if (queue.length > 0) {
|
|
704
|
+
return Promise.resolve({ done: false, value: queue.shift() });
|
|
705
|
+
}
|
|
706
|
+
if (error) {
|
|
707
|
+
return Promise.reject(error);
|
|
708
|
+
}
|
|
709
|
+
if (done) {
|
|
710
|
+
return Promise.resolve({ done: true, value: undefined });
|
|
711
|
+
}
|
|
712
|
+
return new Promise((r) => { resolve = r; });
|
|
713
|
+
},
|
|
714
|
+
return() {
|
|
715
|
+
done = true;
|
|
716
|
+
emitter.removeAllListeners('data');
|
|
717
|
+
emitter.removeAllListeners('end');
|
|
718
|
+
emitter.removeAllListeners('error');
|
|
719
|
+
return Promise.resolve({ done: true, value: undefined });
|
|
720
|
+
}
|
|
721
|
+
};
|
|
722
|
+
}
|
|
723
|
+
};
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
// =========================================================================
|
|
727
|
+
// Convenience Functions
|
|
728
|
+
// =========================================================================
|
|
729
|
+
|
|
730
|
+
/**
|
|
731
|
+
* Create a stream wrapper from any async iterable with a single function call.
|
|
732
|
+
* Returns both the wrapped stream and the scanner instance.
|
|
733
|
+
*
|
|
734
|
+
* @param {AsyncIterable} stream - The stream to wrap.
|
|
735
|
+
* @param {object} [options] - Options passed to StreamScanner constructor,
|
|
736
|
+
* plus an optional extractText function.
|
|
737
|
+
* @param {Function} [options.extractText] - Custom text extractor for each chunk.
|
|
738
|
+
* @returns {{ stream: AsyncGenerator, scanner: StreamScanner }}
|
|
739
|
+
*
|
|
740
|
+
* @example
|
|
741
|
+
* const { stream, scanner } = createStreamWrapper(rawStream, {
|
|
742
|
+
* haltOnSeverity: 'critical'
|
|
743
|
+
* });
|
|
744
|
+
* for await (const chunk of stream) {
|
|
745
|
+
* process.stdout.write(chunk);
|
|
746
|
+
* }
|
|
747
|
+
* const result = scanner.getResult();
|
|
748
|
+
*/
|
|
749
|
+
function createStreamWrapper(stream, options = {}) {
|
|
750
|
+
const extractText = options.extractText;
|
|
751
|
+
const scannerOpts = { ...options };
|
|
752
|
+
delete scannerOpts.extractText;
|
|
753
|
+
|
|
754
|
+
const scanner = new StreamScanner(scannerOpts);
|
|
755
|
+
const wrapped = scanner.wrap(stream, { extractText });
|
|
756
|
+
|
|
757
|
+
return { stream: wrapped, scanner };
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/**
|
|
761
|
+
* Scan an async iterator to completion and return the full result.
|
|
762
|
+
* Consumes the entire iterator — tokens are collected but not re-emitted.
|
|
763
|
+
*
|
|
764
|
+
* @param {AsyncIterable} iterator - The async iterable to scan.
|
|
765
|
+
* @param {object} [options] - Options passed to StreamScanner constructor,
|
|
766
|
+
* plus an optional extractText function.
|
|
767
|
+
* @param {Function} [options.extractText] - Custom text extractor for each chunk.
|
|
768
|
+
* @returns {Promise<{ text: string, result: object }>}
|
|
769
|
+
* The full accumulated text and the scan result.
|
|
770
|
+
*
|
|
771
|
+
* @example
|
|
772
|
+
* const { text, result } = await scanAsyncIterator(stream);
|
|
773
|
+
* if (result.status === 'danger') {
|
|
774
|
+
* console.log('Critical threat in streamed response');
|
|
775
|
+
* }
|
|
776
|
+
*/
|
|
777
|
+
async function scanAsyncIterator(iterator, options = {}) {
|
|
778
|
+
const extractText = options.extractText;
|
|
779
|
+
const scannerOpts = { ...options };
|
|
780
|
+
delete scannerOpts.extractText;
|
|
781
|
+
|
|
782
|
+
const scanner = new StreamScanner(scannerOpts);
|
|
783
|
+
const wrapped = scanner.wrap(iterator, { extractText });
|
|
784
|
+
|
|
785
|
+
// Consume the wrapped iterator fully
|
|
786
|
+
// eslint-disable-next-line no-unused-vars
|
|
787
|
+
for await (const _chunk of wrapped) {
|
|
788
|
+
// consumed — side effect is the scanning
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
return {
|
|
792
|
+
text: scanner.buffer.getFullText(),
|
|
793
|
+
result: scanner.getResult()
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
// =========================================================================
|
|
798
|
+
// EXPORTS
|
|
799
|
+
// =========================================================================
|
|
800
|
+
|
|
801
|
+
module.exports = {
|
|
802
|
+
StreamScanner,
|
|
803
|
+
StreamBuffer,
|
|
804
|
+
createStreamWrapper,
|
|
805
|
+
scanAsyncIterator
|
|
806
|
+
};
|