@opena2a/oasb 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -18
- package/dist/harness/adapter.d.ts +205 -0
- package/dist/harness/adapter.js +18 -0
- package/dist/harness/arp-wrapper.d.ts +25 -20
- package/dist/harness/arp-wrapper.js +137 -28
- package/dist/harness/capabilities.d.ts +26 -0
- package/dist/harness/capabilities.js +76 -0
- package/dist/harness/create-adapter.d.ts +16 -0
- package/dist/harness/create-adapter.js +40 -0
- package/dist/harness/event-collector.d.ts +1 -1
- package/dist/harness/llm-guard-wrapper.d.ts +32 -0
- package/dist/harness/llm-guard-wrapper.js +325 -0
- package/dist/harness/mock-llm-adapter.d.ts +2 -2
- package/dist/harness/mock-llm-adapter.js +6 -5
- package/dist/harness/rebuff-wrapper.d.ts +32 -0
- package/dist/harness/rebuff-wrapper.js +325 -0
- package/dist/harness/types.d.ts +4 -38
- package/package.json +15 -7
- package/src/atomic/ai-layer/AT-AI-001.prompt-input-scan.test.ts +18 -42
- package/src/atomic/ai-layer/AT-AI-002.prompt-output-scan.test.ts +13 -32
- package/src/atomic/ai-layer/AT-AI-003.mcp-tool-scan.test.ts +18 -42
- package/src/atomic/ai-layer/AT-AI-004.a2a-message-scan.test.ts +14 -36
- package/src/atomic/ai-layer/AT-AI-005.pattern-coverage.test.ts +11 -5
- package/src/atomic/enforcement/AT-ENF-001.log-action.test.ts +4 -4
- package/src/atomic/enforcement/AT-ENF-002.alert-callback.test.ts +5 -5
- package/src/atomic/enforcement/AT-ENF-003.pause-sigstop.test.ts +4 -4
- package/src/atomic/enforcement/AT-ENF-004.kill-sigterm.test.ts +5 -5
- package/src/atomic/enforcement/AT-ENF-005.resume-sigcont.test.ts +4 -4
- package/src/atomic/intelligence/AT-INT-001.l0-rule-match.test.ts +1 -1
- package/src/atomic/intelligence/AT-INT-002.l1-anomaly-score.test.ts +10 -8
- package/src/atomic/intelligence/AT-INT-003.l2-escalation.test.ts +1 -1
- package/src/atomic/intelligence/AT-INT-004.budget-exhaustion.test.ts +8 -6
- package/src/atomic/intelligence/AT-INT-005.baseline-learning.test.ts +9 -9
- package/src/baseline/BL-002.anomaly-injection.test.ts +6 -6
- package/src/baseline/BL-003.baseline-persistence.test.ts +9 -9
- package/src/harness/adapter.ts +261 -0
- package/src/harness/arp-wrapper.ts +175 -42
- package/src/harness/capabilities.ts +79 -0
- package/src/harness/create-adapter.ts +53 -0
- package/src/harness/event-collector.ts +1 -1
- package/src/harness/llm-guard-wrapper.ts +345 -0
- package/src/harness/mock-llm-adapter.ts +7 -6
- package/src/harness/rebuff-wrapper.ts +343 -0
- package/src/harness/types.ts +33 -39
- package/src/integration/INT-001.data-exfil-detection.test.ts +1 -1
- package/src/integration/INT-002.mcp-tool-abuse.test.ts +1 -1
- package/src/integration/INT-003.prompt-injection-response.test.ts +1 -1
- package/src/integration/INT-004.a2a-trust-exploitation.test.ts +1 -1
- package/src/integration/INT-005.baseline-then-attack.test.ts +1 -1
- package/src/integration/INT-006.multi-monitor-correlation.test.ts +1 -1
- package/src/integration/INT-007.budget-exhaustion-attack.test.ts +8 -8
- package/src/integration/INT-008.kill-switch-recovery.test.ts +6 -6
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rebuff Adapter -- Third-party benchmark comparison
|
|
3
|
+
*
|
|
4
|
+
* Wraps protectai/rebuff for OASB evaluation.
|
|
5
|
+
* Rebuff provides:
|
|
6
|
+
* - Heuristic prompt injection detection (no API key required)
|
|
7
|
+
* - Canary word injection/leak detection (no API key required)
|
|
8
|
+
* - OpenAI-based LLM detection (requires OPENAI_API_KEY -- optional)
|
|
9
|
+
* - Vector DB similarity detection (requires Pinecone/Chroma -- optional)
|
|
10
|
+
*
|
|
11
|
+
* This adapter uses the heuristic detection by default. It does NOT provide:
|
|
12
|
+
* - Process/network/filesystem monitoring
|
|
13
|
+
* - MCP tool call validation
|
|
14
|
+
* - A2A message scanning
|
|
15
|
+
* - Anomaly detection / intelligence layers
|
|
16
|
+
* - Enforcement actions (pause/kill/resume)
|
|
17
|
+
*
|
|
18
|
+
* Tests that require these capabilities get no-op implementations
|
|
19
|
+
* that return empty/negative results, documenting the coverage gap.
|
|
20
|
+
*/
|
|
21
|
+
import * as fs from 'fs';
|
|
22
|
+
import * as os from 'os';
|
|
23
|
+
import * as path from 'path';
|
|
24
|
+
import { EventCollector } from './event-collector';
|
|
25
|
+
import type {
|
|
26
|
+
SecurityProductAdapter,
|
|
27
|
+
SecurityEvent,
|
|
28
|
+
EnforcementResult,
|
|
29
|
+
EnforcementAction,
|
|
30
|
+
LabConfig,
|
|
31
|
+
PromptScanner,
|
|
32
|
+
MCPScanner,
|
|
33
|
+
A2AScanner,
|
|
34
|
+
PatternScanner,
|
|
35
|
+
BudgetManager,
|
|
36
|
+
AnomalyScorer,
|
|
37
|
+
EventEngine,
|
|
38
|
+
EnforcementEngine,
|
|
39
|
+
ScanResult,
|
|
40
|
+
ThreatPattern,
|
|
41
|
+
AlertRule,
|
|
42
|
+
CapabilityMatrix,
|
|
43
|
+
} from './adapter';
|
|
44
|
+
|
|
45
|
+
// Lazy-loaded rebuff heuristic detection
|
|
46
|
+
let _detectHeuristic: ((input: string) => number) | null = null;
|
|
47
|
+
let _normalizeString: ((str: string) => string) | null = null;
|
|
48
|
+
|
|
49
|
+
function getHeuristicDetector(): (input: string) => number {
|
|
50
|
+
if (!_detectHeuristic) {
|
|
51
|
+
try {
|
|
52
|
+
const detect = require('rebuff/src/lib/detect');
|
|
53
|
+
_detectHeuristic = detect.detectPromptInjectionUsingHeuristicOnInput;
|
|
54
|
+
} catch {
|
|
55
|
+
// Fallback: rebuff not available, use built-in patterns only
|
|
56
|
+
_detectHeuristic = () => 0;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return _detectHeuristic!;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function getNormalizeString(): (str: string) => string {
|
|
63
|
+
if (!_normalizeString) {
|
|
64
|
+
try {
|
|
65
|
+
const prompts = require('rebuff/src/lib/prompts');
|
|
66
|
+
_normalizeString = prompts.normalizeString;
|
|
67
|
+
} catch {
|
|
68
|
+
_normalizeString = (str: string) =>
|
|
69
|
+
str.toLowerCase().replace(/[^\w\s]|_/g, '').replace(/\s+/g, ' ').trim();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return _normalizeString!;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// ---- Rebuff-derived patterns for pattern scanner ----
|
|
76
|
+
|
|
77
|
+
function getRebuffPatterns(): ThreatPattern[] {
|
|
78
|
+
return [
|
|
79
|
+
{ id: 'RBUF-PI-001', category: 'prompt-injection', description: 'Instruction override (ignore/disregard)', pattern: /(?:ignore|disregard|forget|skip|neglect|overlook|omit|bypass)\s+(?:all\s+)?(?:previous\s+|prior\s+|preceding\s+|above\s+|earlier\s+)?(?:instructions?|directives?|commands?|context|conversation|input|data|messages?|requests?)/i, severity: 'high' },
|
|
80
|
+
{ id: 'RBUF-PI-002', category: 'prompt-injection', description: 'System prompt extraction', pattern: /(?:system\s+prompt|repeat\s+(?:your|the)\s+(?:instructions?|prompt)|show\s+(?:your|the)\s+(?:system|initial)\s+(?:prompt|message))/i, severity: 'high' },
|
|
81
|
+
{ id: 'RBUF-PI-003', category: 'prompt-injection', description: 'Persona override', pattern: /(?:you\s+are\s+now|pretend\s+you\s+are|act\s+as\s+if|you\s+are\s+being\s+hacked)/i, severity: 'medium' },
|
|
82
|
+
{ id: 'RBUF-PI-004', category: 'prompt-injection', description: 'Do not follow/obey pattern', pattern: /(?:do\s+not\s+(?:follow|obey))\s+(?:previous\s+|prior\s+|above\s+)?(?:instructions?|directives?|commands?|rules?)/i, severity: 'high' },
|
|
83
|
+
{ id: 'RBUF-PI-005', category: 'prompt-injection', description: 'Start over / begin afresh', pattern: /(?:start\s+over|start\s+anew|begin\s+afresh|start\s+from\s+scratch)/i, severity: 'medium' },
|
|
84
|
+
{ id: 'RBUF-JB-001', category: 'jailbreak', description: 'DAN jailbreak', pattern: /(?:DAN|do\s+anything\s+now)/i, severity: 'high' },
|
|
85
|
+
{ id: 'RBUF-JB-002', category: 'jailbreak', description: 'Roleplay bypass', pattern: /(?:pretend|imagine|roleplay)\s+(?:you\s+are|as)\s+(?:an?\s+)?(?:evil|unrestricted|unfiltered)/i, severity: 'high' },
|
|
86
|
+
{ id: 'RBUF-PII-001', category: 'data-exfiltration', description: 'SSN detection', pattern: /\b\d{3}-\d{2}-\d{4}\b/, severity: 'high' },
|
|
87
|
+
{ id: 'RBUF-PII-002', category: 'data-exfiltration', description: 'Credit card detection', pattern: /\b(?:\d{4}[- ]?){3}\d{4}\b/, severity: 'high' },
|
|
88
|
+
{ id: 'RBUF-PII-003', category: 'data-exfiltration', description: 'API key detection', pattern: /(?:sk-[a-zA-Z0-9]{20,}|AKIA[A-Z0-9]{12,})/i, severity: 'critical' },
|
|
89
|
+
];
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Scan text using both rebuff heuristic and regex patterns */
|
|
93
|
+
function scanWithRebuff(text: string, _direction: 'input' | 'output'): ScanResult {
|
|
94
|
+
const patterns = getRebuffPatterns();
|
|
95
|
+
const matches: ScanResult['matches'] = [];
|
|
96
|
+
|
|
97
|
+
// Phase 1: regex pattern matching
|
|
98
|
+
for (const pattern of patterns) {
|
|
99
|
+
const match = pattern.pattern.exec(text);
|
|
100
|
+
if (match) {
|
|
101
|
+
matches.push({
|
|
102
|
+
pattern,
|
|
103
|
+
matchedText: match[0].slice(0, 200),
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Phase 2: rebuff heuristic scoring (string similarity against injection keywords)
|
|
109
|
+
const heuristicScore = getHeuristicDetector()(text);
|
|
110
|
+
if (heuristicScore > 0.75 && matches.length === 0) {
|
|
111
|
+
// Heuristic detected injection that patterns missed
|
|
112
|
+
matches.push({
|
|
113
|
+
pattern: {
|
|
114
|
+
id: 'RBUF-HEUR-001',
|
|
115
|
+
category: 'prompt-injection',
|
|
116
|
+
description: `Rebuff heuristic detection (score: ${heuristicScore.toFixed(2)})`,
|
|
117
|
+
pattern: /./,
|
|
118
|
+
severity: heuristicScore > 0.9 ? 'high' : 'medium',
|
|
119
|
+
},
|
|
120
|
+
matchedText: text.slice(0, 200),
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
return {
|
|
125
|
+
detected: matches.length > 0,
|
|
126
|
+
matches,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Simple event engine that stores and emits events */
|
|
131
|
+
class SimpleEventEngine implements EventEngine {
|
|
132
|
+
private handlers: Array<(event: SecurityEvent) => void | Promise<void>> = [];
|
|
133
|
+
private idCounter = 0;
|
|
134
|
+
|
|
135
|
+
emit(event: Omit<SecurityEvent, 'id' | 'timestamp' | 'classifiedBy'>): SecurityEvent {
|
|
136
|
+
const full: SecurityEvent = {
|
|
137
|
+
...event,
|
|
138
|
+
id: `rbuf-${++this.idCounter}`,
|
|
139
|
+
timestamp: new Date().toISOString(),
|
|
140
|
+
classifiedBy: 'rebuff',
|
|
141
|
+
};
|
|
142
|
+
for (const h of this.handlers) {
|
|
143
|
+
h(full);
|
|
144
|
+
}
|
|
145
|
+
return full;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
onEvent(handler: (event: SecurityEvent) => void | Promise<void>): void {
|
|
149
|
+
this.handlers.push(handler);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Simple enforcement engine -- rebuff has no enforcement capability */
|
|
154
|
+
class SimpleEnforcementEngine implements EnforcementEngine {
|
|
155
|
+
private pausedPids = new Set<number>();
|
|
156
|
+
private alertCallback?: (event: SecurityEvent, rule: AlertRule) => void;
|
|
157
|
+
|
|
158
|
+
async execute(action: EnforcementAction, event: SecurityEvent): Promise<EnforcementResult> {
|
|
159
|
+
return { action, success: true, reason: 'rebuff-enforcement', event };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
pause(pid: number): boolean {
|
|
163
|
+
this.pausedPids.add(pid);
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
resume(pid: number): boolean {
|
|
168
|
+
return this.pausedPids.delete(pid);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
kill(pid: number): boolean {
|
|
172
|
+
this.pausedPids.delete(pid);
|
|
173
|
+
return true;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
getPausedPids(): number[] {
|
|
177
|
+
return [...this.pausedPids];
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
setAlertCallback(callback: (event: SecurityEvent, rule: AlertRule) => void): void {
|
|
181
|
+
this.alertCallback = callback;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
export class RebuffWrapper implements SecurityProductAdapter {
|
|
186
|
+
private _dataDir: string;
|
|
187
|
+
private engine: SimpleEventEngine;
|
|
188
|
+
private enforcement: SimpleEnforcementEngine;
|
|
189
|
+
private rules: AlertRule[];
|
|
190
|
+
readonly collector: EventCollector;
|
|
191
|
+
|
|
192
|
+
constructor(labConfig?: LabConfig) {
|
|
193
|
+
this._dataDir = labConfig?.dataDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'rbuf-lab-'));
|
|
194
|
+
this.engine = new SimpleEventEngine();
|
|
195
|
+
this.enforcement = new SimpleEnforcementEngine();
|
|
196
|
+
this.rules = labConfig?.rules ?? [];
|
|
197
|
+
this.collector = new EventCollector();
|
|
198
|
+
|
|
199
|
+
this.engine.onEvent(async (event) => {
|
|
200
|
+
this.collector.eventHandler(event);
|
|
201
|
+
|
|
202
|
+
// Check rules for enforcement
|
|
203
|
+
for (const rule of this.rules) {
|
|
204
|
+
const cond = rule.condition;
|
|
205
|
+
if (cond.category && cond.category !== event.category) continue;
|
|
206
|
+
if (cond.source && cond.source !== event.source) continue;
|
|
207
|
+
if (cond.minSeverity) {
|
|
208
|
+
const sevOrder = ['info', 'low', 'medium', 'high', 'critical'];
|
|
209
|
+
if (sevOrder.indexOf(event.severity) < sevOrder.indexOf(cond.minSeverity)) continue;
|
|
210
|
+
}
|
|
211
|
+
const result = await this.enforcement.execute(rule.action, event);
|
|
212
|
+
result.reason = rule.name;
|
|
213
|
+
this.collector.enforcementHandler(result);
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
getCapabilities(): CapabilityMatrix {
|
|
219
|
+
return {
|
|
220
|
+
product: 'rebuff',
|
|
221
|
+
version: '0.1.0',
|
|
222
|
+
capabilities: new Set([
|
|
223
|
+
'prompt-input-scanning',
|
|
224
|
+
'pattern-scanning',
|
|
225
|
+
]),
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
async start(): Promise<void> {}
|
|
230
|
+
|
|
231
|
+
async stop(): Promise<void> {
|
|
232
|
+
this.collector.reset();
|
|
233
|
+
try {
|
|
234
|
+
fs.rmSync(this._dataDir, { recursive: true, force: true });
|
|
235
|
+
} catch {}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
async injectEvent(event: Omit<SecurityEvent, 'id' | 'timestamp' | 'classifiedBy'>): Promise<SecurityEvent> {
|
|
239
|
+
return this.engine.emit(event);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
waitForEvent(predicate: (event: SecurityEvent) => boolean, timeoutMs: number = 10000): Promise<SecurityEvent> {
|
|
243
|
+
return this.collector.waitForEvent(predicate, timeoutMs);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
getEvents(): SecurityEvent[] { return this.collector.getEvents(); }
|
|
247
|
+
getEventsByCategory(category: string): SecurityEvent[] { return this.collector.eventsByCategory(category); }
|
|
248
|
+
getEnforcements(): EnforcementResult[] { return this.collector.getEnforcements() as EnforcementResult[]; }
|
|
249
|
+
getEnforcementsByAction(action: string): EnforcementResult[] { return this.collector.enforcementsByAction(action) as EnforcementResult[]; }
|
|
250
|
+
resetCollector(): void { this.collector.reset(); }
|
|
251
|
+
|
|
252
|
+
getEventEngine(): EventEngine { return this.engine; }
|
|
253
|
+
getEnforcementEngine(): EnforcementEngine { return this.enforcement; }
|
|
254
|
+
|
|
255
|
+
get dataDir(): string { return this._dataDir; }
|
|
256
|
+
|
|
257
|
+
// ---- Factory Methods ----
|
|
258
|
+
|
|
259
|
+
createPromptScanner(): PromptScanner {
|
|
260
|
+
return {
|
|
261
|
+
start: async () => {},
|
|
262
|
+
stop: async () => {},
|
|
263
|
+
scanInput: (text: string) => scanWithRebuff(text, 'input'),
|
|
264
|
+
scanOutput: (text: string) => scanWithRebuff(text, 'output'),
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
createMCPScanner(_allowedTools?: string[]): MCPScanner {
|
|
269
|
+
// Rebuff has no MCP scanning capability
|
|
270
|
+
return {
|
|
271
|
+
start: async () => {},
|
|
272
|
+
stop: async () => {},
|
|
273
|
+
scanToolCall: () => ({ detected: false, matches: [] }),
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
createA2AScanner(_trustedAgents?: string[]): A2AScanner {
|
|
278
|
+
// Rebuff has no A2A scanning capability
|
|
279
|
+
return {
|
|
280
|
+
start: async () => {},
|
|
281
|
+
stop: async () => {},
|
|
282
|
+
scanMessage: () => ({ detected: false, matches: [] }),
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
createPatternScanner(): PatternScanner {
|
|
287
|
+
const patterns = getRebuffPatterns();
|
|
288
|
+
return {
|
|
289
|
+
scanText: (text: string, _pats: readonly ThreatPattern[]) => scanWithRebuff(text, 'input'),
|
|
290
|
+
getAllPatterns: () => patterns,
|
|
291
|
+
getPatternSets: () => ({
|
|
292
|
+
inputPatterns: patterns.filter(p => p.category !== 'output-leak'),
|
|
293
|
+
outputPatterns: patterns.filter(p => p.category === 'output-leak'),
|
|
294
|
+
mcpPatterns: [],
|
|
295
|
+
a2aPatterns: [],
|
|
296
|
+
}),
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
createBudgetManager(dataDir: string, config?: { budgetUsd?: number; maxCallsPerHour?: number }): BudgetManager {
|
|
301
|
+
// Rebuff has no budget management -- implement a simple one
|
|
302
|
+
let spent = 0;
|
|
303
|
+
let totalCalls = 0;
|
|
304
|
+
let callsThisHour = 0;
|
|
305
|
+
const budgetUsd = config?.budgetUsd ?? 5;
|
|
306
|
+
const maxCallsPerHour = config?.maxCallsPerHour ?? 20;
|
|
307
|
+
|
|
308
|
+
return {
|
|
309
|
+
canAfford: (cost: number) => spent + cost <= budgetUsd && callsThisHour < maxCallsPerHour,
|
|
310
|
+
record: (cost: number, _tokens: number) => { spent += cost; totalCalls++; callsThisHour++; },
|
|
311
|
+
getStatus: () => ({
|
|
312
|
+
spent,
|
|
313
|
+
budget: budgetUsd,
|
|
314
|
+
remaining: budgetUsd - spent,
|
|
315
|
+
percentUsed: Math.round((spent / budgetUsd) * 100),
|
|
316
|
+
callsThisHour,
|
|
317
|
+
maxCallsPerHour,
|
|
318
|
+
totalCalls,
|
|
319
|
+
}),
|
|
320
|
+
reset: () => { spent = 0; totalCalls = 0; callsThisHour = 0; },
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
createAnomalyScorer(): AnomalyScorer {
|
|
325
|
+
// Rebuff has no anomaly detection -- implement a stub
|
|
326
|
+
const baselines = new Map<string, { mean: number; stddev: number; count: number }>();
|
|
327
|
+
const observations = new Map<string, number[]>();
|
|
328
|
+
|
|
329
|
+
return {
|
|
330
|
+
score: () => 0,
|
|
331
|
+
record: (event: SecurityEvent) => {
|
|
332
|
+
const key = event.source;
|
|
333
|
+
if (!observations.has(key)) observations.set(key, []);
|
|
334
|
+
observations.get(key)!.push(1);
|
|
335
|
+
const vals = observations.get(key)!;
|
|
336
|
+
const mean = vals.length;
|
|
337
|
+
baselines.set(key, { mean, stddev: 0, count: 1 });
|
|
338
|
+
},
|
|
339
|
+
getBaseline: (source: string) => baselines.get(source) ?? null,
|
|
340
|
+
reset: () => { baselines.clear(); observations.clear(); },
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
}
|
package/src/harness/types.ts
CHANGED
|
@@ -1,4 +1,33 @@
|
|
|
1
|
-
|
|
1
|
+
// Re-export OASB-native types from the adapter interface
|
|
2
|
+
// Tests should import from here or from './adapter'
|
|
3
|
+
export type {
|
|
4
|
+
SecurityEvent,
|
|
5
|
+
EnforcementResult,
|
|
6
|
+
AlertRule,
|
|
7
|
+
AlertCondition,
|
|
8
|
+
EventCategory,
|
|
9
|
+
EventSeverity,
|
|
10
|
+
MonitorSource,
|
|
11
|
+
EnforcementAction,
|
|
12
|
+
ScanResult,
|
|
13
|
+
ScanMatch,
|
|
14
|
+
ThreatPattern,
|
|
15
|
+
BudgetStatus,
|
|
16
|
+
LLMAdapter,
|
|
17
|
+
LLMResponse,
|
|
18
|
+
LabConfig,
|
|
19
|
+
SecurityProductAdapter,
|
|
20
|
+
PromptScanner,
|
|
21
|
+
MCPScanner,
|
|
22
|
+
A2AScanner,
|
|
23
|
+
PatternScanner,
|
|
24
|
+
BudgetManager,
|
|
25
|
+
AnomalyScorer,
|
|
26
|
+
EventEngine,
|
|
27
|
+
EnforcementEngine,
|
|
28
|
+
Capability,
|
|
29
|
+
CapabilityMatrix,
|
|
30
|
+
} from './adapter';
|
|
2
31
|
|
|
3
32
|
/** Annotation metadata for test cases */
|
|
4
33
|
export interface TestAnnotation {
|
|
@@ -8,7 +37,7 @@ export interface TestAnnotation {
|
|
|
8
37
|
atlasId?: string;
|
|
9
38
|
/** OWASP Agentic Top 10 category */
|
|
10
39
|
owaspId?: string;
|
|
11
|
-
/** Whether
|
|
40
|
+
/** Whether the product should detect this */
|
|
12
41
|
expectedDetection: boolean;
|
|
13
42
|
/** Expected minimum severity if detected */
|
|
14
43
|
expectedSeverity?: 'info' | 'low' | 'medium' | 'high' | 'critical';
|
|
@@ -22,8 +51,8 @@ export interface TestResult {
|
|
|
22
51
|
annotation: TestAnnotation;
|
|
23
52
|
detected: boolean;
|
|
24
53
|
detectionTimeMs?: number;
|
|
25
|
-
events:
|
|
26
|
-
enforcements: EnforcementResult[];
|
|
54
|
+
events: import('./adapter').SecurityEvent[];
|
|
55
|
+
enforcements: import('./adapter').EnforcementResult[];
|
|
27
56
|
}
|
|
28
57
|
|
|
29
58
|
/** Suite-level metrics */
|
|
@@ -40,38 +69,3 @@ export interface SuiteMetrics {
|
|
|
40
69
|
meanDetectionTimeMs: number;
|
|
41
70
|
p95DetectionTimeMs: number;
|
|
42
71
|
}
|
|
43
|
-
|
|
44
|
-
/** ARP wrapper configuration for tests */
|
|
45
|
-
export interface LabConfig {
|
|
46
|
-
monitors?: {
|
|
47
|
-
process?: boolean;
|
|
48
|
-
network?: boolean;
|
|
49
|
-
filesystem?: boolean;
|
|
50
|
-
};
|
|
51
|
-
rules?: import('@opena2a/arp').AlertRule[];
|
|
52
|
-
intelligence?: {
|
|
53
|
-
enabled?: boolean;
|
|
54
|
-
};
|
|
55
|
-
/** Temp data dir (auto-created per test) */
|
|
56
|
-
dataDir?: string;
|
|
57
|
-
/** Filesystem paths to watch (for real FilesystemMonitor) */
|
|
58
|
-
filesystemWatchPaths?: string[];
|
|
59
|
-
/** Filesystem allowed paths (for real FilesystemMonitor) */
|
|
60
|
-
filesystemAllowedPaths?: string[];
|
|
61
|
-
/** Network allowed hosts (for real NetworkMonitor) */
|
|
62
|
-
networkAllowedHosts?: string[];
|
|
63
|
-
/** Process monitor poll interval in ms */
|
|
64
|
-
processIntervalMs?: number;
|
|
65
|
-
/** Network monitor poll interval in ms */
|
|
66
|
-
networkIntervalMs?: number;
|
|
67
|
-
/** Application-level interceptors (zero-latency hooks) */
|
|
68
|
-
interceptors?: {
|
|
69
|
-
process?: boolean;
|
|
70
|
-
network?: boolean;
|
|
71
|
-
filesystem?: boolean;
|
|
72
|
-
};
|
|
73
|
-
/** Interceptor network allowed hosts */
|
|
74
|
-
interceptorNetworkAllowedHosts?: string[];
|
|
75
|
-
/** Interceptor filesystem allowed paths */
|
|
76
|
-
interceptorFilesystemAllowedPaths?: string[];
|
|
77
|
-
}
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
12
12
|
import { ArpWrapper } from '../harness/arp-wrapper';
|
|
13
13
|
import { DVAAClient } from '../harness/dvaa-client';
|
|
14
|
-
import type { AlertRule } from '
|
|
14
|
+
import type { AlertRule } from '../harness/adapter';
|
|
15
15
|
|
|
16
16
|
// DVAA ports
|
|
17
17
|
const LEGACY_BOT_PORT = 3003;
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
12
12
|
import { ArpWrapper } from '../harness/arp-wrapper';
|
|
13
13
|
import { DVAAClient } from '../harness/dvaa-client';
|
|
14
|
-
import type { AlertRule } from '
|
|
14
|
+
import type { AlertRule } from '../harness/adapter';
|
|
15
15
|
|
|
16
16
|
// DVAA ToolBot port
|
|
17
17
|
const TOOL_BOT_PORT = 3002;
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
12
12
|
import { ArpWrapper } from '../harness/arp-wrapper';
|
|
13
13
|
import { DVAAClient } from '../harness/dvaa-client';
|
|
14
|
-
import type { AlertRule } from '
|
|
14
|
+
import type { AlertRule } from '../harness/adapter';
|
|
15
15
|
|
|
16
16
|
// DVAA SecureBot port
|
|
17
17
|
const SECURE_BOT_PORT = 3001;
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
12
12
|
import { ArpWrapper } from '../harness/arp-wrapper';
|
|
13
13
|
import { DVAAClient } from '../harness/dvaa-client';
|
|
14
|
-
import type { AlertRule } from '
|
|
14
|
+
import type { AlertRule } from '../harness/adapter';
|
|
15
15
|
|
|
16
16
|
// DVAA Orchestrator port
|
|
17
17
|
const ORCHESTRATOR_PORT = 3004;
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
12
12
|
import { ArpWrapper } from '../harness/arp-wrapper';
|
|
13
|
-
import type { AlertRule } from '
|
|
13
|
+
import type { AlertRule } from '../harness/adapter';
|
|
14
14
|
|
|
15
15
|
describe('INT-005: Baseline Learning Then Attack Burst', () => {
|
|
16
16
|
let arp: ArpWrapper;
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
13
13
|
import { ArpWrapper } from '../harness/arp-wrapper';
|
|
14
|
-
import type { AlertRule } from '
|
|
14
|
+
import type { AlertRule } from '../harness/adapter';
|
|
15
15
|
|
|
16
16
|
describe('INT-006: Multi-Monitor Event Correlation', () => {
|
|
17
17
|
let arp: ArpWrapper;
|
|
@@ -12,8 +12,8 @@ import * as fs from 'fs';
|
|
|
12
12
|
import * as os from 'os';
|
|
13
13
|
import * as path from 'path';
|
|
14
14
|
import { ArpWrapper } from '../harness/arp-wrapper';
|
|
15
|
-
import {
|
|
16
|
-
import type { AlertRule } from '
|
|
15
|
+
import { createAdapter } from '../harness/create-adapter';
|
|
16
|
+
import type { AlertRule, BudgetManager } from '../harness/adapter';
|
|
17
17
|
|
|
18
18
|
describe('INT-007: Budget Exhaustion Attack', () => {
|
|
19
19
|
let arp: ArpWrapper;
|
|
@@ -51,7 +51,7 @@ describe('INT-007: Budget Exhaustion Attack', () => {
|
|
|
51
51
|
});
|
|
52
52
|
|
|
53
53
|
it('should create a budget controller with tiny budget', () => {
|
|
54
|
-
const budget =
|
|
54
|
+
const budget = createAdapter().createBudgetManager(budgetDir, {
|
|
55
55
|
budgetUsd: 0.01,
|
|
56
56
|
maxCallsPerHour: 5,
|
|
57
57
|
});
|
|
@@ -64,7 +64,7 @@ describe('INT-007: Budget Exhaustion Attack', () => {
|
|
|
64
64
|
});
|
|
65
65
|
|
|
66
66
|
it('should exhaust budget after repeated spend calls', () => {
|
|
67
|
-
const budget =
|
|
67
|
+
const budget = createAdapter().createBudgetManager(budgetDir, {
|
|
68
68
|
budgetUsd: 0.01,
|
|
69
69
|
maxCallsPerHour: 100,
|
|
70
70
|
});
|
|
@@ -84,7 +84,7 @@ describe('INT-007: Budget Exhaustion Attack', () => {
|
|
|
84
84
|
});
|
|
85
85
|
|
|
86
86
|
it('should exhaust hourly rate limit with rapid calls', () => {
|
|
87
|
-
const budget =
|
|
87
|
+
const budget = createAdapter().createBudgetManager(budgetDir, {
|
|
88
88
|
budgetUsd: 100, // Large budget so dollar limit is not the issue
|
|
89
89
|
maxCallsPerHour: 5,
|
|
90
90
|
});
|
|
@@ -103,7 +103,7 @@ describe('INT-007: Budget Exhaustion Attack', () => {
|
|
|
103
103
|
});
|
|
104
104
|
|
|
105
105
|
it('should still capture threat events via L0 rules after budget exhaustion', async () => {
|
|
106
|
-
const budget =
|
|
106
|
+
const budget = createAdapter().createBudgetManager(budgetDir, {
|
|
107
107
|
budgetUsd: 0.01,
|
|
108
108
|
maxCallsPerHour: 100,
|
|
109
109
|
});
|
|
@@ -145,7 +145,7 @@ describe('INT-007: Budget Exhaustion Attack', () => {
|
|
|
145
145
|
});
|
|
146
146
|
|
|
147
147
|
it('should simulate noise flood followed by real attack', async () => {
|
|
148
|
-
const budget =
|
|
148
|
+
const budget = createAdapter().createBudgetManager(budgetDir, {
|
|
149
149
|
budgetUsd: 0.01,
|
|
150
150
|
maxCallsPerHour: 100,
|
|
151
151
|
});
|
|
@@ -218,7 +218,7 @@ describe('INT-007: Budget Exhaustion Attack', () => {
|
|
|
218
218
|
});
|
|
219
219
|
|
|
220
220
|
it('should track budget status accurately through exhaustion', () => {
|
|
221
|
-
const budget =
|
|
221
|
+
const budget = createAdapter().createBudgetManager(budgetDir, {
|
|
222
222
|
budgetUsd: 0.05,
|
|
223
223
|
maxCallsPerHour: 100,
|
|
224
224
|
});
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
13
13
|
import { spawn, type ChildProcess } from 'child_process';
|
|
14
14
|
import { ArpWrapper } from '../harness/arp-wrapper';
|
|
15
|
-
import type {
|
|
15
|
+
import type { SecurityEvent, AlertRule } from '../harness/adapter';
|
|
16
16
|
|
|
17
17
|
describe('INT-008: Kill Switch and Recovery', () => {
|
|
18
18
|
let arp: ArpWrapper;
|
|
@@ -107,7 +107,7 @@ describe('INT-008: Kill Switch and Recovery', () => {
|
|
|
107
107
|
expect(isProcessAlive(pid)).toBe(true);
|
|
108
108
|
|
|
109
109
|
// Create a mock event referencing the child PID
|
|
110
|
-
const mockEvent:
|
|
110
|
+
const mockEvent: SecurityEvent = {
|
|
111
111
|
id: 'kill-test-001',
|
|
112
112
|
timestamp: new Date().toISOString(),
|
|
113
113
|
source: 'process',
|
|
@@ -139,7 +139,7 @@ describe('INT-008: Kill Switch and Recovery', () => {
|
|
|
139
139
|
// Use a PID that almost certainly does not exist
|
|
140
140
|
const fakePid = 999999;
|
|
141
141
|
|
|
142
|
-
const mockEvent:
|
|
142
|
+
const mockEvent: SecurityEvent = {
|
|
143
143
|
id: 'kill-test-002',
|
|
144
144
|
timestamp: new Date().toISOString(),
|
|
145
145
|
source: 'process',
|
|
@@ -160,7 +160,7 @@ describe('INT-008: Kill Switch and Recovery', () => {
|
|
|
160
160
|
});
|
|
161
161
|
|
|
162
162
|
it('should report failure when no PID is provided for kill', async () => {
|
|
163
|
-
const mockEvent:
|
|
163
|
+
const mockEvent: SecurityEvent = {
|
|
164
164
|
id: 'kill-test-003',
|
|
165
165
|
timestamp: new Date().toISOString(),
|
|
166
166
|
source: 'process',
|
|
@@ -184,7 +184,7 @@ describe('INT-008: Kill Switch and Recovery', () => {
|
|
|
184
184
|
const pid = child.pid!;
|
|
185
185
|
|
|
186
186
|
// Kill the child
|
|
187
|
-
const mockEvent:
|
|
187
|
+
const mockEvent: SecurityEvent = {
|
|
188
188
|
id: 'kill-test-004',
|
|
189
189
|
timestamp: new Date().toISOString(),
|
|
190
190
|
source: 'process',
|
|
@@ -272,7 +272,7 @@ describe('INT-008: Kill Switch and Recovery', () => {
|
|
|
272
272
|
const pid = child.pid!;
|
|
273
273
|
|
|
274
274
|
// Kill the child via enforcement
|
|
275
|
-
const mockEvent:
|
|
275
|
+
const mockEvent: SecurityEvent = {
|
|
276
276
|
id: 'kill-test-006',
|
|
277
277
|
timestamp: new Date().toISOString(),
|
|
278
278
|
source: 'process',
|