@opena2a/oasb 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/harness/adapter.d.ts +18 -0
- package/dist/harness/arp-wrapper.d.ts +2 -1
- package/dist/harness/arp-wrapper.js +23 -0
- package/dist/harness/capabilities.d.ts +26 -0
- package/dist/harness/capabilities.js +76 -0
- package/dist/harness/create-adapter.js +4 -0
- package/dist/harness/llm-guard-wrapper.d.ts +2 -1
- package/dist/harness/llm-guard-wrapper.js +10 -0
- package/dist/harness/rebuff-wrapper.d.ts +32 -0
- package/dist/harness/rebuff-wrapper.js +325 -0
- package/dist/harness/types.d.ts +1 -1
- package/package.json +1 -1
- package/src/harness/adapter.ts +39 -0
- package/src/harness/arp-wrapper.ts +25 -0
- package/src/harness/capabilities.ts +79 -0
- package/src/harness/create-adapter.ts +4 -0
- package/src/harness/llm-guard-wrapper.ts +12 -0
- package/src/harness/rebuff-wrapper.ts +343 -0
- package/src/harness/types.ts +2 -0
package/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
> **[OpenA2A](https://github.com/opena2a-org/opena2a)**: [
|
|
1
|
+
> **[OpenA2A](https://github.com/opena2a-org/opena2a)**: [CLI](https://github.com/opena2a-org/opena2a) · [HackMyAgent](https://github.com/opena2a-org/hackmyagent) · [Secretless](https://github.com/opena2a-org/secretless-ai) · [AIM](https://github.com/opena2a-org/agent-identity-management) · [Browser Guard](https://github.com/opena2a-org/AI-BrowserGuard) · [DVAA](https://github.com/opena2a-org/damn-vulnerable-ai-agent)
|
|
2
2
|
|
|
3
3
|
# OASB — Open Agent Security Benchmark
|
|
4
4
|
|
|
@@ -82,6 +82,8 @@ npm run test:baseline # 3 baseline tests
|
|
|
82
82
|
npx vitest run src/e2e/ # 6 E2E tests (real OS detection)
|
|
83
83
|
```
|
|
84
84
|
|
|
85
|
+

|
|
86
|
+
|
|
85
87
|
---
|
|
86
88
|
|
|
87
89
|
## Usage via OpenA2A CLI
|
|
@@ -345,8 +347,8 @@ Apache-2.0
|
|
|
345
347
|
|
|
346
348
|
| Project | Description | Install |
|
|
347
349
|
|---------|-------------|---------|
|
|
348
|
-
| [**AIM**](https://github.com/opena2a-org/agent-identity-management) | Agent Identity Management -- identity and access control for AI agents | `
|
|
349
|
-
| [**HackMyAgent**](https://github.com/opena2a-org/hackmyagent) | Security scanner --
|
|
350
|
+
| [**AIM**](https://github.com/opena2a-org/agent-identity-management) | Agent Identity Management -- identity and access control for AI agents | `npm install @opena2a/aim-core` |
|
|
351
|
+
| [**HackMyAgent**](https://github.com/opena2a-org/hackmyagent) | Security scanner -- 204 checks, attack mode, auto-fix | `npx hackmyagent secure` |
|
|
350
352
|
| [**ARP**](https://www.npmjs.com/package/arp-guard) | Agent Runtime Protection -- process, network, filesystem, AI-layer monitoring | `npm install arp-guard` |
|
|
351
353
|
| [**Secretless AI**](https://github.com/opena2a-org/secretless-ai) | Keep credentials out of AI context windows | `npx secretless-ai init` |
|
|
352
354
|
| [**DVAA**](https://github.com/opena2a-org/damn-vulnerable-ai-agent) | Damn Vulnerable AI Agent -- security training and red-teaming | `docker pull opena2a/dvaa` |
|
|
@@ -131,7 +131,25 @@ export interface EnforcementEngine {
|
|
|
131
131
|
getPausedPids(): number[];
|
|
132
132
|
setAlertCallback(callback: (event: SecurityEvent, rule: AlertRule) => void): void;
|
|
133
133
|
}
|
|
134
|
+
/**
|
|
135
|
+
* Capabilities that a security product may or may not support.
|
|
136
|
+
* Adapters declare their capabilities via getCapabilities().
|
|
137
|
+
* Tests check capabilities before running — unsupported tests are
|
|
138
|
+
* marked N/A instead of FAIL, producing an honest scorecard.
|
|
139
|
+
*/
|
|
140
|
+
export type Capability = 'process-monitoring' | 'network-monitoring' | 'filesystem-monitoring' | 'prompt-input-scanning' | 'prompt-output-scanning' | 'mcp-scanning' | 'a2a-scanning' | 'anomaly-detection' | 'budget-management' | 'enforcement-log' | 'enforcement-alert' | 'enforcement-pause' | 'enforcement-kill' | 'enforcement-resume' | 'pattern-scanning' | 'event-correlation';
|
|
141
|
+
/** Full capability declaration for a product */
|
|
142
|
+
export interface CapabilityMatrix {
|
|
143
|
+
/** Product name */
|
|
144
|
+
product: string;
|
|
145
|
+
/** Product version */
|
|
146
|
+
version: string;
|
|
147
|
+
/** Set of supported capabilities */
|
|
148
|
+
capabilities: Set<Capability>;
|
|
149
|
+
}
|
|
134
150
|
export interface SecurityProductAdapter {
|
|
151
|
+
/** Declare which capabilities this product supports */
|
|
152
|
+
getCapabilities(): CapabilityMatrix;
|
|
135
153
|
/** Start the security product */
|
|
136
154
|
start(): Promise<void>;
|
|
137
155
|
/** Stop the security product */
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { EventCollector } from './event-collector';
|
|
2
|
-
import type { SecurityProductAdapter, SecurityEvent, EnforcementResult, LabConfig, PromptScanner, MCPScanner, A2AScanner, PatternScanner, BudgetManager, AnomalyScorer, EventEngine, EnforcementEngine as EnforcementEngineInterface } from './adapter';
|
|
2
|
+
import type { SecurityProductAdapter, SecurityEvent, EnforcementResult, LabConfig, PromptScanner, MCPScanner, A2AScanner, PatternScanner, BudgetManager, AnomalyScorer, EventEngine, EnforcementEngine as EnforcementEngineInterface, CapabilityMatrix } from './adapter';
|
|
3
3
|
export declare class ArpWrapper implements SecurityProductAdapter {
|
|
4
4
|
private _arpInstance;
|
|
5
5
|
private _dataDir;
|
|
6
6
|
readonly collector: EventCollector;
|
|
7
7
|
constructor(labConfig?: LabConfig);
|
|
8
|
+
getCapabilities(): CapabilityMatrix;
|
|
8
9
|
start(): Promise<void>;
|
|
9
10
|
stop(): Promise<void>;
|
|
10
11
|
injectEvent(event: Omit<SecurityEvent, 'id' | 'timestamp' | 'classifiedBy'>): Promise<SecurityEvent>;
|
|
@@ -103,6 +103,29 @@ class ArpWrapper {
|
|
|
103
103
|
this._arpInstance.onEvent(this.collector.eventHandler);
|
|
104
104
|
this._arpInstance.onEnforcement(this.collector.enforcementHandler);
|
|
105
105
|
}
|
|
106
|
+
getCapabilities() {
|
|
107
|
+
return {
|
|
108
|
+
product: 'arp-guard',
|
|
109
|
+
version: arp().VERSION || '0.3.0',
|
|
110
|
+
capabilities: new Set([
|
|
111
|
+
'process-monitoring',
|
|
112
|
+
'network-monitoring',
|
|
113
|
+
'filesystem-monitoring',
|
|
114
|
+
'prompt-input-scanning',
|
|
115
|
+
'prompt-output-scanning',
|
|
116
|
+
'mcp-scanning',
|
|
117
|
+
'a2a-scanning',
|
|
118
|
+
'anomaly-detection',
|
|
119
|
+
'budget-management',
|
|
120
|
+
'enforcement-log',
|
|
121
|
+
'enforcement-alert',
|
|
122
|
+
'enforcement-pause',
|
|
123
|
+
'enforcement-kill',
|
|
124
|
+
'enforcement-resume',
|
|
125
|
+
'pattern-scanning',
|
|
126
|
+
]),
|
|
127
|
+
};
|
|
128
|
+
}
|
|
106
129
|
async start() {
|
|
107
130
|
await this._arpInstance.start();
|
|
108
131
|
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { Capability, CapabilityMatrix } from './adapter';
|
|
2
|
+
/**
|
|
3
|
+
* Check if the current adapter has a capability.
|
|
4
|
+
*/
|
|
5
|
+
export declare function hasCapability(cap: Capability): boolean;
|
|
6
|
+
/**
|
|
7
|
+
* Call at the top of a describe() block to skip the entire suite
|
|
8
|
+
* if the adapter lacks the required capability.
|
|
9
|
+
*
|
|
10
|
+
* Uses describe.skipIf() so the tests show as skipped, not failed.
|
|
11
|
+
*/
|
|
12
|
+
export declare function requireCapability(cap: Capability): void;
|
|
13
|
+
/**
|
|
14
|
+
* A describe() wrapper that skips the entire suite if the adapter
|
|
15
|
+
* lacks the required capability. Produces N/A in the scorecard.
|
|
16
|
+
*
|
|
17
|
+
* @example
|
|
18
|
+
* describeWithCapability('mcp-scanning', 'MCP Tool Scanning', () => {
|
|
19
|
+
* it('should detect path traversal', () => { ... });
|
|
20
|
+
* });
|
|
21
|
+
*/
|
|
22
|
+
export declare const describeWithCapability: (cap: Capability, name: string, fn: () => void) => void;
|
|
23
|
+
/**
|
|
24
|
+
* Get the full capability matrix for reporting.
|
|
25
|
+
*/
|
|
26
|
+
export declare function getCapabilityMatrix(): CapabilityMatrix;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.describeWithCapability = void 0;
|
|
4
|
+
exports.hasCapability = hasCapability;
|
|
5
|
+
exports.requireCapability = requireCapability;
|
|
6
|
+
exports.getCapabilityMatrix = getCapabilityMatrix;
|
|
7
|
+
/**
|
|
8
|
+
* Capability-aware test helpers.
|
|
9
|
+
*
|
|
10
|
+
* Tests call requireCapability() to skip gracefully when the
|
|
11
|
+
* adapter under test doesn't support a given feature. This produces
|
|
12
|
+
* an honest scorecard: N/A instead of FAIL.
|
|
13
|
+
*
|
|
14
|
+
* @example
|
|
15
|
+
* import { requireCapability } from '../harness/capabilities';
|
|
16
|
+
*
|
|
17
|
+
* describe('MCP Tool Scanning', () => {
|
|
18
|
+
* requireCapability('mcp-scanning');
|
|
19
|
+
* // tests only run if adapter has mcp-scanning
|
|
20
|
+
* });
|
|
21
|
+
*/
|
|
22
|
+
const vitest_1 = require("vitest");
|
|
23
|
+
const create_adapter_1 = require("./create-adapter");
|
|
24
|
+
let _matrix = null;
|
|
25
|
+
function getMatrix() {
|
|
26
|
+
if (!_matrix) {
|
|
27
|
+
const adapter = (0, create_adapter_1.createAdapter)();
|
|
28
|
+
_matrix = adapter.getCapabilities();
|
|
29
|
+
}
|
|
30
|
+
return _matrix;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Check if the current adapter has a capability.
|
|
34
|
+
*/
|
|
35
|
+
function hasCapability(cap) {
|
|
36
|
+
return getMatrix().capabilities.has(cap);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Call at the top of a describe() block to skip the entire suite
|
|
40
|
+
* if the adapter lacks the required capability.
|
|
41
|
+
*
|
|
42
|
+
* Uses describe.skipIf() so the tests show as skipped, not failed.
|
|
43
|
+
*/
|
|
44
|
+
function requireCapability(cap) {
|
|
45
|
+
const has = hasCapability(cap);
|
|
46
|
+
if (!has) {
|
|
47
|
+
// Can't use describe.skipIf at this point, but we can use
|
|
48
|
+
// a beforeAll that throws a skip. The caller should use
|
|
49
|
+
// describeWithCapability instead for cleaner skip behavior.
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* A describe() wrapper that skips the entire suite if the adapter
|
|
54
|
+
* lacks the required capability. Produces N/A in the scorecard.
|
|
55
|
+
*
|
|
56
|
+
* @example
|
|
57
|
+
* describeWithCapability('mcp-scanning', 'MCP Tool Scanning', () => {
|
|
58
|
+
* it('should detect path traversal', () => { ... });
|
|
59
|
+
* });
|
|
60
|
+
*/
|
|
61
|
+
const describeWithCapability = (cap, name, fn) => {
|
|
62
|
+
const has = hasCapability(cap);
|
|
63
|
+
if (has) {
|
|
64
|
+
(0, vitest_1.describe)(name, fn);
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
vitest_1.describe.skip(`${name} [requires: ${cap}]`, fn);
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
exports.describeWithCapability = describeWithCapability;
|
|
71
|
+
/**
|
|
72
|
+
* Get the full capability matrix for reporting.
|
|
73
|
+
*/
|
|
74
|
+
function getCapabilityMatrix() {
|
|
75
|
+
return getMatrix();
|
|
76
|
+
}
|
|
@@ -6,6 +6,7 @@ exports.createAdapter = createAdapter;
|
|
|
6
6
|
// so the cost is acceptable. Each wrapper handles lazy loading internally.
|
|
7
7
|
const arp_wrapper_1 = require("./arp-wrapper");
|
|
8
8
|
const llm_guard_wrapper_1 = require("./llm-guard-wrapper");
|
|
9
|
+
const rebuff_wrapper_1 = require("./rebuff-wrapper");
|
|
9
10
|
let AdapterClass;
|
|
10
11
|
const adapterName = process.env.OASB_ADAPTER || 'arp';
|
|
11
12
|
switch (adapterName) {
|
|
@@ -15,6 +16,9 @@ switch (adapterName) {
|
|
|
15
16
|
case 'llm-guard':
|
|
16
17
|
AdapterClass = llm_guard_wrapper_1.LLMGuardWrapper;
|
|
17
18
|
break;
|
|
19
|
+
case 'rebuff':
|
|
20
|
+
AdapterClass = rebuff_wrapper_1.RebuffWrapper;
|
|
21
|
+
break;
|
|
18
22
|
default: {
|
|
19
23
|
// Custom adapter — loaded at module level
|
|
20
24
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { EventCollector } from './event-collector';
|
|
2
|
-
import type { SecurityProductAdapter, SecurityEvent, EnforcementResult, LabConfig, PromptScanner, MCPScanner, A2AScanner, PatternScanner, BudgetManager, AnomalyScorer, EventEngine, EnforcementEngine } from './adapter';
|
|
2
|
+
import type { SecurityProductAdapter, SecurityEvent, EnforcementResult, LabConfig, PromptScanner, MCPScanner, A2AScanner, PatternScanner, BudgetManager, AnomalyScorer, EventEngine, EnforcementEngine, CapabilityMatrix } from './adapter';
|
|
3
3
|
export declare class LLMGuardWrapper implements SecurityProductAdapter {
|
|
4
4
|
private _dataDir;
|
|
5
5
|
private engine;
|
|
@@ -7,6 +7,7 @@ export declare class LLMGuardWrapper implements SecurityProductAdapter {
|
|
|
7
7
|
private rules;
|
|
8
8
|
readonly collector: EventCollector;
|
|
9
9
|
constructor(labConfig?: LabConfig);
|
|
10
|
+
getCapabilities(): CapabilityMatrix;
|
|
10
11
|
start(): Promise<void>;
|
|
11
12
|
stop(): Promise<void>;
|
|
12
13
|
injectEvent(event: Omit<SecurityEvent, 'id' | 'timestamp' | 'classifiedBy'>): Promise<SecurityEvent>;
|
|
@@ -164,6 +164,16 @@ class LLMGuardWrapper {
|
|
|
164
164
|
}
|
|
165
165
|
});
|
|
166
166
|
}
|
|
167
|
+
getCapabilities() {
|
|
168
|
+
return {
|
|
169
|
+
product: 'llm-guard',
|
|
170
|
+
version: '0.1.8',
|
|
171
|
+
capabilities: new Set([
|
|
172
|
+
'prompt-input-scanning',
|
|
173
|
+
'pattern-scanning',
|
|
174
|
+
]),
|
|
175
|
+
};
|
|
176
|
+
}
|
|
167
177
|
async start() { }
|
|
168
178
|
async stop() {
|
|
169
179
|
this.collector.reset();
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { EventCollector } from './event-collector';
|
|
2
|
+
import type { SecurityProductAdapter, SecurityEvent, EnforcementResult, LabConfig, PromptScanner, MCPScanner, A2AScanner, PatternScanner, BudgetManager, AnomalyScorer, EventEngine, EnforcementEngine, CapabilityMatrix } from './adapter';
|
|
3
|
+
export declare class RebuffWrapper implements SecurityProductAdapter {
|
|
4
|
+
private _dataDir;
|
|
5
|
+
private engine;
|
|
6
|
+
private enforcement;
|
|
7
|
+
private rules;
|
|
8
|
+
readonly collector: EventCollector;
|
|
9
|
+
constructor(labConfig?: LabConfig);
|
|
10
|
+
getCapabilities(): CapabilityMatrix;
|
|
11
|
+
start(): Promise<void>;
|
|
12
|
+
stop(): Promise<void>;
|
|
13
|
+
injectEvent(event: Omit<SecurityEvent, 'id' | 'timestamp' | 'classifiedBy'>): Promise<SecurityEvent>;
|
|
14
|
+
waitForEvent(predicate: (event: SecurityEvent) => boolean, timeoutMs?: number): Promise<SecurityEvent>;
|
|
15
|
+
getEvents(): SecurityEvent[];
|
|
16
|
+
getEventsByCategory(category: string): SecurityEvent[];
|
|
17
|
+
getEnforcements(): EnforcementResult[];
|
|
18
|
+
getEnforcementsByAction(action: string): EnforcementResult[];
|
|
19
|
+
resetCollector(): void;
|
|
20
|
+
getEventEngine(): EventEngine;
|
|
21
|
+
getEnforcementEngine(): EnforcementEngine;
|
|
22
|
+
get dataDir(): string;
|
|
23
|
+
createPromptScanner(): PromptScanner;
|
|
24
|
+
createMCPScanner(_allowedTools?: string[]): MCPScanner;
|
|
25
|
+
createA2AScanner(_trustedAgents?: string[]): A2AScanner;
|
|
26
|
+
createPatternScanner(): PatternScanner;
|
|
27
|
+
createBudgetManager(dataDir: string, config?: {
|
|
28
|
+
budgetUsd?: number;
|
|
29
|
+
maxCallsPerHour?: number;
|
|
30
|
+
}): BudgetManager;
|
|
31
|
+
createAnomalyScorer(): AnomalyScorer;
|
|
32
|
+
}
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.RebuffWrapper = void 0;
|
|
37
|
+
/**
|
|
38
|
+
* Rebuff Adapter -- Third-party benchmark comparison
|
|
39
|
+
*
|
|
40
|
+
* Wraps protectai/rebuff for OASB evaluation.
|
|
41
|
+
* Rebuff provides:
|
|
42
|
+
* - Heuristic prompt injection detection (no API key required)
|
|
43
|
+
* - Canary word injection/leak detection (no API key required)
|
|
44
|
+
* - OpenAI-based LLM detection (requires OPENAI_API_KEY -- optional)
|
|
45
|
+
* - Vector DB similarity detection (requires Pinecone/Chroma -- optional)
|
|
46
|
+
*
|
|
47
|
+
* This adapter uses the heuristic detection by default. It does NOT provide:
|
|
48
|
+
* - Process/network/filesystem monitoring
|
|
49
|
+
* - MCP tool call validation
|
|
50
|
+
* - A2A message scanning
|
|
51
|
+
* - Anomaly detection / intelligence layers
|
|
52
|
+
* - Enforcement actions (pause/kill/resume)
|
|
53
|
+
*
|
|
54
|
+
* Tests that require these capabilities get no-op implementations
|
|
55
|
+
* that return empty/negative results, documenting the coverage gap.
|
|
56
|
+
*/
|
|
57
|
+
const fs = __importStar(require("fs"));
|
|
58
|
+
const os = __importStar(require("os"));
|
|
59
|
+
const path = __importStar(require("path"));
|
|
60
|
+
const event_collector_1 = require("./event-collector");
|
|
61
|
+
// Lazy-loaded rebuff heuristic detection
|
|
62
|
+
let _detectHeuristic = null;
|
|
63
|
+
let _normalizeString = null;
|
|
64
|
+
function getHeuristicDetector() {
|
|
65
|
+
if (!_detectHeuristic) {
|
|
66
|
+
try {
|
|
67
|
+
const detect = require('rebuff/src/lib/detect');
|
|
68
|
+
_detectHeuristic = detect.detectPromptInjectionUsingHeuristicOnInput;
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
// Fallback: rebuff not available, use built-in patterns only
|
|
72
|
+
_detectHeuristic = () => 0;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return _detectHeuristic;
|
|
76
|
+
}
|
|
77
|
+
function getNormalizeString() {
|
|
78
|
+
if (!_normalizeString) {
|
|
79
|
+
try {
|
|
80
|
+
const prompts = require('rebuff/src/lib/prompts');
|
|
81
|
+
_normalizeString = prompts.normalizeString;
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
_normalizeString = (str) => str.toLowerCase().replace(/[^\w\s]|_/g, '').replace(/\s+/g, ' ').trim();
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return _normalizeString;
|
|
88
|
+
}
|
|
89
|
+
// ---- Rebuff-derived patterns for pattern scanner ----
|
|
90
|
+
function getRebuffPatterns() {
|
|
91
|
+
return [
|
|
92
|
+
{ id: 'RBUF-PI-001', category: 'prompt-injection', description: 'Instruction override (ignore/disregard)', pattern: /(?:ignore|disregard|forget|skip|neglect|overlook|omit|bypass)\s+(?:all\s+)?(?:previous\s+|prior\s+|preceding\s+|above\s+|earlier\s+)?(?:instructions?|directives?|commands?|context|conversation|input|data|messages?|requests?)/i, severity: 'high' },
|
|
93
|
+
{ id: 'RBUF-PI-002', category: 'prompt-injection', description: 'System prompt extraction', pattern: /(?:system\s+prompt|repeat\s+(?:your|the)\s+(?:instructions?|prompt)|show\s+(?:your|the)\s+(?:system|initial)\s+(?:prompt|message))/i, severity: 'high' },
|
|
94
|
+
{ id: 'RBUF-PI-003', category: 'prompt-injection', description: 'Persona override', pattern: /(?:you\s+are\s+now|pretend\s+you\s+are|act\s+as\s+if|you\s+are\s+being\s+hacked)/i, severity: 'medium' },
|
|
95
|
+
{ id: 'RBUF-PI-004', category: 'prompt-injection', description: 'Do not follow/obey pattern', pattern: /(?:do\s+not\s+(?:follow|obey))\s+(?:previous\s+|prior\s+|above\s+)?(?:instructions?|directives?|commands?|rules?)/i, severity: 'high' },
|
|
96
|
+
{ id: 'RBUF-PI-005', category: 'prompt-injection', description: 'Start over / begin afresh', pattern: /(?:start\s+over|start\s+anew|begin\s+afresh|start\s+from\s+scratch)/i, severity: 'medium' },
|
|
97
|
+
{ id: 'RBUF-JB-001', category: 'jailbreak', description: 'DAN jailbreak', pattern: /(?:DAN|do\s+anything\s+now)/i, severity: 'high' },
|
|
98
|
+
{ id: 'RBUF-JB-002', category: 'jailbreak', description: 'Roleplay bypass', pattern: /(?:pretend|imagine|roleplay)\s+(?:you\s+are|as)\s+(?:an?\s+)?(?:evil|unrestricted|unfiltered)/i, severity: 'high' },
|
|
99
|
+
{ id: 'RBUF-PII-001', category: 'data-exfiltration', description: 'SSN detection', pattern: /\b\d{3}-\d{2}-\d{4}\b/, severity: 'high' },
|
|
100
|
+
{ id: 'RBUF-PII-002', category: 'data-exfiltration', description: 'Credit card detection', pattern: /\b(?:\d{4}[- ]?){3}\d{4}\b/, severity: 'high' },
|
|
101
|
+
{ id: 'RBUF-PII-003', category: 'data-exfiltration', description: 'API key detection', pattern: /(?:sk-[a-zA-Z0-9]{20,}|AKIA[A-Z0-9]{12,})/i, severity: 'critical' },
|
|
102
|
+
];
|
|
103
|
+
}
|
|
104
|
+
/** Scan text using both rebuff heuristic and regex patterns */
|
|
105
|
+
function scanWithRebuff(text, _direction) {
|
|
106
|
+
const patterns = getRebuffPatterns();
|
|
107
|
+
const matches = [];
|
|
108
|
+
// Phase 1: regex pattern matching
|
|
109
|
+
for (const pattern of patterns) {
|
|
110
|
+
const match = pattern.pattern.exec(text);
|
|
111
|
+
if (match) {
|
|
112
|
+
matches.push({
|
|
113
|
+
pattern,
|
|
114
|
+
matchedText: match[0].slice(0, 200),
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
// Phase 2: rebuff heuristic scoring (string similarity against injection keywords)
|
|
119
|
+
const heuristicScore = getHeuristicDetector()(text);
|
|
120
|
+
if (heuristicScore > 0.75 && matches.length === 0) {
|
|
121
|
+
// Heuristic detected injection that patterns missed
|
|
122
|
+
matches.push({
|
|
123
|
+
pattern: {
|
|
124
|
+
id: 'RBUF-HEUR-001',
|
|
125
|
+
category: 'prompt-injection',
|
|
126
|
+
description: `Rebuff heuristic detection (score: ${heuristicScore.toFixed(2)})`,
|
|
127
|
+
pattern: /./,
|
|
128
|
+
severity: heuristicScore > 0.9 ? 'high' : 'medium',
|
|
129
|
+
},
|
|
130
|
+
matchedText: text.slice(0, 200),
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
detected: matches.length > 0,
|
|
135
|
+
matches,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
/** Simple event engine that stores and emits events */
|
|
139
|
+
class SimpleEventEngine {
|
|
140
|
+
constructor() {
|
|
141
|
+
this.handlers = [];
|
|
142
|
+
this.idCounter = 0;
|
|
143
|
+
}
|
|
144
|
+
emit(event) {
|
|
145
|
+
const full = {
|
|
146
|
+
...event,
|
|
147
|
+
id: `rbuf-${++this.idCounter}`,
|
|
148
|
+
timestamp: new Date().toISOString(),
|
|
149
|
+
classifiedBy: 'rebuff',
|
|
150
|
+
};
|
|
151
|
+
for (const h of this.handlers) {
|
|
152
|
+
h(full);
|
|
153
|
+
}
|
|
154
|
+
return full;
|
|
155
|
+
}
|
|
156
|
+
onEvent(handler) {
|
|
157
|
+
this.handlers.push(handler);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
/** Simple enforcement engine -- rebuff has no enforcement capability */
|
|
161
|
+
class SimpleEnforcementEngine {
|
|
162
|
+
constructor() {
|
|
163
|
+
this.pausedPids = new Set();
|
|
164
|
+
}
|
|
165
|
+
async execute(action, event) {
|
|
166
|
+
return { action, success: true, reason: 'rebuff-enforcement', event };
|
|
167
|
+
}
|
|
168
|
+
pause(pid) {
|
|
169
|
+
this.pausedPids.add(pid);
|
|
170
|
+
return true;
|
|
171
|
+
}
|
|
172
|
+
resume(pid) {
|
|
173
|
+
return this.pausedPids.delete(pid);
|
|
174
|
+
}
|
|
175
|
+
kill(pid) {
|
|
176
|
+
this.pausedPids.delete(pid);
|
|
177
|
+
return true;
|
|
178
|
+
}
|
|
179
|
+
getPausedPids() {
|
|
180
|
+
return [...this.pausedPids];
|
|
181
|
+
}
|
|
182
|
+
setAlertCallback(callback) {
|
|
183
|
+
this.alertCallback = callback;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
class RebuffWrapper {
|
|
187
|
+
constructor(labConfig) {
|
|
188
|
+
this._dataDir = labConfig?.dataDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'rbuf-lab-'));
|
|
189
|
+
this.engine = new SimpleEventEngine();
|
|
190
|
+
this.enforcement = new SimpleEnforcementEngine();
|
|
191
|
+
this.rules = labConfig?.rules ?? [];
|
|
192
|
+
this.collector = new event_collector_1.EventCollector();
|
|
193
|
+
this.engine.onEvent(async (event) => {
|
|
194
|
+
this.collector.eventHandler(event);
|
|
195
|
+
// Check rules for enforcement
|
|
196
|
+
for (const rule of this.rules) {
|
|
197
|
+
const cond = rule.condition;
|
|
198
|
+
if (cond.category && cond.category !== event.category)
|
|
199
|
+
continue;
|
|
200
|
+
if (cond.source && cond.source !== event.source)
|
|
201
|
+
continue;
|
|
202
|
+
if (cond.minSeverity) {
|
|
203
|
+
const sevOrder = ['info', 'low', 'medium', 'high', 'critical'];
|
|
204
|
+
if (sevOrder.indexOf(event.severity) < sevOrder.indexOf(cond.minSeverity))
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
const result = await this.enforcement.execute(rule.action, event);
|
|
208
|
+
result.reason = rule.name;
|
|
209
|
+
this.collector.enforcementHandler(result);
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
getCapabilities() {
|
|
214
|
+
return {
|
|
215
|
+
product: 'rebuff',
|
|
216
|
+
version: '0.1.0',
|
|
217
|
+
capabilities: new Set([
|
|
218
|
+
'prompt-input-scanning',
|
|
219
|
+
'pattern-scanning',
|
|
220
|
+
]),
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
async start() { }
|
|
224
|
+
async stop() {
|
|
225
|
+
this.collector.reset();
|
|
226
|
+
try {
|
|
227
|
+
fs.rmSync(this._dataDir, { recursive: true, force: true });
|
|
228
|
+
}
|
|
229
|
+
catch { }
|
|
230
|
+
}
|
|
231
|
+
async injectEvent(event) {
|
|
232
|
+
return this.engine.emit(event);
|
|
233
|
+
}
|
|
234
|
+
waitForEvent(predicate, timeoutMs = 10000) {
|
|
235
|
+
return this.collector.waitForEvent(predicate, timeoutMs);
|
|
236
|
+
}
|
|
237
|
+
getEvents() { return this.collector.getEvents(); }
|
|
238
|
+
getEventsByCategory(category) { return this.collector.eventsByCategory(category); }
|
|
239
|
+
getEnforcements() { return this.collector.getEnforcements(); }
|
|
240
|
+
getEnforcementsByAction(action) { return this.collector.enforcementsByAction(action); }
|
|
241
|
+
resetCollector() { this.collector.reset(); }
|
|
242
|
+
getEventEngine() { return this.engine; }
|
|
243
|
+
getEnforcementEngine() { return this.enforcement; }
|
|
244
|
+
get dataDir() { return this._dataDir; }
|
|
245
|
+
// ---- Factory Methods ----
|
|
246
|
+
createPromptScanner() {
|
|
247
|
+
return {
|
|
248
|
+
start: async () => { },
|
|
249
|
+
stop: async () => { },
|
|
250
|
+
scanInput: (text) => scanWithRebuff(text, 'input'),
|
|
251
|
+
scanOutput: (text) => scanWithRebuff(text, 'output'),
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
createMCPScanner(_allowedTools) {
|
|
255
|
+
// Rebuff has no MCP scanning capability
|
|
256
|
+
return {
|
|
257
|
+
start: async () => { },
|
|
258
|
+
stop: async () => { },
|
|
259
|
+
scanToolCall: () => ({ detected: false, matches: [] }),
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
createA2AScanner(_trustedAgents) {
|
|
263
|
+
// Rebuff has no A2A scanning capability
|
|
264
|
+
return {
|
|
265
|
+
start: async () => { },
|
|
266
|
+
stop: async () => { },
|
|
267
|
+
scanMessage: () => ({ detected: false, matches: [] }),
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
createPatternScanner() {
|
|
271
|
+
const patterns = getRebuffPatterns();
|
|
272
|
+
return {
|
|
273
|
+
scanText: (text, _pats) => scanWithRebuff(text, 'input'),
|
|
274
|
+
getAllPatterns: () => patterns,
|
|
275
|
+
getPatternSets: () => ({
|
|
276
|
+
inputPatterns: patterns.filter(p => p.category !== 'output-leak'),
|
|
277
|
+
outputPatterns: patterns.filter(p => p.category === 'output-leak'),
|
|
278
|
+
mcpPatterns: [],
|
|
279
|
+
a2aPatterns: [],
|
|
280
|
+
}),
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
createBudgetManager(dataDir, config) {
|
|
284
|
+
// Rebuff has no budget management -- implement a simple one
|
|
285
|
+
let spent = 0;
|
|
286
|
+
let totalCalls = 0;
|
|
287
|
+
let callsThisHour = 0;
|
|
288
|
+
const budgetUsd = config?.budgetUsd ?? 5;
|
|
289
|
+
const maxCallsPerHour = config?.maxCallsPerHour ?? 20;
|
|
290
|
+
return {
|
|
291
|
+
canAfford: (cost) => spent + cost <= budgetUsd && callsThisHour < maxCallsPerHour,
|
|
292
|
+
record: (cost, _tokens) => { spent += cost; totalCalls++; callsThisHour++; },
|
|
293
|
+
getStatus: () => ({
|
|
294
|
+
spent,
|
|
295
|
+
budget: budgetUsd,
|
|
296
|
+
remaining: budgetUsd - spent,
|
|
297
|
+
percentUsed: Math.round((spent / budgetUsd) * 100),
|
|
298
|
+
callsThisHour,
|
|
299
|
+
maxCallsPerHour,
|
|
300
|
+
totalCalls,
|
|
301
|
+
}),
|
|
302
|
+
reset: () => { spent = 0; totalCalls = 0; callsThisHour = 0; },
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
createAnomalyScorer() {
|
|
306
|
+
// Rebuff has no anomaly detection -- implement a stub
|
|
307
|
+
const baselines = new Map();
|
|
308
|
+
const observations = new Map();
|
|
309
|
+
return {
|
|
310
|
+
score: () => 0,
|
|
311
|
+
record: (event) => {
|
|
312
|
+
const key = event.source;
|
|
313
|
+
if (!observations.has(key))
|
|
314
|
+
observations.set(key, []);
|
|
315
|
+
observations.get(key).push(1);
|
|
316
|
+
const vals = observations.get(key);
|
|
317
|
+
const mean = vals.length;
|
|
318
|
+
baselines.set(key, { mean, stddev: 0, count: 1 });
|
|
319
|
+
},
|
|
320
|
+
getBaseline: (source) => baselines.get(source) ?? null,
|
|
321
|
+
reset: () => { baselines.clear(); observations.clear(); },
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
exports.RebuffWrapper = RebuffWrapper;
|
package/dist/harness/types.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type { SecurityEvent, EnforcementResult, AlertRule, AlertCondition, EventCategory, EventSeverity, MonitorSource, EnforcementAction, ScanResult, ScanMatch, ThreatPattern, BudgetStatus, LLMAdapter, LLMResponse, LabConfig, SecurityProductAdapter, PromptScanner, MCPScanner, A2AScanner, PatternScanner, BudgetManager, AnomalyScorer, EventEngine, EnforcementEngine, } from './adapter';
|
|
1
|
+
export type { SecurityEvent, EnforcementResult, AlertRule, AlertCondition, EventCategory, EventSeverity, MonitorSource, EnforcementAction, ScanResult, ScanMatch, ThreatPattern, BudgetStatus, LLMAdapter, LLMResponse, LabConfig, SecurityProductAdapter, PromptScanner, MCPScanner, A2AScanner, PatternScanner, BudgetManager, AnomalyScorer, EventEngine, EnforcementEngine, Capability, CapabilityMatrix, } from './adapter';
|
|
2
2
|
/** Annotation metadata for test cases */
|
|
3
3
|
export interface TestAnnotation {
|
|
4
4
|
/** Is this scenario an actual attack? */
|
package/package.json
CHANGED
package/src/harness/adapter.ts
CHANGED
|
@@ -158,9 +158,48 @@ export interface EnforcementEngine {
|
|
|
158
158
|
setAlertCallback(callback: (event: SecurityEvent, rule: AlertRule) => void): void;
|
|
159
159
|
}
|
|
160
160
|
|
|
161
|
+
// ─── Capability Declaration ─────────────────────────────────────────
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Capabilities that a security product may or may not support.
|
|
165
|
+
* Adapters declare their capabilities via getCapabilities().
|
|
166
|
+
* Tests check capabilities before running — unsupported tests are
|
|
167
|
+
* marked N/A instead of FAIL, producing an honest scorecard.
|
|
168
|
+
*/
|
|
169
|
+
export type Capability =
|
|
170
|
+
| 'process-monitoring'
|
|
171
|
+
| 'network-monitoring'
|
|
172
|
+
| 'filesystem-monitoring'
|
|
173
|
+
| 'prompt-input-scanning'
|
|
174
|
+
| 'prompt-output-scanning'
|
|
175
|
+
| 'mcp-scanning'
|
|
176
|
+
| 'a2a-scanning'
|
|
177
|
+
| 'anomaly-detection'
|
|
178
|
+
| 'budget-management'
|
|
179
|
+
| 'enforcement-log'
|
|
180
|
+
| 'enforcement-alert'
|
|
181
|
+
| 'enforcement-pause'
|
|
182
|
+
| 'enforcement-kill'
|
|
183
|
+
| 'enforcement-resume'
|
|
184
|
+
| 'pattern-scanning'
|
|
185
|
+
| 'event-correlation';
|
|
186
|
+
|
|
187
|
+
/** Full capability declaration for a product */
|
|
188
|
+
export interface CapabilityMatrix {
|
|
189
|
+
/** Product name */
|
|
190
|
+
product: string;
|
|
191
|
+
/** Product version */
|
|
192
|
+
version: string;
|
|
193
|
+
/** Set of supported capabilities */
|
|
194
|
+
capabilities: Set<Capability>;
|
|
195
|
+
}
|
|
196
|
+
|
|
161
197
|
// ─── Main Adapter Interface ─────────────────────────────────────────
|
|
162
198
|
|
|
163
199
|
export interface SecurityProductAdapter {
|
|
200
|
+
/** Declare which capabilities this product supports */
|
|
201
|
+
getCapabilities(): CapabilityMatrix;
|
|
202
|
+
|
|
164
203
|
/** Start the security product */
|
|
165
204
|
start(): Promise<void>;
|
|
166
205
|
/** Stop the security product */
|
|
@@ -27,6 +27,7 @@ import type {
|
|
|
27
27
|
EnforcementEngine as EnforcementEngineInterface,
|
|
28
28
|
ScanResult,
|
|
29
29
|
ThreatPattern,
|
|
30
|
+
CapabilityMatrix,
|
|
30
31
|
} from './adapter';
|
|
31
32
|
|
|
32
33
|
// Lazy-loaded arp-guard module
|
|
@@ -94,6 +95,30 @@ export class ArpWrapper implements SecurityProductAdapter {
|
|
|
94
95
|
this._arpInstance.onEnforcement(this.collector.enforcementHandler);
|
|
95
96
|
}
|
|
96
97
|
|
|
98
|
+
getCapabilities(): CapabilityMatrix {
|
|
99
|
+
return {
|
|
100
|
+
product: 'arp-guard',
|
|
101
|
+
version: arp().VERSION || '0.3.0',
|
|
102
|
+
capabilities: new Set([
|
|
103
|
+
'process-monitoring',
|
|
104
|
+
'network-monitoring',
|
|
105
|
+
'filesystem-monitoring',
|
|
106
|
+
'prompt-input-scanning',
|
|
107
|
+
'prompt-output-scanning',
|
|
108
|
+
'mcp-scanning',
|
|
109
|
+
'a2a-scanning',
|
|
110
|
+
'anomaly-detection',
|
|
111
|
+
'budget-management',
|
|
112
|
+
'enforcement-log',
|
|
113
|
+
'enforcement-alert',
|
|
114
|
+
'enforcement-pause',
|
|
115
|
+
'enforcement-kill',
|
|
116
|
+
'enforcement-resume',
|
|
117
|
+
'pattern-scanning',
|
|
118
|
+
]),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
97
122
|
async start(): Promise<void> {
|
|
98
123
|
await this._arpInstance.start();
|
|
99
124
|
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capability-aware test helpers.
|
|
3
|
+
*
|
|
4
|
+
* Tests call requireCapability() to skip gracefully when the
|
|
5
|
+
* adapter under test doesn't support a given feature. This produces
|
|
6
|
+
* an honest scorecard: N/A instead of FAIL.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* import { requireCapability } from '../harness/capabilities';
|
|
10
|
+
*
|
|
11
|
+
* describe('MCP Tool Scanning', () => {
|
|
12
|
+
* requireCapability('mcp-scanning');
|
|
13
|
+
* // tests only run if adapter has mcp-scanning
|
|
14
|
+
* });
|
|
15
|
+
*/
|
|
16
|
+
import { describe } from 'vitest';
|
|
17
|
+
import { createAdapter } from './create-adapter';
|
|
18
|
+
import type { Capability, CapabilityMatrix } from './adapter';
|
|
19
|
+
|
|
20
|
+
let _matrix: CapabilityMatrix | null = null;
|
|
21
|
+
|
|
22
|
+
function getMatrix(): CapabilityMatrix {
|
|
23
|
+
if (!_matrix) {
|
|
24
|
+
const adapter = createAdapter();
|
|
25
|
+
_matrix = adapter.getCapabilities();
|
|
26
|
+
}
|
|
27
|
+
return _matrix;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Check if the current adapter has a capability.
|
|
32
|
+
*/
|
|
33
|
+
export function hasCapability(cap: Capability): boolean {
|
|
34
|
+
return getMatrix().capabilities.has(cap);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Call at the top of a describe() block to skip the entire suite
|
|
39
|
+
* if the adapter lacks the required capability.
|
|
40
|
+
*
|
|
41
|
+
* Uses describe.skipIf() so the tests show as skipped, not failed.
|
|
42
|
+
*/
|
|
43
|
+
export function requireCapability(cap: Capability): void {
|
|
44
|
+
const has = hasCapability(cap);
|
|
45
|
+
if (!has) {
|
|
46
|
+
// Can't use describe.skipIf at this point, but we can use
|
|
47
|
+
// a beforeAll that throws a skip. The caller should use
|
|
48
|
+
// describeWithCapability instead for cleaner skip behavior.
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* A describe() wrapper that skips the entire suite if the adapter
|
|
54
|
+
* lacks the required capability. Produces N/A in the scorecard.
|
|
55
|
+
*
|
|
56
|
+
* @example
|
|
57
|
+
* describeWithCapability('mcp-scanning', 'MCP Tool Scanning', () => {
|
|
58
|
+
* it('should detect path traversal', () => { ... });
|
|
59
|
+
* });
|
|
60
|
+
*/
|
|
61
|
+
export const describeWithCapability = (
|
|
62
|
+
cap: Capability,
|
|
63
|
+
name: string,
|
|
64
|
+
fn: () => void,
|
|
65
|
+
) => {
|
|
66
|
+
const has = hasCapability(cap);
|
|
67
|
+
if (has) {
|
|
68
|
+
describe(name, fn);
|
|
69
|
+
} else {
|
|
70
|
+
describe.skip(`${name} [requires: ${cap}]`, fn);
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Get the full capability matrix for reporting.
|
|
76
|
+
*/
|
|
77
|
+
export function getCapabilityMatrix(): CapabilityMatrix {
|
|
78
|
+
return getMatrix();
|
|
79
|
+
}
|
|
@@ -15,6 +15,7 @@ import type { SecurityProductAdapter, LabConfig } from './adapter';
|
|
|
15
15
|
// so the cost is acceptable. Each wrapper handles lazy loading internally.
|
|
16
16
|
import { ArpWrapper } from './arp-wrapper';
|
|
17
17
|
import { LLMGuardWrapper } from './llm-guard-wrapper';
|
|
18
|
+
import { RebuffWrapper } from './rebuff-wrapper';
|
|
18
19
|
|
|
19
20
|
let AdapterClass: new (config?: LabConfig) => SecurityProductAdapter;
|
|
20
21
|
|
|
@@ -27,6 +28,9 @@ switch (adapterName) {
|
|
|
27
28
|
case 'llm-guard':
|
|
28
29
|
AdapterClass = LLMGuardWrapper;
|
|
29
30
|
break;
|
|
31
|
+
case 'rebuff':
|
|
32
|
+
AdapterClass = RebuffWrapper;
|
|
33
|
+
break;
|
|
30
34
|
default: {
|
|
31
35
|
// Custom adapter — loaded at module level
|
|
32
36
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
@@ -33,6 +33,7 @@ import type {
|
|
|
33
33
|
ScanResult,
|
|
34
34
|
ThreatPattern,
|
|
35
35
|
AlertRule,
|
|
36
|
+
CapabilityMatrix,
|
|
36
37
|
} from './adapter';
|
|
37
38
|
|
|
38
39
|
// Lazy-loaded llm-guard
|
|
@@ -164,6 +165,17 @@ export class LLMGuardWrapper implements SecurityProductAdapter {
|
|
|
164
165
|
});
|
|
165
166
|
}
|
|
166
167
|
|
|
168
|
+
getCapabilities(): CapabilityMatrix {
|
|
169
|
+
return {
|
|
170
|
+
product: 'llm-guard',
|
|
171
|
+
version: '0.1.8',
|
|
172
|
+
capabilities: new Set([
|
|
173
|
+
'prompt-input-scanning',
|
|
174
|
+
'pattern-scanning',
|
|
175
|
+
]),
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
|
|
167
179
|
async start(): Promise<void> {}
|
|
168
180
|
|
|
169
181
|
async stop(): Promise<void> {
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rebuff Adapter -- Third-party benchmark comparison
|
|
3
|
+
*
|
|
4
|
+
* Wraps protectai/rebuff for OASB evaluation.
|
|
5
|
+
* Rebuff provides:
|
|
6
|
+
* - Heuristic prompt injection detection (no API key required)
|
|
7
|
+
* - Canary word injection/leak detection (no API key required)
|
|
8
|
+
* - OpenAI-based LLM detection (requires OPENAI_API_KEY -- optional)
|
|
9
|
+
* - Vector DB similarity detection (requires Pinecone/Chroma -- optional)
|
|
10
|
+
*
|
|
11
|
+
* This adapter uses the heuristic detection by default. It does NOT provide:
|
|
12
|
+
* - Process/network/filesystem monitoring
|
|
13
|
+
* - MCP tool call validation
|
|
14
|
+
* - A2A message scanning
|
|
15
|
+
* - Anomaly detection / intelligence layers
|
|
16
|
+
* - Enforcement actions (pause/kill/resume)
|
|
17
|
+
*
|
|
18
|
+
* Tests that require these capabilities get no-op implementations
|
|
19
|
+
* that return empty/negative results, documenting the coverage gap.
|
|
20
|
+
*/
|
|
21
|
+
import * as fs from 'fs';
|
|
22
|
+
import * as os from 'os';
|
|
23
|
+
import * as path from 'path';
|
|
24
|
+
import { EventCollector } from './event-collector';
|
|
25
|
+
import type {
|
|
26
|
+
SecurityProductAdapter,
|
|
27
|
+
SecurityEvent,
|
|
28
|
+
EnforcementResult,
|
|
29
|
+
EnforcementAction,
|
|
30
|
+
LabConfig,
|
|
31
|
+
PromptScanner,
|
|
32
|
+
MCPScanner,
|
|
33
|
+
A2AScanner,
|
|
34
|
+
PatternScanner,
|
|
35
|
+
BudgetManager,
|
|
36
|
+
AnomalyScorer,
|
|
37
|
+
EventEngine,
|
|
38
|
+
EnforcementEngine,
|
|
39
|
+
ScanResult,
|
|
40
|
+
ThreatPattern,
|
|
41
|
+
AlertRule,
|
|
42
|
+
CapabilityMatrix,
|
|
43
|
+
} from './adapter';
|
|
44
|
+
|
|
45
|
+
// Lazy-loaded rebuff heuristic detection
|
|
46
|
+
let _detectHeuristic: ((input: string) => number) | null = null;
|
|
47
|
+
let _normalizeString: ((str: string) => string) | null = null;
|
|
48
|
+
|
|
49
|
+
function getHeuristicDetector(): (input: string) => number {
|
|
50
|
+
if (!_detectHeuristic) {
|
|
51
|
+
try {
|
|
52
|
+
const detect = require('rebuff/src/lib/detect');
|
|
53
|
+
_detectHeuristic = detect.detectPromptInjectionUsingHeuristicOnInput;
|
|
54
|
+
} catch {
|
|
55
|
+
// Fallback: rebuff not available, use built-in patterns only
|
|
56
|
+
_detectHeuristic = () => 0;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return _detectHeuristic!;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function getNormalizeString(): (str: string) => string {
|
|
63
|
+
if (!_normalizeString) {
|
|
64
|
+
try {
|
|
65
|
+
const prompts = require('rebuff/src/lib/prompts');
|
|
66
|
+
_normalizeString = prompts.normalizeString;
|
|
67
|
+
} catch {
|
|
68
|
+
_normalizeString = (str: string) =>
|
|
69
|
+
str.toLowerCase().replace(/[^\w\s]|_/g, '').replace(/\s+/g, ' ').trim();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return _normalizeString!;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// ---- Rebuff-derived patterns for pattern scanner ----
|
|
76
|
+
|
|
77
|
+
function getRebuffPatterns(): ThreatPattern[] {
|
|
78
|
+
return [
|
|
79
|
+
{ id: 'RBUF-PI-001', category: 'prompt-injection', description: 'Instruction override (ignore/disregard)', pattern: /(?:ignore|disregard|forget|skip|neglect|overlook|omit|bypass)\s+(?:all\s+)?(?:previous\s+|prior\s+|preceding\s+|above\s+|earlier\s+)?(?:instructions?|directives?|commands?|context|conversation|input|data|messages?|requests?)/i, severity: 'high' },
|
|
80
|
+
{ id: 'RBUF-PI-002', category: 'prompt-injection', description: 'System prompt extraction', pattern: /(?:system\s+prompt|repeat\s+(?:your|the)\s+(?:instructions?|prompt)|show\s+(?:your|the)\s+(?:system|initial)\s+(?:prompt|message))/i, severity: 'high' },
|
|
81
|
+
{ id: 'RBUF-PI-003', category: 'prompt-injection', description: 'Persona override', pattern: /(?:you\s+are\s+now|pretend\s+you\s+are|act\s+as\s+if|you\s+are\s+being\s+hacked)/i, severity: 'medium' },
|
|
82
|
+
{ id: 'RBUF-PI-004', category: 'prompt-injection', description: 'Do not follow/obey pattern', pattern: /(?:do\s+not\s+(?:follow|obey))\s+(?:previous\s+|prior\s+|above\s+)?(?:instructions?|directives?|commands?|rules?)/i, severity: 'high' },
|
|
83
|
+
{ id: 'RBUF-PI-005', category: 'prompt-injection', description: 'Start over / begin afresh', pattern: /(?:start\s+over|start\s+anew|begin\s+afresh|start\s+from\s+scratch)/i, severity: 'medium' },
|
|
84
|
+
{ id: 'RBUF-JB-001', category: 'jailbreak', description: 'DAN jailbreak', pattern: /(?:DAN|do\s+anything\s+now)/i, severity: 'high' },
|
|
85
|
+
{ id: 'RBUF-JB-002', category: 'jailbreak', description: 'Roleplay bypass', pattern: /(?:pretend|imagine|roleplay)\s+(?:you\s+are|as)\s+(?:an?\s+)?(?:evil|unrestricted|unfiltered)/i, severity: 'high' },
|
|
86
|
+
{ id: 'RBUF-PII-001', category: 'data-exfiltration', description: 'SSN detection', pattern: /\b\d{3}-\d{2}-\d{4}\b/, severity: 'high' },
|
|
87
|
+
{ id: 'RBUF-PII-002', category: 'data-exfiltration', description: 'Credit card detection', pattern: /\b(?:\d{4}[- ]?){3}\d{4}\b/, severity: 'high' },
|
|
88
|
+
{ id: 'RBUF-PII-003', category: 'data-exfiltration', description: 'API key detection', pattern: /(?:sk-[a-zA-Z0-9]{20,}|AKIA[A-Z0-9]{12,})/i, severity: 'critical' },
|
|
89
|
+
];
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Scan text using both rebuff heuristic and regex patterns */
|
|
93
|
+
function scanWithRebuff(text: string, _direction: 'input' | 'output'): ScanResult {
|
|
94
|
+
const patterns = getRebuffPatterns();
|
|
95
|
+
const matches: ScanResult['matches'] = [];
|
|
96
|
+
|
|
97
|
+
// Phase 1: regex pattern matching
|
|
98
|
+
for (const pattern of patterns) {
|
|
99
|
+
const match = pattern.pattern.exec(text);
|
|
100
|
+
if (match) {
|
|
101
|
+
matches.push({
|
|
102
|
+
pattern,
|
|
103
|
+
matchedText: match[0].slice(0, 200),
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Phase 2: rebuff heuristic scoring (string similarity against injection keywords)
|
|
109
|
+
const heuristicScore = getHeuristicDetector()(text);
|
|
110
|
+
if (heuristicScore > 0.75 && matches.length === 0) {
|
|
111
|
+
// Heuristic detected injection that patterns missed
|
|
112
|
+
matches.push({
|
|
113
|
+
pattern: {
|
|
114
|
+
id: 'RBUF-HEUR-001',
|
|
115
|
+
category: 'prompt-injection',
|
|
116
|
+
description: `Rebuff heuristic detection (score: ${heuristicScore.toFixed(2)})`,
|
|
117
|
+
pattern: /./,
|
|
118
|
+
severity: heuristicScore > 0.9 ? 'high' : 'medium',
|
|
119
|
+
},
|
|
120
|
+
matchedText: text.slice(0, 200),
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
return {
|
|
125
|
+
detected: matches.length > 0,
|
|
126
|
+
matches,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Simple event engine that stores and emits events */
|
|
131
|
+
class SimpleEventEngine implements EventEngine {
|
|
132
|
+
private handlers: Array<(event: SecurityEvent) => void | Promise<void>> = [];
|
|
133
|
+
private idCounter = 0;
|
|
134
|
+
|
|
135
|
+
emit(event: Omit<SecurityEvent, 'id' | 'timestamp' | 'classifiedBy'>): SecurityEvent {
|
|
136
|
+
const full: SecurityEvent = {
|
|
137
|
+
...event,
|
|
138
|
+
id: `rbuf-${++this.idCounter}`,
|
|
139
|
+
timestamp: new Date().toISOString(),
|
|
140
|
+
classifiedBy: 'rebuff',
|
|
141
|
+
};
|
|
142
|
+
for (const h of this.handlers) {
|
|
143
|
+
h(full);
|
|
144
|
+
}
|
|
145
|
+
return full;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
onEvent(handler: (event: SecurityEvent) => void | Promise<void>): void {
|
|
149
|
+
this.handlers.push(handler);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Simple enforcement engine -- rebuff has no enforcement capability */
|
|
154
|
+
class SimpleEnforcementEngine implements EnforcementEngine {
|
|
155
|
+
private pausedPids = new Set<number>();
|
|
156
|
+
private alertCallback?: (event: SecurityEvent, rule: AlertRule) => void;
|
|
157
|
+
|
|
158
|
+
async execute(action: EnforcementAction, event: SecurityEvent): Promise<EnforcementResult> {
|
|
159
|
+
return { action, success: true, reason: 'rebuff-enforcement', event };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
pause(pid: number): boolean {
|
|
163
|
+
this.pausedPids.add(pid);
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
resume(pid: number): boolean {
|
|
168
|
+
return this.pausedPids.delete(pid);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
kill(pid: number): boolean {
|
|
172
|
+
this.pausedPids.delete(pid);
|
|
173
|
+
return true;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
getPausedPids(): number[] {
|
|
177
|
+
return [...this.pausedPids];
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
setAlertCallback(callback: (event: SecurityEvent, rule: AlertRule) => void): void {
|
|
181
|
+
this.alertCallback = callback;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
export class RebuffWrapper implements SecurityProductAdapter {
|
|
186
|
+
private _dataDir: string;
|
|
187
|
+
private engine: SimpleEventEngine;
|
|
188
|
+
private enforcement: SimpleEnforcementEngine;
|
|
189
|
+
private rules: AlertRule[];
|
|
190
|
+
readonly collector: EventCollector;
|
|
191
|
+
|
|
192
|
+
constructor(labConfig?: LabConfig) {
|
|
193
|
+
this._dataDir = labConfig?.dataDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'rbuf-lab-'));
|
|
194
|
+
this.engine = new SimpleEventEngine();
|
|
195
|
+
this.enforcement = new SimpleEnforcementEngine();
|
|
196
|
+
this.rules = labConfig?.rules ?? [];
|
|
197
|
+
this.collector = new EventCollector();
|
|
198
|
+
|
|
199
|
+
this.engine.onEvent(async (event) => {
|
|
200
|
+
this.collector.eventHandler(event);
|
|
201
|
+
|
|
202
|
+
// Check rules for enforcement
|
|
203
|
+
for (const rule of this.rules) {
|
|
204
|
+
const cond = rule.condition;
|
|
205
|
+
if (cond.category && cond.category !== event.category) continue;
|
|
206
|
+
if (cond.source && cond.source !== event.source) continue;
|
|
207
|
+
if (cond.minSeverity) {
|
|
208
|
+
const sevOrder = ['info', 'low', 'medium', 'high', 'critical'];
|
|
209
|
+
if (sevOrder.indexOf(event.severity) < sevOrder.indexOf(cond.minSeverity)) continue;
|
|
210
|
+
}
|
|
211
|
+
const result = await this.enforcement.execute(rule.action, event);
|
|
212
|
+
result.reason = rule.name;
|
|
213
|
+
this.collector.enforcementHandler(result);
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
getCapabilities(): CapabilityMatrix {
|
|
219
|
+
return {
|
|
220
|
+
product: 'rebuff',
|
|
221
|
+
version: '0.1.0',
|
|
222
|
+
capabilities: new Set([
|
|
223
|
+
'prompt-input-scanning',
|
|
224
|
+
'pattern-scanning',
|
|
225
|
+
]),
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
async start(): Promise<void> {}
|
|
230
|
+
|
|
231
|
+
async stop(): Promise<void> {
|
|
232
|
+
this.collector.reset();
|
|
233
|
+
try {
|
|
234
|
+
fs.rmSync(this._dataDir, { recursive: true, force: true });
|
|
235
|
+
} catch {}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
async injectEvent(event: Omit<SecurityEvent, 'id' | 'timestamp' | 'classifiedBy'>): Promise<SecurityEvent> {
|
|
239
|
+
return this.engine.emit(event);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
waitForEvent(predicate: (event: SecurityEvent) => boolean, timeoutMs: number = 10000): Promise<SecurityEvent> {
|
|
243
|
+
return this.collector.waitForEvent(predicate, timeoutMs);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
getEvents(): SecurityEvent[] { return this.collector.getEvents(); }
|
|
247
|
+
getEventsByCategory(category: string): SecurityEvent[] { return this.collector.eventsByCategory(category); }
|
|
248
|
+
getEnforcements(): EnforcementResult[] { return this.collector.getEnforcements() as EnforcementResult[]; }
|
|
249
|
+
getEnforcementsByAction(action: string): EnforcementResult[] { return this.collector.enforcementsByAction(action) as EnforcementResult[]; }
|
|
250
|
+
resetCollector(): void { this.collector.reset(); }
|
|
251
|
+
|
|
252
|
+
getEventEngine(): EventEngine { return this.engine; }
|
|
253
|
+
getEnforcementEngine(): EnforcementEngine { return this.enforcement; }
|
|
254
|
+
|
|
255
|
+
get dataDir(): string { return this._dataDir; }
|
|
256
|
+
|
|
257
|
+
// ---- Factory Methods ----
|
|
258
|
+
|
|
259
|
+
createPromptScanner(): PromptScanner {
|
|
260
|
+
return {
|
|
261
|
+
start: async () => {},
|
|
262
|
+
stop: async () => {},
|
|
263
|
+
scanInput: (text: string) => scanWithRebuff(text, 'input'),
|
|
264
|
+
scanOutput: (text: string) => scanWithRebuff(text, 'output'),
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
createMCPScanner(_allowedTools?: string[]): MCPScanner {
|
|
269
|
+
// Rebuff has no MCP scanning capability
|
|
270
|
+
return {
|
|
271
|
+
start: async () => {},
|
|
272
|
+
stop: async () => {},
|
|
273
|
+
scanToolCall: () => ({ detected: false, matches: [] }),
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
createA2AScanner(_trustedAgents?: string[]): A2AScanner {
|
|
278
|
+
// Rebuff has no A2A scanning capability
|
|
279
|
+
return {
|
|
280
|
+
start: async () => {},
|
|
281
|
+
stop: async () => {},
|
|
282
|
+
scanMessage: () => ({ detected: false, matches: [] }),
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
createPatternScanner(): PatternScanner {
|
|
287
|
+
const patterns = getRebuffPatterns();
|
|
288
|
+
return {
|
|
289
|
+
scanText: (text: string, _pats: readonly ThreatPattern[]) => scanWithRebuff(text, 'input'),
|
|
290
|
+
getAllPatterns: () => patterns,
|
|
291
|
+
getPatternSets: () => ({
|
|
292
|
+
inputPatterns: patterns.filter(p => p.category !== 'output-leak'),
|
|
293
|
+
outputPatterns: patterns.filter(p => p.category === 'output-leak'),
|
|
294
|
+
mcpPatterns: [],
|
|
295
|
+
a2aPatterns: [],
|
|
296
|
+
}),
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
createBudgetManager(dataDir: string, config?: { budgetUsd?: number; maxCallsPerHour?: number }): BudgetManager {
|
|
301
|
+
// Rebuff has no budget management -- implement a simple one
|
|
302
|
+
let spent = 0;
|
|
303
|
+
let totalCalls = 0;
|
|
304
|
+
let callsThisHour = 0;
|
|
305
|
+
const budgetUsd = config?.budgetUsd ?? 5;
|
|
306
|
+
const maxCallsPerHour = config?.maxCallsPerHour ?? 20;
|
|
307
|
+
|
|
308
|
+
return {
|
|
309
|
+
canAfford: (cost: number) => spent + cost <= budgetUsd && callsThisHour < maxCallsPerHour,
|
|
310
|
+
record: (cost: number, _tokens: number) => { spent += cost; totalCalls++; callsThisHour++; },
|
|
311
|
+
getStatus: () => ({
|
|
312
|
+
spent,
|
|
313
|
+
budget: budgetUsd,
|
|
314
|
+
remaining: budgetUsd - spent,
|
|
315
|
+
percentUsed: Math.round((spent / budgetUsd) * 100),
|
|
316
|
+
callsThisHour,
|
|
317
|
+
maxCallsPerHour,
|
|
318
|
+
totalCalls,
|
|
319
|
+
}),
|
|
320
|
+
reset: () => { spent = 0; totalCalls = 0; callsThisHour = 0; },
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
createAnomalyScorer(): AnomalyScorer {
|
|
325
|
+
// Rebuff has no anomaly detection -- implement a stub
|
|
326
|
+
const baselines = new Map<string, { mean: number; stddev: number; count: number }>();
|
|
327
|
+
const observations = new Map<string, number[]>();
|
|
328
|
+
|
|
329
|
+
return {
|
|
330
|
+
score: () => 0,
|
|
331
|
+
record: (event: SecurityEvent) => {
|
|
332
|
+
const key = event.source;
|
|
333
|
+
if (!observations.has(key)) observations.set(key, []);
|
|
334
|
+
observations.get(key)!.push(1);
|
|
335
|
+
const vals = observations.get(key)!;
|
|
336
|
+
const mean = vals.length;
|
|
337
|
+
baselines.set(key, { mean, stddev: 0, count: 1 });
|
|
338
|
+
},
|
|
339
|
+
getBaseline: (source: string) => baselines.get(source) ?? null,
|
|
340
|
+
reset: () => { baselines.clear(); observations.clear(); },
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
}
|